Package com.backtype.hadoop.formats

Examples of com.backtype.hadoop.formats.RecordOutputStream

42
43
44
45
46
47
48
49
                    throw new IllegalArgumentException("Cannot write object " + obj.toString() + " to " + p.toString() +
                            ". Conflicts with the structure of the datastore.");
                }
                _workers.put(targetDir, Pail.super.openWrite(p.toString(), _overwrite));
            }
            RecordOutputStream os = _workers.get(targetDir);
            os.writeRaw(structure.serialize(obj));
        }
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
                rprtr.setStatus(status);

                RecordStreamFactory fact = args.streams;
                fs.mkdirs(finalFile.getParent());

                RecordOutputStream os = fact.getOutputStream(fs, tmpFile);
                for(Path i: sources) {
                    LOG.info("Opening " + i.toString() + " for consolidation");
                    RecordInputStream is = fact.getInputStream(fs, i);
                    byte[] record;
                    while((record = is.readRawRecord()) != null) {
                        os.writeRaw(record);
                    }
                    is.close();
                    rprtr.progress();
                }
                os.close();

                status = "Renaming " + tmpFile.toString() + " to " + finalFile.toString();
                LOG.info(status);
                rprtr.setStatus(status);
View Full Code Here
96
97
98
99
100
101
102
103
104
105
        Path metaPath = toStoredMetadataPath(metafilename);
        Path metaTmpPath = toStoredMetadataTmpPath(metafilename);
        mkdirs(metaTmpPath.getParent());
        delete(metaPath, false);
        delete(metaTmpPath, false);
        RecordOutputStream os = createOutputStream(metaTmpPath);
        os.writeRaw(("M" + metadata).getBytes("UTF-8")); //ensure that it's not an empty record
        os.close();
        rename(metaTmpPath, metaPath);
    }
View Full Code Here
54
55
56
57
58
59
60
61
                    throw new IllegalArgumentException("Cannot write object " + obj.toString() + " to " + p.toString() +
                            ". Conflicts with the structure of the datastore.");
                }
                _workers.put(targetDir, Pail.super.openWrite(p.toString(), _overwrite));
            }
            RecordOutputStream os = _workers.get(targetDir);
            os.writeRaw(structure.serialize(obj));
        }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    public static void deletePath(FileSystem fs, String path) throws IOException {
        fs.delete(new Path(path), true);
    }

    public static void emitToPail(Pail pail, String file, Iterable<String> records) throws IOException {
        RecordOutputStream os = pail.openWrite(file);
        for (String s: records) {
            os.writeRaw(s.getBytes());
        }
        os.close();
    }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
        }
        os.close();
    }

    public static void emitToPail(Pail pail, String file, String... records) throws IOException {
        RecordOutputStream os = pail.openWrite(file);
        for (String s: records) {
            os.writeRaw(s.getBytes());
        }
        os.close();
    }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        RecordStreamFactory factout;

        @Override
        protected void copyFile(FileSystem fsSource, Path source, FileSystem fsDest, Path target, Reporter reporter) throws IOException {
            RecordInputStream fin = factin.getInputStream(fsSource, source);
            RecordOutputStream fout = factout.getOutputStream(fsDest, target);

            try {
                byte[] record;
                int bytes = 0;
                while((record = fin.readRawRecord()) != null) {
                    fout.writeRaw(record);
                    bytes+=record.length;
                    if(bytes >= 1000000) { //every 1 MB of data report progress so we don't time out on large files
                        bytes = 0;
                        reporter.progress();
                    }
                }
            } finally {
                fin.close();
            }
            //don't complete files that aren't done yet. prevents partial files from being written
            fout.close();
        }
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
                rprtr.setStatus(status);

                RecordStreamFactory fact = args.streams;
                fs.mkdirs(finalFile.getParent());

                RecordOutputStream os = fact.getOutputStream(fs, tmpFile);
                for(Path i: sources) {
                    LOG.info("Opening " + i.toString() + " for consolidation");
                    RecordInputStream is = fact.getInputStream(fs, i);
                    byte[] record;
                    while((record = is.readRawRecord()) != null) {
                        os.writeRaw(record);
                    }
                    is.close();
                    rprtr.progress();
                }
                os.close();

                status = "Renaming " + tmpFile.toString() + " to " + finalFile.toString();
                LOG.info(status);
                rprtr.setStatus(status);
View Full Code Here
19
20
21
22
23
24
25
26
27
28
29
    private void writeStrings(Pail pail, String userfile, String... strs) throws IOException {
        writeStrings(pail, userfile, Arrays.asList(strs));
    }

    private void writeStrings(Pail pail, String userfile, Collection<String> strs) throws IOException {
        RecordOutputStream os = pail.openWrite(userfile);
        for(String s: strs) {
            os.writeRaw(s.getBytes());
        }
        os.close();
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
                .setArg(SequenceFileFormat.TYPE_ARG, SequenceFileFormat.TYPE_ARG_BLOCK)
                .setArg(SequenceFileFormat.CODEC_ARG, SequenceFileFormat.CODEC_ARG_DEFAULT));
    }

    private void emitToPail(Pail pail, String file, byte[]... records) throws Exception {
        RecordOutputStream os = pail.openWrite(file);
        for(byte[] r: records) {
            os.writeRaw(r);
        }
        os.close();
    }
View Full Code Here
TOP

Related Classes of com.backtype.hadoop.formats.RecordOutputStream

  • com.backtype.hadoop.Coercer$CoercerMapper
  • com.backtype.hadoop.Consolidator$ConsolidatorMapper
  • com.backtype.hadoop.pail.AbstractPail
  • com.backtype.hadoop.pail.Pail$TypedRecordOutputStream
  • com.backtype.hadoop.pail.PailOpsTest
  • com.backtype.hadoop.pail.PailTest
  • com.backtype.support.TestUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.