Examples of readRawRecord()

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

171
172
173
174
175
176
177
178
179
180
181
                RecordOutputStream os = fact.getOutputStream(fs, tmpFile);
                for(Path i: sources) {
                    LOG.info("Opening " + i.toString() + " for consolidation");
                    RecordInputStream is = fact.getInputStream(fs, i);
                    byte[] record;
                    while((record = is.readRawRecord()) != null) {
                        os.writeRaw(record);
                    }
                    is.close();
                    rprtr.progress();
                }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

106
107
108
109
110
111
112
113
114
115
116
    public String getMetadata(String metafilename) throws IOException {
        Path metaPath = toStoredMetadataPath(metafilename);
        if(exists(metaPath)) {
            RecordInputStream is = createInputStream(metaPath);
            String metaStr = new String(is.readRawRecord(), "UTF-8");
            is.close();
            return metaStr.substring(1);
        } else {
            return null;
        }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

85
86
87
88
89
90
91
92
93
94
95
    public static List<String> getPailRecords(Pail pail) throws IOException {
        List<String> ret = new ArrayList<String>();
        for(String s: pail.getUserFileNames()) {
            RecordInputStream is = pail.openRead(s);
            while(true) {
                byte[] r = is.readRawRecord();
                if(r==null) break;
                ret.add(new String(r));
            }
            is.close();
        }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

109
110
111
112
113
114
115
116
117
118
119
            RecordOutputStream fout = factout.getOutputStream(fsDest, target);

            try {
                byte[] record;
                int bytes = 0;
                while((record = fin.readRawRecord()) != null) {
                    fout.writeRaw(record);
                    bytes+=record.length;
                    if(bytes >= 1000000) { //every 1 MB of data report progress so we don't time out on large files
                        bytes = 0;
                        reporter.progress();
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

172
173
174
175
176
177
178
179
180
181
182
                RecordOutputStream os = fact.getOutputStream(fs, tmpFile);
                for(Path i: sources) {
                    LOG.info("Opening " + i.toString() + " for consolidation");
                    RecordInputStream is = fact.getInputStream(fs, i);
                    byte[] record;
                    while((record = is.readRawRecord()) != null) {
                        os.writeRaw(record);
                    }
                    is.close();
                    rprtr.progress();
                }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

199
200
201
202
203
204
205
206
207
208
209
        }
        assertTrue(parents.contains(""));
        assertTrue(parents.contains("1"));

        RecordInputStream in = p1.openRead("file1");
        String s = new String(in.readRawRecord());
        assertTrue(in.readRawRecord()==null);
        in.close();
        assertEquals("a", s);

    }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

200
201
202
203
204
205
206
207
208
209
210
        assertTrue(parents.contains(""));
        assertTrue(parents.contains("1"));

        RecordInputStream in = p1.openRead("file1");
        String s = new String(in.readRawRecord());
        assertTrue(in.readRawRecord()==null);
        in.close();
        assertEquals("a", s);

    }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

68
69
70
71
72
73
74
75
76
77
78
    private void checkContains(Pail pail, String file, byte[]... expected) throws Exception {
        RecordInputStream is = pail.openRead(file);
        List<byte[]> records = new ArrayList<byte[]>();
        while(true) {
            byte[] arr = is.readRawRecord();
            if(arr==null) break;
            records.add(arr);
        }
        assertEquals(expected.length, records.size());
        for(int i=0; i < expected.length; i++) {
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

239
240
241
242
243
244
245
246
247
248
249
    protected List<byte[]> getRecords(Pail p, String userfile) throws Exception {
        List<byte[]> ret = new ArrayList<byte[]>();
        RecordInputStream is = p.openRead(userfile);
        byte[] record;
        while((record = is.readRawRecord())!=null) {
            ret.add(record);
        }
        is.close();
        return ret;
    }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordInputStream.readRawRecord()

103
104
105
106
107
108
109
110
111
112
113
            RecordOutputStream fout = factout.getOutputStream(fsDest, target);

            try {
                byte[] record;
                int bytes = 0;
                while((record = fin.readRawRecord()) != null) {
                    fout.writeRaw(record);
                    bytes+=record.length;
                    if(bytes >= 1000000) { //every 1 MB of data report progress so we don't time out on large files
                        bytes = 0;
                        reporter.progress();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.