Package org.apache.tools.bzip2r

Examples of org.apache.tools.bzip2r.CBZip2InputStream

677
678
679
680
681
682
683
684
685
686
687
                ps.println("");
            }
            ps.close();

            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( bzis );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );

            Text value = new Text();
            int counter = 0;
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
        FileSpec lFile = new FileSpec((String)idxEntry.get(keysCnt-2),this.rightLoaderFuncSpec);
        currentFileName = lFile.getFileName();
        loader = (LoadFunc)PigContext.instantiateFuncFromSpec(lFile.getFuncSpec());
        is = FileLocalizer.open(currentFileName, offset, pc);
        if (currentFileName.endsWith(".bz") || currentFileName.endsWith(".bz2")) {
            is = new CBZip2InputStream((SeekableInputStream)is, 9);
        } else if (currentFileName.endsWith(".gz")) {
            is = new GZIPInputStream(is);
        }

       
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    
        fsis = base.asElement(base.getActiveContainer(), file).sopen();
        fsis.seek(start, FLAGS.SEEK_CUR);

        if (file.endsWith(".bz") || file.endsWith(".bz2")) {
            is = new CBZip2InputStream(fsis, 9);
        } else if (file.endsWith(".gz")) {
            is = new GZIPInputStream(fsis);
            // We can't tell how much of the underlying stream GZIPInputStream
            // has actually consumed
            end = Long.MAX_VALUE;
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(job);
            FSDataInputStream fileIn = fs.open(split.getPath());
            fileIn.seek(start);

            in = new CBZip2InputStream(fileIn, 9, end);
            if (start != 0) {
                // skip first line and re-establish "start".
                // LineRecordReader.readLine(this.in, null);
                readLine(this.in, null);
                start = in.getPos();
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutput +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
       
        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "/';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutput +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
       
        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutputFilePath) + "';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutputFilePath +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutputFilePath) + "';");
        pig.openIterator("B");
       
        in.delete();
View Full Code Here
349
350
351
352
353
354
355
356
357
358
359
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        FileSystem fs = FileSystem.getLocal(new Configuration(false));
        CBZip2InputStream cis = new CBZip2InputStream(
                fs.open(new Path(tmp.getAbsolutePath())), -1, tmp.length());
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        tmp.delete();
    }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutput +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
       
        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "/';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutput +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
       
        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
TOP

Related Classes of org.apache.tools.bzip2r.CBZip2InputStream

  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.bzip2r.Bzip2TextInputFormat$BZip2LineRecordReader
  • org.apache.pig.impl.builtin.DefaultIndexableLoader
  • org.apache.pig.piggybank.storage.XMLLoader$XMLFileRecordReader
  • org.apache.pig.test.TestBZip
  • org.apache.pig.test.TestPigLineRecordReader
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.