Package org.apache.tools.bzip2r

Examples of org.apache.tools.bzip2r.CBZip2InputStream

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
        pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutputFilePath) + "';");
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FSDataInputStream is = fs.open(new Path(clusterOutputFilePath +
                "/part-r-00000.bz2"));
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
       
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
       
        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutputFilePath) + "';");
        pig.openIterator("B");
       
        in.delete();
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        FileSystem fs = FileSystem.getLocal(new Configuration(false));
        CBZip2InputStream cis = new CBZip2InputStream(
                fs.open(new Path(tmp.getAbsolutePath())), -1, tmp.length());
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        tmp.delete();
    }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        }
        cos.close();
        pig.registerQuery("AA=load '" + Util.generateURI(in.getAbsolutePath()) + "';");
        pig.registerQuery("A=foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
        pig.store("A", Util.generateURI(out.getAbsolutePath()));
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(new File(out, "part-00000.bz")));
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();
        pig.registerQuery("B=load '" + Util.generateURI(out.getAbsolutePath()) + "';");
        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
        while (i.hasNext()) {
            Integer val = DataType.toInteger(i.next().get(0));
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
        System.out.println(in.getAbsolutePath());
        pig.registerQuery("AA=load '" + Util.generateURI(in.getAbsolutePath()) + "';");
        pig
                .registerQuery("A=foreach (group (filter AA by $0 < '0') all) generate flatten($1);");
        pig.store("A", Util.generateURI(out.getAbsolutePath()));
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(new File(out, "part-00000.bz")));
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        pig.registerQuery("B=load '" + Util.generateURI(out.getAbsolutePath()) + "';");
        pig.openIterator("B");
        in.delete();
        out.delete();
    }
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
        tmp.deleteOnExit();
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(tmp));
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        tmp.delete();

    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FileStatus[] outputFiles = fs.listStatus(new Path(clusterOutput),
                Util.getSuccessMarkerPathFilter());
        FSDataInputStream is = fs.open(outputFiles[0].getPath());
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());

        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();

        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");

        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FileStatus[] outputFiles = fs.listStatus(new Path(clusterOutput),
                Util.getSuccessMarkerPathFilter());
        FSDataInputStream is = fs.open(outputFiles[0].getPath());
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());

        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
        cis.close();

        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");

        Iterator<Tuple> i = pig.openIterator("B");
        HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FileStatus[] outputFiles = fs.listStatus(new Path(clusterOutputFilePath),
                Util.getSuccessMarkerPathFilter());
        FSDataInputStream is = fs.open(outputFiles[0].getPath());
        CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());

        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();

        pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutputFilePath) + "';");
        pig.openIterator("B");

        in.delete();
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        FileSystem fs = FileSystem.getLocal(new Configuration(false));
        CBZip2InputStream cis = new CBZip2InputStream(
                fs.open(new Path(tmp.getAbsolutePath())), -1, tmp.length());
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        tmp.delete();
    }
View Full Code Here
655
656
657
658
659
660
661
662
663
664
665
            fileIn.seek(start);
       
            if(file.toString().endsWith(".bz2") || file.toString().endsWith(".bz"))
            {
              // For bzip2 files use CBZip2InputStream to read and supply the upper input stream.
               CBZip2InputStream in = new CBZip2InputStream(fileIn,9, end);
               this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(in,start,end);
            }
            else if (file.toString().endsWith(".gz"))
            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
View Full Code Here
TOP

Related Classes of org.apache.tools.bzip2r.CBZip2InputStream

  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.bzip2r.Bzip2TextInputFormat$BZip2LineRecordReader
  • org.apache.pig.impl.builtin.DefaultIndexableLoader
  • org.apache.pig.piggybank.storage.XMLLoader$XMLFileRecordReader
  • org.apache.pig.test.TestBZip
  • org.apache.pig.test.TestPigLineRecordReader
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.