Package org.apache.tools.bzip2r

Examples of org.apache.tools.bzip2r.CBZip2OutputStream

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
       
        File out = File.createTempFile("junit", ".bz2");
        out.delete();
        String clusterOutput = Util.removeColon(out.getAbsolutePath());
              
        CBZip2OutputStream cos =
            new CBZip2OutputStream(new FileOutputStream(in));
        for (int i = 1; i < 100; i++) {
            StringBuffer sb = new StringBuffer();
            sb.append(i).append("\n").append(-i).append("\n");
            byte bytes[] = sb.toString().getBytes();
            cos.write(bytes);
        }
        cos.close();
                      
        pig.registerQuery("AA = load '"
                + Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
                + "';");
        pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
        File in = File.createTempFile("junit", ".bz2");
        String compressedInputFileName = in.getAbsolutePath();
        in.deleteOnExit();
       
        try {
            CBZip2OutputStream cos =
                new CBZip2OutputStream(new FileOutputStream(in));
            for (int i = 0; i < inputData.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos.write(bytes);
            }
            cos.close();
           
            Util.copyFromLocalToCluster(cluster, compressedInputFileName,
                    compressedInputFileName);
           
            // pig script to read compressed input
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
        String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
        Util.createInputFile(cluster, unCompressedInputFileName, inputData);
       
        try {
            CBZip2OutputStream cos =
                new CBZip2OutputStream(new FileOutputStream(in));
            for (int i = 0; i < inputData.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos.write(bytes);
            }
            cos.close();
           
            Util.copyFromLocalToCluster(cluster, compressedInputFileName,
                    clusterCompressedFilePath);
           
            // pig script to read uncompressed input
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
354
355
     */
    @Test
    public void testEmptyBzip() throws Exception {
        File tmp = File.createTempFile("junit", ".tmp");
        tmp.deleteOnExit();
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        FileSystem fs = FileSystem.getLocal(new Configuration(false));
        CBZip2InputStream cis = new CBZip2InputStream(
                fs.open(new Path(tmp.getAbsolutePath())), -1, tmp.length());
        assertEquals(-1, cis.read(new byte[100]));
View Full Code Here
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
        String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
        Util.createInputFile(cluster, unCompressedInputFileName, inputDataMerged);
       
        try {
            CBZip2OutputStream cos =
                new CBZip2OutputStream(new FileOutputStream(in1));
            for (int i = 0; i < inputData1.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData1[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos.write(bytes);
            }
            cos.close();
           
            CBZip2OutputStream cos2 =
                new CBZip2OutputStream(new FileOutputStream(in2));
            for (int i = 0; i < inputData2.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData2[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos2.write(bytes);
            }
            cos2.close();

            // cat
            catInto(compressedInputFileName2, compressedInputFileName1);
            Util.copyFromLocalToCluster(cluster, compressedInputFileName1,
                    compressedInputFileName1);
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
        File in = File.createTempFile("junit", ".bz");
        in.deleteOnExit();
        File out = File.createTempFile("junit", ".bz");
        out.deleteOnExit();
        out.delete();
        CBZip2OutputStream cos = new CBZip2OutputStream(
                new FileOutputStream(in));
        for (int i = 1; i < 100; i++) {
            cos.write((i + "\n").getBytes());
            cos.write((-i + "\n").getBytes());
        }
        cos.close();
        pig.registerQuery("AA=load '" + Util.generateURI(in.getAbsolutePath()) + "';");
        pig.registerQuery("A=foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
        pig.store("A", Util.generateURI(out.getAbsolutePath()));
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(new File(out, "part-00000.bz")));
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
     */
    @Test
    public void testEmptyBzip() throws Exception {
        File tmp = File.createTempFile("junit", ".tmp");
        tmp.deleteOnExit();
        CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(
                tmp));
        cos.close();
        assertNotSame(0, tmp.length());
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(tmp));
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
        File out = File.createTempFile("junit", ".bz2");
        out.delete();
        String clusterOutput = Util.removeColon(out.getAbsolutePath());

        CBZip2OutputStream cos =
            new CBZip2OutputStream(new FileOutputStream(in));
        for (int i = 1; i < 100; i++) {
            StringBuffer sb = new StringBuffer();
            sb.append(i).append("\n").append(-i).append("\n");
            byte bytes[] = sb.toString().getBytes();
            cos.write(bytes);
        }
        cos.close();

        pig.registerQuery("AA = load '"
                + Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
                + "';");
        pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
        File out = File.createTempFile("junit", ".bz2");
        out.delete();
        String clusterOutput = Util.removeColon(out.getAbsolutePath());

        CBZip2OutputStream cos =
            new CBZip2OutputStream(new FileOutputStream(in));
        for (int i = 1; i < 100; i++) {
            StringBuffer sb = new StringBuffer();
            sb.append(i).append("\n").append(-i).append("\n");
            byte bytes[] = sb.toString().getBytes();
            cos.write(bytes);
        }
        cos.close();

        pig.registerQuery("AA = load '"
                + Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
                + "';");
        pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        String compressedInputFileName = in.getAbsolutePath();
        String clusterCompressedFilePath = Util.removeColon(compressedInputFileName);
        in.deleteOnExit();

        try {
            CBZip2OutputStream cos =
                new CBZip2OutputStream(new FileOutputStream(in));
            for (int i = 0; i < inputData.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos.write(bytes);
            }
            cos.close();

            Util.copyFromLocalToCluster(cluster, compressedInputFileName,
                    clusterCompressedFilePath);

            // pig script to read compressed input
View Full Code Here
TOP

Related Classes of org.apache.tools.bzip2r.CBZip2OutputStream

  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigOutputFormat$PigRecordWriter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter
  • org.apache.pig.piggybank.storage.MultiStorage
  • org.apache.pig.test.TestBZip
  • org.apache.pig.test.TestPigLineRecordReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.