Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

32
33
34
35
36
37
38
39
40
        final Path file = super.getDefaultWorkFile(job, extension);
        final FileSystem fs = file.getFileSystem(conf);
        if (!isCompressed) {
            return new DataOutputStream(fs.create(file, false));
        } else {
            return new DataOutputStream(codec.createOutputStream(fs.create(file, false)));
        }
    }
}
View Full Code Here
55
56
57
58
59
60
61
62
63
64
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    if (!isCompressed) {
      return new BinaryRecordWriter<K, V>(fileOut);
    } else {
      return new BinaryRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)));
    }
  }

}
View Full Code Here
68
69
70
71
72
73
74
75
76
77
                               ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter(new DataOutputStream
                                  (codec.createOutputStream(fileOut)));
    }
  }     
}
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
            Configuration conf = new Configuration();
            CompressionCodec codec = (CompressionCodec)
              ReflectionUtils.newInstance(conf.getClassByName(codecClassName), conf);

            DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
            CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);

            byte[] data = new byte[1024];
            DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
            deflateOut.write(data, 0, data.length);
            deflateOut.flush();
View Full Code Here
118
119
120
121
122
123
124
125
126
127
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }     
}
View Full Code Here
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here
782
783
784
785
786
787
788
789
790
791
792
      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    File testFile = new File(System.getProperty("test.build.data"), "randdata");
    String fileName = testFile.getAbsolutePath();

    // Create the file
    OutputStream fos = new FileOutputStream(fileName);
    fos = codec.createOutputStream(fos);

    // Write file
    byte[] data = new byte[chunkSize];
    System.out.println("Start to write to file...");
    for (int i = 0; i < numChunks; i++) {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    DataOutputStream ostream = fileOut;

    if (isCompressed) {
      ostream = new DataOutputStream(codec.createOutputStream(fileOut));
    }

    return new RawKeyRecordWriter<K, V>(ostream);
  }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    fs.mkdirs(tablePath);
    OutputStream os = fs.create(filePath);
    if (gzip) {
      CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
      CompressionCodec codec = ccf.getCodec(filePath);
      os = codec.createOutputStream(os);
    }
    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < numRecords; i++) {
      w.write(getRecordLine(startId + i, extraCols));
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.