Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec

1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
   *          Path to be created
   * @return output stream over the created rcfile
   */
  public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file,
      boolean isCompressed) throws IOException {
    CompressionCodec codec = null;
    Class<?> codecClass = null;
    if (isCompressed) {
      codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
    }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);

      processINodesUC(in, v, skipBlocks);
View Full Code Here
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
    throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();

      Writer writer = createWriter(outputFile.getFileSystem(conf), conf,
                                   outputFile, keyClass, valClass, compress,
                                   blockCompress, codec, prog,
View Full Code Here
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        boolean isCompressed = false;
        boolean isBlockCompressed = false;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }
       
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
                                      String name, Progressable progress)
    throws IOException {

    Path file = new Path(job.getOutputPath(), name);
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = getOutputCompressionType(job);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
    throws IOException {
    long start = split.getStart();
    long end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    InputStream in = fileIn;
    boolean skipFirstLine = false;
    if (codec != null) {
      in = codec.createInputStream(fileIn);
      end = Long.MAX_VALUE;
    } else if (start != 0) {
      skipFirstLine = true;  // wait till BufferedInputStream to skip
      --start;
      fileIn.seek(start);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
      FSDataOutputStream fileOut = fs.create(new Path(dir, name), progress);
      return new LineRecordWriter<K, V>(fileOut);
    } else {
      Class codecClass = getOutputCompressorClass(job, GzipCodec.class);
      // create the named codec
      CompressionCodec codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }     
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
                                      String name, Progressable progress)
    throws IOException {

    Path file = new Path(job.getOutputPath(), name);
   
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);
View Full Code Here
578
579
580
581
582
583
584
585
586
587
588
    Path testDir = new Path(System.getProperty("test.data.dir", ".")
        + "/mapred/testsync");
    Path testFile = new Path(testDir, "test_rcfile");
    fs.delete(testFile, true);
    int intervalRecordCount = 500;
    CompressionCodec codec = null;
    int writeCount = 2500;
    Configuration cloneConf = new Configuration(conf);
    RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length);
    cloneConf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, intervalRecordCount);
    RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);   
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
      PerformTestRCFileAndSeqFile testcase = new PerformTestRCFileAndSeqFile(
          isLocalFS, file);

      // change these parameters
      boolean checkCorrect = true;
      CompressionCodec codec = new DefaultCodec();
      testcase.columnMaxSize = 30;

      // testcase.testWithColumnNumber(count, 2, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 10, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 25, checkCorrect, codec);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.compression.lzo.TestLzoCodec
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.