Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.DefaultCodec

42
43
44
45
46
47
48
49
  public TestSequenceFile(String name) { super(name); }

  /** Unit tests for SequenceFile. */
  public void testZlibSequenceFile() throws Exception {
    LOG.info("Testing SequenceFile with DefaultCodec");
    compressedSeqFileTest(new DefaultCodec());
    LOG.info("Successfully tested SequenceFile with DefaultCodec");
  }
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
  public void testSequenceFileMetadata() throws Exception {
    LOG.info("Testing SequenceFile with metadata");
    int count = 1024 * 10;
    int megabytes = 1;
    int factor = 5;
    CompressionCodec codec = new DefaultCodec();
    Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata");
    Path recordCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata");
    Path blockCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata");
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
   
    if (compressionType == CompressionType.NONE) {
      writer = new Writer(fs, conf, name, keyClass, valClass, null, new Metadata());
    } else if (compressionType == CompressionType.RECORD) {
      writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass,
                                        new DefaultCodec());
    } else if (compressionType == CompressionType.BLOCK){
      writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass,
                                       new DefaultCodec());
    }
   
    return writer;
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
   
    if (compressionType == CompressionType.NONE) {
      writer = new Writer(fs, conf, name, keyClass, valClass, progress, new Metadata());
    } else if (compressionType == CompressionType.RECORD) {
      writer = new RecordCompressWriter(fs, conf, name,
                                        keyClass, valClass, new DefaultCodec(), progress, new Metadata());
    } else if (compressionType == CompressionType.BLOCK){
      writer = new BlockCompressWriter(fs, conf, name,
                                       keyClass, valClass, new DefaultCodec(), progress, new Metadata());
    }
   
    return writer;
  }
View Full Code Here
296
297
298
299
300
301
302
                 Class keyClass, Class valClass, CompressionType compressionType)
    throws IOException {
    return createWriter(fs, conf, name, keyClass, valClass,
            fs.getConf().getInt("io.file.buffer.size", 4096),
            fs.getDefaultReplication(), fs.getDefaultBlockSize(),
            compressionType, new DefaultCodec(), null, new Metadata());
  }
View Full Code Here
318
319
320
321
322
323
324
                 Class keyClass, Class valClass, CompressionType compressionType,
                 Progressable progress) throws IOException {
    return createWriter(fs, conf, name, keyClass, valClass,
            fs.getConf().getInt("io.file.buffer.size", 4096),
            fs.getDefaultReplication(), fs.getDefaultBlockSize(),
            compressionType, new DefaultCodec(), progress, new Metadata());
  }
View Full Code Here
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
          } catch (ClassNotFoundException cnfe) {
            throw new IllegalArgumentException("Unknown codec: " +
                                               codecClassname, cnfe);
          }
        } else {
          codec = new DefaultCodec();
          ((Configurable)codec).setConf(conf);
        }
      }
     
      this.metadata = new Metadata();
View Full Code Here
150
151
152
153
154
155
156
                  WritableComparator comparator, Class valClass,
                  SequenceFile.CompressionType compress,
                  Progressable progress)
      throws IOException {
      this(conf, fs, dirName, comparator, valClass,
           compress, new DefaultCodec(), progress);
    }
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
        this.writer = SequenceFile.createWriter(this.fs, this.conf, newPath,
          HLogKey.class, HLogEdit.class,
          fs.getConf().getInt("io.file.buffer.size", 4096),
          fs.getDefaultReplication(), this.blocksize,
          SequenceFile.CompressionType.NONE, new DefaultCodec(), null,
          new Metadata());

        LOG.info((oldFile != null?
          "Closed " + oldFile + ", entries=" + this.numEntries + ". ": "") +
          "New log writer: " + FSUtils.getPath(newPath));
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
        new byte[0], "\\N".getBytes("UTF-8")};

    RCFileOutputFormat.setColumnNumber(conf, 8);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
    for (int i = 0; i < record_1.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
          record_1[i].length);
      bytes.set(i, cu);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.DefaultCodec

  • com.alexholmes.hadooputils.combine.avro.AvroFileGenerator
  • com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator
  • com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest
  • com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest
  • com.cloudera.flume.handlers.hdfs.TestDFSWrite
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.