Examples of org.apache.hadoop.io.compress.DefaultCodec

org.apache.hadoop.io.compress.DefaultCodec

      new RCFile.Writer(fs, conf, file, null,
                        RCFile.createMetadata(new Text("apple"),
                                              new Text("block"),
                                              new Text("cat"),
                                              new Text("dog")),
                        new DefaultCodec());
    
    byte[][] record_1 = {
        "123".getBytes("UTF-8"), 
        "456".getBytes("UTF-8"),
        "789".getBytes("UTF-8"),

View Full Code Here


    byte[][] record = {null, null, null, null, null, null, null, null};


    RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);
    final int recCount = 100;
    Random rand = new Random();
    for (int recIdx = 0; recIdx < recCount; recIdx++) {
      for (int i = 0; i < record.length; i++) {

View Full Code Here

      byte[][] fieldsData, Configuration conf) throws IOException, SerDeException {
    fs.delete(file, true);


    RCFileOutputFormat.setColumnNumber(conf, fieldsData.length);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());


    BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length);
    for (int i = 0; i < fieldsData.length; i++) {
      BytesRefWritable cu = null;
      cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length);

View Full Code Here

      PerformTestRCFileAndSeqFile testcase = new PerformTestRCFileAndSeqFile(
          isLocalFS, file);


      // change these parameters
      boolean checkCorrect = true;
      CompressionCodec codec = new DefaultCodec();
      testcase.columnMaxSize = 30;


      // testcase.testWithColumnNumber(count, 2, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 10, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 25, checkCorrect, codec);

View Full Code Here

                  WritableComparator comparator, Class valClass,
                  SequenceFile.CompressionType compress,
                  Progressable progress)
      throws IOException {
      this(conf, fs, dirName, comparator, valClass, 
           compress, new DefaultCodec(), progress);
    }

View Full Code Here

  public TestSequenceFile(String name) { super(name); }


  /** Unit tests for SequenceFile. */
  public void testZlibSequenceFile() throws Exception {
    LOG.info("Testing SequenceFile with DefaultCodec");
    compressedSeqFileTest(new DefaultCodec());
    LOG.info("Successfully tested SequenceFile with DefaultCodec");
  }

View Full Code Here

  public void testSequenceFileMetadata() throws Exception {
    LOG.info("Testing SequenceFile with metadata");
    int count = 1024 * 10;
    int megabytes = 1;
    int factor = 5;
    CompressionCodec codec = new DefaultCodec();
    Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata");
    Path recordCompressedFile = 
      new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata");
    Path blockCompressedFile = 
      new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata");

View Full Code Here

            for(int pi = 0; pi < 5; pi++){
                Path path = new Path(new Path(file.getAbsolutePath()), 
                        "parition" + pi);


                datePartitionedRowCount += 
                    writeRCFileTest(fs, simpleRowCount, path, columnCount, new DefaultCodec(), columnCount);


                new File(path.toString()).deleteOnExit();
                datePartitionedRCFiles.add(path.toString());
                datePartitionedDirs.add(file.toString());

View Full Code Here

        simpleDataFile = File.createTempFile("testhiveColumnarLoader", ".txt");
        simpleDataFile.deleteOnExit();


        Path path = new Path(simpleDataFile.getPath());


        writeRCFileTest(fs, simpleRowCount, path, columnCount, new DefaultCodec(), columnCount);


    }

View Full Code Here

                  WritableComparator comparator, Class valClass,
                  SequenceFile.CompressionType compress,
                  Progressable progress)
      throws IOException {
      this(conf, fs, dirName, comparator, valClass, 
           compress, new DefaultCodec(), progress);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.compress.DefaultCodec

com.alexholmes.hadooputils.combine.avro.AvroFileGenerator

com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileTest

com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileTest

com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator

com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest

com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest

com.cloudera.flume.handlers.hdfs.TestDFSWrite

com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs

com.m6d.filecrush.crush.CrushReducerParameterizedTest

com.m6d.filecrush.crush.integration.CrushMapReduceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.