Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Metadata

747
748
749
750
751
752
753
754
755
756
757
        Metadata metadata, CompressionCodec codec) throws IOException {
      RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL);
      columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0);

      if (metadata == null) {
        metadata = new Metadata();
      }
      metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text(""
          + columnNumber));

      columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR,
View Full Code Here
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
              "Unknown codec: " + codecClassname, cnfe);
        }
        keyDecompressor = CodecPool.getDecompressor(codec);
      }

      metadata = new Metadata();
      if (version >= VERSION_WITH_METADATA) { // if version >= 6
        metadata.readFields(in);
      }

      if (version > 1) { // if version > 1
View Full Code Here
106
107
108
109
110
111
112
    metaMap.put(WAL_VERSION_KEY, WAL_VERSION);
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
78
79
80
81
82
83
84
    metaMap.put(WAL_VERSION_KEY, new Text("1"));
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
      conf.getLong("hbase.regionserver.hlog.blocksize",
      fs.getDefaultBlockSize()),
      SequenceFile.CompressionType.NONE,
      new DefaultCodec(),
      null,
      new Metadata());

    // Get at the private FSDataOutputStream inside in SequenceFile so we can
    // call sync on it.  Make it accessible.  Stash it aside for call up in
    // the sync method.
    final Field fields [] = this.writer.getClass().getDeclaredFields();
View Full Code Here
534
535
536
537
538
539
540
        "createWriterOnExistingFile") , "file");

    fs.create(name);
    SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
        RandomDatum.class, 512, (short) 1, 4096, false,
        CompressionType.NONE, null, new Metadata());
  }
View Full Code Here
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
    boolean createParent = false;

    try {
      SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
          RandomDatum.class, 512, (short) 1, 4096, createParent,
          CompressionType.NONE, null, new Metadata());
      fail("Expected an IOException due to missing parent");
    } catch (IOException ioe) {
      // Expected
    }

    createParent = true;
    SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
        RandomDatum.class, 512, (short) 1, 4096, createParent,
        CompressionType.NONE, null, new Metadata());
    // should succeed, fails if exception thrown
  }
View Full Code Here
77
78
79
80
81
82
83
    metaMap.put(WAL_VERSION_KEY, new Text("1"));
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
794
795
796
797
798
799
800
801
802
803
804
    if (values.length % 2 != 0) {
      throw new IllegalArgumentException("Must have a matched set of " +
                                         "key-value pairs. " + values.length+
                                         " strings supplied.");
    }
    Metadata result = new Metadata();
    for(int i=0; i < values.length; i += 2) {
      result.set(values[i], values[i+1]);
    }
    return result;
  }
View Full Code Here
923
924
925
926
927
928
929
      return out.getPos();
    }

    /** Constructs a RCFile Writer. */
    public Writer(FileSystem fs, Configuration conf, Path name) throws IOException {
      this(fs, conf, name, null, new Metadata(), null);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Metadata

  • com.twitter.elephantbird.mapreduce.output.RCFileOutputFormat
  • com.twitter.elephantbird.util.RCFileUtil
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.avro.hadoop.io.AvroSequenceFile$Reader$Options
  • org.apache.avro.hadoop.io.AvroSequenceFile$Writer$Options
  • org.apache.hadoop.hbase.regionserver.HLog
  • org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter
  • org.apache.hadoop.hive.ql.io.RCFile
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.