Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Metadata

450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
    boolean createParent = false;

    try {
      SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
          RandomDatum.class, 512, (short) 1, 4096, createParent,
          CompressionType.NONE, null, new Metadata());
      fail("Expected an IOException due to missing parent");
    } catch (IOException ioe) {
      // Expected
    }

    createParent = true;
    SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
        RandomDatum.class, 512, (short) 1, 4096, createParent,
        CompressionType.NONE, null, new Metadata());

    // should succeed, fails if exception thrown
  }
View Full Code Here
796
797
798
799
800
801
802
803
804
805
806
    if (values.length % 2 != 0) {
      throw new IllegalArgumentException("Must have a matched set of " +
                                         "key-value pairs. " + values.length+
                                         " strings supplied.");
    }
    Metadata result = new Metadata();
    for(int i=0; i < values.length; i += 2) {
      result.set(values[i], values[i+1]);
    }
    return result;
  }
View Full Code Here
925
926
927
928
929
930
931
      return out.getPos();
    }

    /** Constructs a RCFile Writer. */
    public Writer(FileSystem fs, Configuration conf, Path name) throws IOException {
      this(fs, conf, name, null, new Metadata(), null);
    }
View Full Code Here
941
942
943
944
945
946
947
     *          the file name
     * @throws IOException
     */
    public Writer(FileSystem fs, Configuration conf, Path name,
        Progressable progress, CompressionCodec codec) throws IOException {
      this(fs, conf, name, progress, new Metadata(), codec);
    }
View Full Code Here
989
990
991
992
993
994
995
996
997
998
999
        Metadata metadata, CompressionCodec codec) throws IOException {
      RECORD_INTERVAL = HiveConf.getIntVar(conf, HIVE_RCFILE_RECORD_INTERVAL);
      columnNumber = HiveConf.getIntVar(conf, HIVE_RCFILE_COLUMN_NUMBER_CONF);

      if (metadata == null) {
        metadata = new Metadata();
      }
      metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text(""
          + columnNumber));

      columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR,
View Full Code Here
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
              "Unknown codec: " + codecClassname, cnfe);
        }
        keyDecompressor = CodecPool.getDecompressor(codec);
      }

      metadata = new Metadata();
      metadata.readFields(in);

      in.readFully(sync); // read sync bytes
      headerEnd = in.getPos();
    }
View Full Code Here
78
79
80
81
82
83
84
    metaMap.put(WAL_VERSION_KEY, new Text("1"));
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
794
795
796
797
798
799
800
801
802
803
804
    if (values.length % 2 != 0) {
      throw new IllegalArgumentException("Must have a matched set of " +
                                         "key-value pairs. " + values.length+
                                         " strings supplied.");
    }
    Metadata result = new Metadata();
    for(int i=0; i < values.length; i += 2) {
      result.set(values[i], values[i+1]);
    }
    return result;
  }
View Full Code Here
923
924
925
926
927
928
929
      return out.getPos();
    }

    /** Constructs a RCFile Writer. */
    public Writer(FileSystem fs, Configuration conf, Path name) throws IOException {
      this(fs, conf, name, null, new Metadata(), null);
    }
View Full Code Here
939
940
941
942
943
944
945
     *          the file name
     * @throws IOException
     */
    public Writer(FileSystem fs, Configuration conf, Path name,
        Progressable progress, CompressionCodec codec) throws IOException {
      this(fs, conf, name, progress, new Metadata(), codec);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Metadata

  • com.twitter.elephantbird.mapreduce.output.RCFileOutputFormat
  • com.twitter.elephantbird.util.RCFileUtil
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.avro.hadoop.io.AvroSequenceFile$Reader$Options
  • org.apache.avro.hadoop.io.AvroSequenceFile$Writer$Options
  • org.apache.hadoop.hbase.regionserver.HLog
  • org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter
  • org.apache.hadoop.hive.ql.io.RCFile
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.