Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Metadata

939
940
941
942
943
944
945
     *          the file name
     * @throws IOException
     */
    public Writer(FileSystem fs, Configuration conf, Path name,
        Progressable progress, CompressionCodec codec) throws IOException {
      this(fs, conf, name, progress, new Metadata(), codec);
    }
View Full Code Here
987
988
989
990
991
992
993
994
995
996
997
        Metadata metadata, CompressionCodec codec) throws IOException {
      RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL);
      columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0);

      if (metadata == null) {
        metadata = new Metadata();
      }
      metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text(""
          + columnNumber));

      columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR,
View Full Code Here
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
              "Unknown codec: " + codecClassname, cnfe);
        }
        keyDecompressor = CodecPool.getDecompressor(codec);
      }

      metadata = new Metadata();
      metadata.readFields(in);

      in.readFully(sync); // read sync bytes
      headerEnd = in.getPos();
    }
View Full Code Here
593
594
595
596
597
598
599
      return out.getPos();
    }

    /** Constructs a RCFile Writer. */
    public Writer(FileSystem fs, Configuration conf, Path name) throws IOException {
      this(fs, conf, name, null, new Metadata(), null);
    }
View Full Code Here
609
610
611
612
613
614
615
     *          the file name
     * @throws IOException
     */
    public Writer(FileSystem fs, Configuration conf, Path name,
        Progressable progress, CompressionCodec codec) throws IOException {
      this(fs, conf, name, null, new Metadata(), codec);
    }
View Full Code Here
656
657
658
659
660
661
662
663
664
665
666
        Metadata metadata, CompressionCodec codec) throws IOException {
      RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL);
      columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0);

      if (metadata == null) {
        metadata = new Metadata();
      }
      metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text(""
          + columnNumber));

      columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR,
View Full Code Here
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
              "Unknown codec: " + codecClassname, cnfe);
        }
        keyDecompressor = CodecPool.getDecompressor(codec);
      }

      metadata = new Metadata();
      if (version >= VERSION_WITH_METADATA) { // if version >= 6
        metadata.readFields(in);
      }

      if (version > 1) { // if version > 1
View Full Code Here
529
530
531
532
533
534
535
        "createWriterOnExistingFile") , "file");

    fs.create(name);
    SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
        RandomDatum.class, 512, (short) 1, 4096, false,
        CompressionType.NONE, null, new Metadata());
  }
View Full Code Here
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
    boolean createParent = false;

    try {
      SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
          RandomDatum.class, 512, (short) 1, 4096, createParent,
          CompressionType.NONE, null, new Metadata());
      fail("Expected an IOException due to missing parent");
    } catch (IOException ioe) {
      // Expected
    }

    createParent = true;
    SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
        RandomDatum.class, 512, (short) 1, 4096, createParent,
        CompressionType.NONE, null, new Metadata());
    // should succeed, fails if exception thrown
  }
View Full Code Here
105
106
107
108
109
110
111
    metaMap.put(WAL_VERSION_KEY, WAL_VERSION);
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Metadata

  • com.twitter.elephantbird.mapreduce.output.RCFileOutputFormat
  • com.twitter.elephantbird.util.RCFileUtil
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.avro.hadoop.io.AvroSequenceFile$Reader$Options
  • org.apache.avro.hadoop.io.AvroSequenceFile$Writer$Options
  • org.apache.hadoop.hbase.regionserver.HLog
  • org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter
  • org.apache.hadoop.hive.ql.io.RCFile
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.