Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Metadata

711
712
713
714
715
716
717
     *          the file name
     * @throws IOException
     */
    public Writer(FileSystem fs, Configuration conf, Path name,
        Progressable progress, CompressionCodec codec) throws IOException {
      this(fs, conf, name, progress, new Metadata(), codec);
    }
View Full Code Here
758
759
760
761
762
763
764
765
766
767
768
        Metadata metadata, CompressionCodec codec) throws IOException {
      RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL);
      columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0);

      if (metadata == null) {
        metadata = new Metadata();
      }
      metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text(""
          + columnNumber));

      columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR,
View Full Code Here
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
              "Unknown codec: " + codecClassname, cnfe);
        }
        keyDecompressor = CodecPool.getDecompressor(codec);
      }

      metadata = new Metadata();
      metadata.readFields(in);

      in.readFully(sync); // read sync bytes
      headerEnd = in.getPos();
    }
View Full Code Here
77
78
79
80
81
82
83
    metaMap.put(WAL_VERSION_KEY, new Text("1"));
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
              fs.getDefaultReplication())),
            new Long(conf.getLong("hbase.regionserver.hlog.blocksize",
                fs.getDefaultBlockSize())),
            new Boolean(false) /*createParent*/,
            SequenceFile.CompressionType.NONE, new DefaultCodec(),
            new Metadata()
            });
    } catch (InvocationTargetException ite) {
      // function was properly called, but threw it's own exception
      throw new IOException(ite.getCause());
    } catch (Exception e) {
      // ignore all other exceptions. related to reflection failure
    }

    // if reflection failed, use the old createWriter
    if (this.writer == null) {
      LOG.debug("new createWriter -- HADOOP-6840 -- not available");
      this.writer = SequenceFile.createWriter(fs, conf, path,
        HLog.getKeyClass(conf), WALEdit.class,
        fs.getConf().getInt("io.file.buffer.size", 4096),
        (short) conf.getInt("hbase.regionserver.hlog.replication",
          fs.getDefaultReplication()),
        conf.getLong("hbase.regionserver.hlog.blocksize",
          fs.getDefaultBlockSize()),
        SequenceFile.CompressionType.NONE,
        new DefaultCodec(),
        null,
        new Metadata());
    } else {
      LOG.debug("using new createWriter -- HADOOP-6840");
    }
   
    this.writer_out = getSequenceFilePrivateFSDataOutputStreamAccessible();
View Full Code Here
352
353
354
355
356
357
358
      throws IOException {
    return SequenceFile.createWriter(this.fs, this.conf, path, keyClass,
        valueClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs
            .getDefaultReplication(), this.blocksize,
        SequenceFile.CompressionType.NONE, new DefaultCodec(), null,
        new Metadata());
  }
View Full Code Here
78
79
80
81
82
83
84
    metaMap.put(WAL_VERSION_KEY, new Text("1"));
    if (compress) {
      // Currently we only do one compression type.
      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
    }
    return new Metadata(metaMap);
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
      conf.getLong("hbase.regionserver.hlog.blocksize",
        fs.getDefaultBlockSize()),
      SequenceFile.CompressionType.NONE,
      new DefaultCodec(),
      null,
      new Metadata());

    // Get at the private FSDataOutputStream inside in SequenceFile so we can
    // call sync on it.  Make it accessible.  Stash it aside for call up in
    // the sync method.
    final Field fields [] = this.writer.getClass().getDeclaredFields();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
          if (getCompressOutput(job)) {
            Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
            codec = ReflectionUtils.newInstance(codecClass, conf);
          }

          Metadata metadata = null;

          String ext = conf.get(EXTENSION_OVERRIDE_CONF, DEFAULT_EXTENSION);
          Path file = getDefaultWorkFile(job, ext.equalsIgnoreCase("none") ? null : ext);

          LOG.info("writing to rcfile " + file.toString());
View Full Code Here
508
509
510
511
512
513
514
    }

    /** Constructs a RCFile Writer. */
    public Writer(FileSystem fs, Configuration conf, Path name)
        throws IOException {
      this(fs, conf, name, null, new Metadata(), null);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Metadata

  • com.twitter.elephantbird.mapreduce.output.RCFileOutputFormat
  • com.twitter.elephantbird.util.RCFileUtil
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.avro.hadoop.io.AvroSequenceFile$Reader$Options
  • org.apache.avro.hadoop.io.AvroSequenceFile$Writer$Options
  • org.apache.hadoop.hbase.regionserver.HLog
  • org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter
  • org.apache.hadoop.hive.ql.io.RCFile
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.