Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.CompressionType

39
40
41
42
43
44
45
46
47
48
49
                                      String name, Progressable progress)
                                      throws IOException {

    Path file = new Path(job.getOutputPath(), name);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFile.getCompressionType(job);

      // find the right codec
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
        if (rwonly && (!create || merge || fast)) {
          System.err.println(usage);
          System.exit(-1);
        }

        CompressionType compressionType =
          CompressionType.valueOf(compressType);

        if (rwonly || (create && !merge)) {
            writeTest(fs, count, seed, file, compressionType);
            readTest(fs, count, seed, file);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
      throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    // Configure compression if requested.
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
      // Find the kind of compression to do.
      compressionType = getOutputCompressionType(conf);

      // Find the right codec.
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
      RecordReader sampleReader = inputFormat.getRecordReader(splits[0], job);

      List<SequenceFile.Writer> writers = new ArrayList<SequenceFile.Writer>(
          splits.length);

      CompressionType compressionType = getOutputCompressionType(job);
      Class<? extends CompressionCodec> outputCompressorClass = getOutputCompressorClass(
          job, null);
      CompressionCodec codec = null;
      if (outputCompressorClass != null) {
        codec = ReflectionUtils.newInstance(outputCompressorClass,
View Full Code Here
418
419
420
421
422
423
424
425
426
427
428
  public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs,
                                                         Path file, Class<?> keyClass,
                                                         Class<?> valClass) throws IOException {
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (SequenceFileOutputFormat.getCompressOutput(jc)) {
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
      Class codecClass = SequenceFileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, jc);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    // get the path of the temporary output file
    Path file = FileOutputFormat.getTaskOutputPath(job, name);
   
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = getOutputCompressionType(job);

      // find the right codec
View Full Code Here
957
958
959
960
961
962
963
964
965
966
967
   * @return output stream over the created sequencefile
   */
  public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file,
      Class<?> keyClass, Class<?> valClass, boolean isCompressed) throws IOException {
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    Class codecClass = null;
    if (isCompressed) {
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
      codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
View Full Code Here
375
376
377
378
379
380
381
382
383
384
385
        if (rwonly && (!create || merge || fast)) {
          System.err.println(usage);
          System.exit(-1);
        }

        CompressionType compressionType =
          CompressionType.valueOf(compressType);
        CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                    conf.getClassByName(compressionCodec),
                                    conf);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
      throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    // Configure compression if requested.
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
      // Find the kind of compression to do.
      compressionType = getOutputCompressionType(conf);

      // Find the right codec.
View Full Code Here
634
635
636
637
638
639
640
641
642
643
644
      if (rwonly && (!create || merge || fast)) {
        System.err.println(usage);
        System.exit(-1);
      }

      CompressionType compressionType =
        CompressionType.valueOf(compressType);
      CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                                                             conf.getClassByName(compressionCodec),
                                                                             conf);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.CompressionType

  • com.asakusafw.bulkloader.extractor.DfsFileImport
  • com.asakusafw.bulkloader.extractor.DfsFileImportTest
  • com.backtype.hadoop.pail.SequenceFileFormat
  • com.linkedin.camus.etl.kafka.common.SequenceFileRecordWriterProvider
  • com.linkedin.json.JsonSequenceFileOutputFormat
  • com.mozilla.pig.storage.SeqFileMultiStorage$MultiStorageSequenceOutputFormat
  • org.apache.accumulo.server.master.LogSort$LoggerMapFileOutputFormat
  • org.apache.avro.mapreduce.AvroSequenceFileOutputFormat
  • org.apache.flume.sink.hdfs.HDFSCompressedDataStream
  • org.apache.hadoop.hive.ql.exec.FileSinkOperator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.