Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context)
      throws IOException {
    boolean isCompressed = getCompressOutput(context);
    Configuration conf = context.getConfiguration();
    String ext = "";
    CompressionCodec codec = null;

    if (isCompressed) {
      // create the named codec
      Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(
          context, GzipCodec.class);
      codec = ReflectionUtils.newInstance(codecClass, conf);

      ext = codec.getDefaultExtension();
    }

    Path file = getDefaultWorkFile(context, ext);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    DataOutputStream ostream = fileOut;

    if (isCompressed) {
      ostream = new DataOutputStream(codec.createOutputStream(fileOut));
    }

    return new NoKeyRecordWriter<K, V>(ostream);
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    f.login(usr, pwd);
    f.changeWorkingDirectory(dir);
    f.setFileType(FTP.BINARY_FILE_TYPE);

    boolean isCompressed = getCompressOutput(job);
    CompressionCodec codec = null;
    String extension = "";
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(
          job, GzipCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass,
          conf);
      extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    String filename = file.getName();
    if (!isCompressed) {
      // FSDataOutputStream fileOut = fs.create(file, false);
      OutputStream os = f.appendFileStream(filename);
      DataOutputStream fileOut = new DataOutputStream(os);
      return new FTPLineRecordWriter<K, V>(fileOut, new String(
          keyValueSeparator), f);

    } else {
      // FSDataOutputStream fileOut = fs.create(file, false);
      OutputStream os = f.appendFileStream(filename);
      DataOutputStream fileOut = new DataOutputStream(os);
      return new FTPLineRecordWriter<K, V>(new DataOutputStream(
          codec.createOutputStream(fileOut)), keyValueSeparator, f);
    }
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
    return new DelimitedLineRecordReader();
  }

  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    CompressionCodec codec = new CompressionCodecFactory(
        context.getConfiguration()).getCodec(file);
    return codec == null;
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    mapper = new ObjectMapper();
    mapper.configure(
        DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
    this.clazz = clazz;
    FileSystem fs = path.getFileSystem(conf);
    CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path);
    InputStream input;
    if (codec == null) {
      input = fs.open(path);
      decompressor = null;
    } else {
      FSDataInputStream fsdis = fs.open(path);
      decompressor = CodecPool.getDecompressor(codec);
      input = codec.createInputStream(fsdis, decompressor);
    }
    jsonParser = mapper.getJsonFactory().createJsonParser(input);
  }
View Full Code Here
394
395
396
397
398
399
400
401
402
403
404
405
406
  private InputStream forMagic(Path p, FileSystem srcFs) throws IOException {
    FSDataInputStream i = srcFs.open(p);

    // check codecs
    CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
    CompressionCodec codec = cf.getCodec(p);
    if (codec != null) {
      return codec.createInputStream(i);
    }

    switch(i.readShort()) {
      case 0x1f8b: // RFC 1952
        i.seek(0);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
            return fs.create(file, false);
        } else {
            Class<? extends CompressionCodec> codecClass =
                    getOutputCompressorClass(ctx, GzipCodec.class);

            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, ctx.getConfiguration());
            Path file =  getDefaultWorkFile(ctx, ".nt"+codec.getDefaultExtension());
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataOutputStream fileOut = fs.create(file, false);
            return codec.createOutputStream(fileOut);
        }
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
            return fs.create(file, false);
        } else {
            Class<? extends CompressionCodec> codecClass =
                    getOutputCompressorClass(ctx, GzipCodec.class);

            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, ctx.getConfiguration());
            Path file =  getDefaultWorkFile(ctx, ".nt"+codec.getDefaultExtension());
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataOutputStream fileOut = fs.create(file, false);
            return new DataOutputStream(codec.createOutputStream(fileOut));
        }
    }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
            throw new IllegalArgumentException("name must not be null"); //$NON-NLS-1$
        }
        if (dataType == null) {
            throw new IllegalArgumentException("dataType must not be null"); //$NON-NLS-1$
        }
        CompressionCodec codec = null;
        Configuration conf = context.getConfiguration();
        if (FileOutputFormat.getCompressOutput(context)) {
            Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(context, DefaultCodec.class);
            codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        }
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
            FileSystem fileSystem,
            Path path,
            final Counter counter) throws IOException, InterruptedException {
        final K keyBuffer = createKeyObject();
        final V valueBuffer = createValueObject();
        CompressionCodec codec = getCompressionCodec(path);
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format(
                    "Creating sequence file (path={0}, type={1}, codec={2})",
                    path,
                    dataType.getName(),
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
        }
        default: {
          // Check the type of compression instead, depending on Codec class's
          // own detection methods, based on the provided path.
          CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
          CompressionCodec codec = cf.getCodec(item.path);
          if (codec != null) {
            i.seek(0);
            return codec.createInputStream(i);
          }
          break;
        }
        case 0x4f62: { // 'O' 'b'
          if (i.readByte() == 'j') {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.compression.lzo.TestLzoCodec
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.