Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

346
347
348
349
350
351
352
353
354
355
356
                new CompressionCodecFactory(conf).getCodec(nextSegment);
            OutputStream output;
            Compressor compressor = null;
            if (codec != null) {
              compressor = CodecPool.getCompressor(codec);
              output = codec.createOutputStream(tempUncompOut, compressor);
            } else {
              output = tempUncompOut;
            }

            tempUncompOut = null;
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    FileSystem fs = path.getFileSystem(conf);
    CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path);
    OutputStream output;
    if (codec != null) {
      compressor = CodecPool.getCompressor(codec);
      output = codec.createOutputStream(fs.create(path), compressor);
    } else {
      output = fs.create(path);
    }
    writer = new JsonObjectMapperWriter<T>(output,
        conf.getBoolean("rumen.output.pretty.print", true));
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
      FSDataOutputStream fileOut = fs.create(file, false);
      return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
    } else {
      FSDataOutputStream fileOut = fs.create(file, false);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)),
                                        keyValueSeparator);
    }
  }
}
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
        FileOutputFormat.getTaskOutputPath(job,
                                           name + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job);
      FSDataOutputStream fileOut = fs.create(file, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)),
                                        keyValueSeparator);
    }
  }
}
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    if (!isCompressed) {
      return new LineRecordWriter<Text, Text>(fileOut, separator);
    } else {
      DataOutputStream out =
        new DataOutputStream(codec.createOutputStream(fileOut));
      return new LineRecordWriter<Text, Text>(out, separator);
    }
  }

  /**
 
View Full Code Here
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
                  getFs().create(path);


              DataOutputStream stream = codec == null ? uncompressedStream :
                  new DataOutputStream(
                      codec.createOutputStream(uncompressedStream));

              Partition<I, V, E> partition =
                  getPartitionStore().getOrCreatePartition(partitionId);

              partition.write(stream);
View Full Code Here
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
    // open input stream
    InputStream in = srcFs.open(srcPath);

    // Create compression stream
    OutputStream out = dstFs.create(dstPath);
    out = codec.createOutputStream(out);

    IOUtils.copyBytes(in, out, conf, true);

    return 0;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
      return new CSVRecordWriter(fileOut,
          keyValueSeparator);
    } else {
      FSDataOutputStream fileOut = fs.create(file, false);
      return new CSVRecordWriter(
          new DataOutputStream(codec.createOutputStream(fileOut)),
          keyValueSeparator);
    }
  }

  protected static class CSVRecordWriter
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
    InputStream is = hdfs.open(new Path(args[0]));
    OutputStream os = hdfs.create(
        new Path(args[0] + codec.getDefaultExtension()));


    OutputStream cos = codec.createOutputStream(os);

    IOUtils.copyBytes(is, cos, config, true);

    IOUtils.closeStream(os);
    IOUtils.closeStream(is);
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
      FSDataOutputStream fileOut = fs.create(file, false);
      return new PasswdRecordWriter<K, V>(fileOut);
    } else {
      FSDataOutputStream fileOut = fs.create(file, false);
      return new PasswdRecordWriter<K, V>(new DataOutputStream
          (codec.createOutputStream(fileOut)));
    }
  }

}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.