Package com.hadoop.compression.lzo

Examples of com.hadoop.compression.lzo.LzopCodec


  @Override
  protected List<FileStatus> listStatus(JobContext job) throws IOException {
    List<FileStatus> files = super.listStatus(job);

    String fileExtension = new LzopCodec().getDefaultExtension();
    Configuration conf = job.getConfiguration();

    for (Iterator<FileStatus> iterator = files.iterator(); iterator.hasNext();) {
      FileStatus fileStatus = iterator.next();
      Path file = fileStatus.getPath();
View Full Code Here


  @Override
  protected FileStatus[] listStatus(JobConf conf) throws IOException {
    List<FileStatus> files = new ArrayList<FileStatus>(Arrays.asList(super.listStatus(conf)));

    String fileExtension = new LzopCodec().getDefaultExtension();

    Iterator<FileStatus> it = files.iterator();
    while (it.hasNext()) {
      FileStatus fileStatus = it.next();
      Path file = fileStatus.getPath();
View Full Code Here

    conf = new Configuration();

    conf.set("io.compression.codecs", "com.hadoop.compression.lzo.LzopCodec");
    conf.setInt(LzoCodec.LZO_BUFFER_SIZE_KEY, 256);
    codec = new LzopCodec();
    codec.setConf(conf);
    FileUtil.fullyDelete( new File(TESTDIR));
    inputDir.mkdirs();

    // close any FileySystem from previous tests:
View Full Code Here

    Configuration config = new Configuration();
    FileSystem hdfs = FileSystem.get(config);

    Path destFile = new Path(args[0]);

    LzopCodec codec = new LzopCodec();
    codec.setConf(config);
    InputStream is = codec.createInputStream(hdfs.open(destFile));

    readFromProtoBuf(is);
  }
View Full Code Here

    Configuration config = new Configuration();
    FileSystem hdfs = FileSystem.get(config);

    Path destFile = new Path(args[0]);

    LzopCodec codec = new LzopCodec();
    codec.setConf(config);
    InputStream is = codec.createInputStream(hdfs.open(destFile));

    readFromProtoBuf(is);
  }
View Full Code Here

public class LzopFileReadWrite {

  public static void main(String... args) throws Exception {
    Configuration config = new Configuration();

    LzopCodec codec = new LzopCodec();
    codec.setConf(config);

    Path srcFile = new Path(args[0]);
    Path restoredFile = new Path(args[0] + ".restored");

    Path destFile = compress(srcFile, config);
View Full Code Here

  public static Path compress(Path src,
                              Configuration config)
      throws IOException {
    Path destFile =
        new Path(
            src.toString() + new LzopCodec().getDefaultExtension());

    LzopCodec codec = new LzopCodec();
    codec.setConf(config);

    FileSystem hdfs = FileSystem.get(config);
    InputStream is = null;
    OutputStream os = null;
    try {
      is = hdfs.open(src);
      os = codec.createOutputStream(hdfs.create(destFile));

      IOUtils.copyBytes(is, os, config);
    } finally {
      IOUtils.closeStream(os);
      IOUtils.closeStream(is);
View Full Code Here

  }

  public static void decompress(Path src, Path dest,
                                Configuration config)
      throws IOException {
    LzopCodec codec = new LzopCodec();
    codec.setConf(config);

    FileSystem hdfs = FileSystem.get(config);
    InputStream is = null;
    OutputStream os = null;
    try {
      is = codec.createInputStream(hdfs.open(src));
      os = hdfs.create(dest);

      IOUtils.copyBytes(is, os, config);
    } finally {
      IOUtils.closeStream(os);
View Full Code Here

                                      File inputFile,
                                      Path input) throws IOException {
      FileSystem hdfs = FileSystem.get(config);
      OutputStream os = hdfs.create(input);

      LzopCodec codec = new LzopCodec();
      codec.setConf(config);
      OutputStream lzopOutputStream = codec.createOutputStream(os);

      ThriftBlockWriter<Stock> writer =
          new ThriftBlockWriter<Stock>(
              lzopOutputStream, Stock.class);
View Full Code Here

                                    File inputFile,
                                    Path input) throws IOException {
    FileSystem hdfs = FileSystem.get(config);
    OutputStream os = hdfs.create(input);

    LzopCodec codec = new LzopCodec();
    codec.setConf(config);
    OutputStream lzopOutputStream = codec.createOutputStream(os);

    ProtobufBlockWriter<Stock> writer =
        new ProtobufBlockWriter<Stock>(
            lzopOutputStream, Stock.class);
View Full Code Here

TOP

Related Classes of com.hadoop.compression.lzo.LzopCodec

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.