Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

Class org.apache.hadoop.io.compress.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()
Create a stream compressor that will write to the given output stream. @param out the location for the final output stream @return a stream the user can write uncompressed data to

    InputStream is = FileSystem.getLocal(config).open(new Path(args[1]));
    OutputStream os = hdfs.create(
        new Path(args[2] + codec.getDefaultExtension()));




    OutputStream cos = codec.createOutputStream(os);


    IOUtils.copyBytes(is, cos, config, true);


    IOUtils.closeStream(os);
    IOUtils.closeStream(is);

View Full Code Here

    dstPath = new Path(path + codec.getDefaultExtension());
    hdfs = dstPath.getFileSystem(conf);
    pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName());
    writer = pathManager.open();
    try {
      writer = codec.createOutputStream(writer, cmp);
    } catch (NullPointerException npe) {
      // tries to find "native" version of codec, if that fails, then tries to
      // find java version. If there is no java version, the createOutputStream
      // exits via NPE. We capture this and convert it into a IOE with a more
      // useful error message.

View Full Code Here

      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }

View Full Code Here

    
    if (logger.isDebugEnabled()){
      logger.debug("reflection using compression codec class: " + codec.getClass().getName());
    }
    FSDataOutputStream out = fs.create(new Path(file), false, BUFFER_SIZE);
    CompressionOutputStream co = codec.createOutputStream(out);
    BufferedWriter bw = new BufferedWriter(
        new OutputStreamWriter(co, ENCODING), BUFFER_SIZE);
    
    return bw;
  }

View Full Code Here

        FileOutputFormat.getTaskOutputPath(job, 
                                           name + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job);
      FSDataOutputStream fileOut = fs.create(file, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)),
                                        keyValueSeparator);
    }
  }
}

View Full Code Here

      // add the appropriate extension
      file = file.suffix(codec.getDefaultExtension());


      if (isCompressionEmulationEnabled(conf)) {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new DataOutputStream(codec.createOutputStream(fileOut));
      }
    }
    return fs.create(file, false);
  }

View Full Code Here

      // Bzip2 cause its non-native.
      CompressionCodec codec = (CompressionCodec)
          ReflectionUtils.newInstance(BZip2Codec.class, conf);
      String extension = codec.getDefaultExtension();
      Path p = new Path(root, "file." + extension);
      OutputStream fout = new DataOutputStream(codec.createOutputStream(
          fs.create(p, true)));
      byte[] writebytes = "foo".getBytes();
      fout.write(writebytes);
      fout.close();
      out = new ByteArrayOutputStream();

View Full Code Here

            logger.info("reflection using compression codec class: " + codec.getClass().getName());
            compressionTypePrintVirgin = false;
          }
          
          out = fs.create(p, false, bufferSize);
          co = codec.createOutputStream(out);
          bw = new BufferedWriter(
              new OutputStreamWriter(co, encoding), bufferSize);
        } else {
          out = fs.create(p, false, bufferSize);
          bw = new BufferedWriter(new OutputStreamWriter(out,

View Full Code Here

        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }      
}

View Full Code Here

      Path file = FileOutputFormat.getTaskOutputPath(job,
          name + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job.getConf());
      FSDataOutputStream fileOut = fs.create(file);
      return new LineRecordWriter<K, V>(new DataOutputStream(
          codec.createOutputStream(fileOut)), keyValueSeparator);
    }
  }
}

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.