Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

Class org.apache.hadoop.io.compress.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()
Create a stream compressor that will write to the given output stream. @param out the location for the final output stream @return a stream the user can write uncompressed data to


      BufferedWriter filewriter;
      DataOutputStream filestream = fs.create(filepath, false);
      if (codec != null) {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            codec.createOutputStream(filestream, codec.createCompressor()),
            Data.CHARSET_NAME));
      } else {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            filestream, Data.CHARSET_NAME));
      }

View Full Code Here

    fs.mkdirs(tablePath);
    OutputStream os = fs.create(filePath);
    if (gzip) {
      CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
      CompressionCodec codec = ccf.getCodec(filePath);
      os = codec.createOutputStream(os);
    }
    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < numRecords; i++) {
      w.write(getRecordLine(startId + i, extraCols));
    }

View Full Code Here

    if(isCompressed) {
      Class<? extends CompressionCodec> codecClass =
        FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }

View Full Code Here

      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }

View Full Code Here

              ReflectionUtils.newInstance(
                conf.getClassByName("org.apache.hadoop.io.compress.SnappyCodec"), conf);


            DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
            CompressionOutputStream deflateFilter =
              codec.createOutputStream(compressedDataBuffer);


            byte[] data = new byte[1024];
            DataOutputStream deflateOut = new DataOutputStream(
              new BufferedOutputStream(deflateFilter));
            deflateOut.write(data, 0, data.length);

View Full Code Here

    long fileSize = 4096;
    OutputStream out = fs.create(input, true, 4096, (short) 1, fileSize);
    CompressionCodecFactory codecFactory = new CompressionCodecFactory(new Configuration());
    CompressionCodec codec = codecFactory.getCodec(input);
    if (codec != null) {
      out = codec.createOutputStream(out);
    }
    Writer writer = new OutputStreamWriter(out);
    try {
      for (int n = 0; n < records; n++) {
        writer.write(line(n, recordLength));

View Full Code Here

    Class<?> codecClass = Class.forName(codecClassname);
    Configuration conf = new Configuration();
    CompressionCodec codec = (CompressionCodec)
      ReflectionUtils.newInstance(codecClass, conf);
    
    CompressionOutputStream out = codec.createOutputStream(System.out);
    IOUtils.copyBytes(System.in, out, 4096, false);
    out.finish();
  }
}
// ^^ StreamCompressor

View Full Code Here

          Path file = new Path(workOutputPath, path);
          FileSystem fs = file.getFileSystem(conf);                
          FSDataOutputStream fileOut = fs.create(file, false);
          
          if (isCompressed)
             return new DataOutputStream(codec.createOutputStream(fileOut));
          else
             return fileOut;
        }
          
      };

View Full Code Here

      // Bzip2 cause its non-native.
      CompressionCodec codec = (CompressionCodec)
          ReflectionUtils.newInstance(BZip2Codec.class, conf);
      String extension = codec.getDefaultExtension();
      Path p = new Path(root, "file." + extension);
      OutputStream fout = new DataOutputStream(codec.createOutputStream(
          fs.create(p, true)));
      byte[] writebytes = "foo".getBytes();
      fout.write(writebytes);
      fout.close();
      out = new ByteArrayOutputStream();

View Full Code Here

                               ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter(new DataOutputStream
                                  (codec.createOutputStream(fileOut)));
    }
  }      
}

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.