Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

88
89
90
91
92
93
94
95
96
97
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }     
}
View Full Code Here
123
124
125
126
127
128
129
130
131
132
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }     
}
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    fs.mkdirs(tablePath);
    OutputStream os = fs.create(filePath);
    if (gzip) {
      CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
      CompressionCodec codec = ccf.getCodec(filePath);
      os = codec.createOutputStream(os);
    }
    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < numRecords; i++) {
      w.write(getRecordLine(startId + i, extraCols));
    }
View Full Code Here
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
    // open input stream
    InputStream in = srcFs.open(srcPath);

    // Create compression stream
    OutputStream out = dstFs.create(dstPath);
    out = codec.createOutputStream(out);

    IOUtils.copyBytes(in, out, conf, true);

    return 0;
  }
View Full Code Here
859
860
861
862
863
864
865
866
867
868
869
      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here
666
667
668
669
670
671
672
673
674
675
676
      // Test a simple codec. Extension based detection. We use
      // Bzip2 cause its non-native.
      CompressionCodec codec = ReflectionUtils.newInstance(BZip2Codec.class, conf);
      String extension = codec.getDefaultExtension();
      Path p = new Path(root, "file." + extension);
      OutputStream fout = new DataOutputStream(codec.createOutputStream(
          fs.create(p, true)));
      byte[] writebytes = "foo".getBytes();
      fout.write(writebytes);
      fout.close();
      out = new ByteArrayOutputStream();
View Full Code Here
117
118
119
120
121
122
123
124
125
126
      // build the filename including the extension
      Path file = FileOutputFormat.getTaskOutputPath(job, name
          + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job.getConf());
      FSDataOutputStream fileOut = fs.create(file);
      return new LineRecordWriter<K, V>(new DataOutputStream(codec
          .createOutputStream(fileOut)), keyValueSeparator);
    }
  }
}
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
      String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension;
      OutputStream filestream = FileUtils.create(fileName);
      BufferedWriter filewriter;
      if (codec != null) {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            codec.createOutputStream(filestream, codec.createCompressor()),
            Data.CHARSET_NAME));
      } else {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            filestream, Data.CHARSET_NAME));
      }
View Full Code Here
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
      String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension;
      OutputStream filestream = FileUtils.create(fileName);
      BufferedWriter filewriter;
      if (codec != null) {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            codec.createOutputStream(filestream, codec.createCompressor()),
            Data.CHARSET_NAME));
      } else {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            filestream, Data.CHARSET_NAME));
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.