Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.DefaultCodec

47
48
49
50
51
52
53
54
55
    // no-op
  }

  @Override
  public void open(String filePath, FlumeFormatter fmt) throws IOException {
    DefaultCodec defCodec = new DefaultCodec();
    CompressionType cType = CompressionType.BLOCK;
    open(filePath, defCodec, cType, fmt);
  }
View Full Code Here
382
383
384
385
386
387
388
389
      Class<? extends HLogKey> keyClass, Class<? extends KeyValue> valueClass)
      throws IOException {
    return SequenceFile.createWriter(this.fs, this.conf, path, keyClass,
        valueClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs
            .getDefaultReplication(), this.blocksize,
        SequenceFile.CompressionType.NONE, new DefaultCodec(), null,
        new Metadata());
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
        + useRawLocalFileSystem);
  }

  @Override
  public void open(String filePath) throws IOException {
    DefaultCodec defCodec = new DefaultCodec();
    CompressionType cType = CompressionType.BLOCK;
    open(filePath, defCodec, cType);
  }
View Full Code Here
298
299
300
301
302
303
304
305
306
307
308
309
     * Creates a compressed sequence file.
     * @throws Exception if failed
     */
    @Test
    public void write_compressed() throws Exception {
        DefaultCodec codec = new DefaultCodec();
        codec.setConf(conf);

        Path path = new Path("testing");

        LongWritable key = new LongWritable();
        LongWritable value = new LongWritable();
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
     */
    @Test
    public void output_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.codec(new DefaultCodec())
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
        } finally {
            out.close();
View Full Code Here
387
388
389
390
391
392
393
394
395
396
397
      for (int pi = 0; pi < 5; pi++) {
    Path path = new Path(new Path(file.getAbsolutePath()),
      "parition" + pi);

    datePartitionedRowCount += writeRCFileTest(fs, simpleRowCount,
      path, columnCount, new DefaultCodec(), columnCount);

    new File(path.toString()).deleteOnExit();
    datePartitionedRCFiles.add(path.toString());
    datePartitionedDirs.add(file.toString());
View Full Code Here
456
457
458
459
460
461
462
463
464
465
466
        + "/attempt-00000");
      Path hourFilePath = new Path(rcFile.getAbsolutePath());
      rcFile.deleteOnExit();

      writeRCFileTest(fs, simpleRowCount, hourFilePath,
        columnCount, new DefaultCodec(), columnCount);

      yearMonthDayHourcalendar.add(Calendar.HOUR_OF_DAY, 1);
        }

        yearMonthDayHourcalendar.add(Calendar.DAY_OF_MONTH, 1);
View Full Code Here
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
  simpleDataFile.deleteOnExit();

  Path path = new Path(simpleDataFile.getPath());

  writeRCFileTest(fs, simpleRowCount, path, columnCount,
    new DefaultCodec(), columnCount);

  // produce a folder of simple data
  simpleDataDir = new File("simpleDataDir" + System.currentTimeMillis());
  simpleDataDir.mkdir();

  for (int i = 0; i < simpleDirFileCount; i++) {

      simpleDataFile = new File(simpleDataDir, "testhiveColumnarLoader-"
        + i + ".txt");

      Path filePath = new Path(simpleDataFile.getPath());

      writeRCFileTest(fs, simpleRowCount, filePath, columnCount,
        new DefaultCodec(), columnCount);

  }

    }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
              conf.getInt("hbase.regionserver.hlog.replication",
              FSUtils.getDefaultReplication(fs, path))),
            Long.valueOf(conf.getLong("hbase.regionserver.hlog.blocksize",
                FSUtils.getDefaultBlockSize(fs, path))),
            Boolean.valueOf(false) /*createParent*/,
            SequenceFile.CompressionType.NONE, new DefaultCodec(),
            createMetadata(conf, compress)
            });
    } catch (InvocationTargetException ite) {
      // function was properly called, but threw it's own exception
      throw new IOException(ite.getCause());
    } catch (Exception e) {
      // ignore all other exceptions. related to reflection failure
    }

    // if reflection failed, use the old createWriter
    if (this.writer == null) {
      LOG.debug("new createWriter -- HADOOP-6840 -- not available");
      this.writer = SequenceFile.createWriter(fs, conf, path,
        HLog.getKeyClass(conf), WALEdit.class,
        fs.getConf().getInt("io.file.buffer.size", 4096),
        (short) conf.getInt("hbase.regionserver.hlog.replication",
          FSUtils.getDefaultReplication(fs, path)),
        conf.getLong("hbase.regionserver.hlog.blocksize",
          FSUtils.getDefaultBlockSize(fs, path)),
        SequenceFile.CompressionType.NONE,
        new DefaultCodec(),
        null,
        createMetadata(conf, compress));
    } else {
      LOG.debug("using new createWriter -- HADOOP-6840");
    }
View Full Code Here
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
          } catch (ClassNotFoundException cnfe) {
            throw new IllegalArgumentException("Unknown codec: " +
                                               codecClassname, cnfe);
          }
        } else {
          codec = new DefaultCodec();
          ((Configurable)codec).setConf(conf);
        }
      }
     
      this.metadata = new Metadata();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.DefaultCodec

  • com.alexholmes.hadooputils.combine.avro.AvroFileGenerator
  • com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator
  • com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest
  • com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest
  • com.cloudera.flume.handlers.hdfs.TestDFSWrite
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.