Package com.asakusafw.runtime.directio

Examples of com.asakusafw.runtime.directio.Counter

260
261
262
263
264
265
266
267
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        in.close();
    }
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
    @Test
    public void output() throws Exception {
        final int count = 10000;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            StringOption value = new StringOption();
            for (int i = 0; i < count; i++) {
                value.modify("Hello, world at " + i);
                out.write(value);
View Full Code Here
310
311
312
313
314
315
316
317
318
319
320
    @Test
    public void output_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.codec(new DefaultCodec())
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
        } finally {
            out.close();
        }
View Full Code Here
334
335
336
337
338
339
340
341
342
343
344
    @Test
    public void output_no_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing.gz").toURI());
        ModelOutput<StringOption> out = format.codec(null)
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
        } finally {
            out.close();
        }
View Full Code Here
358
359
360
361
362
363
364
365
366
367
368
    @Test
    public void output_compressed_conf() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, DefaultCodec.class.getName());
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
        } finally {
            out.close();
        }
View Full Code Here
382
383
384
385
386
387
388
389
390
391
392
    @Test
    public void output_compressed_invalid() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, "__INVALID__");
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
        } finally {
            out.close();
        }
View Full Code Here
307
308
309
310
311
312
313
314
315
        assert definition != null;
        assert split != null;
        assert context != null;
        Configuration conf = context.getConfiguration();
        T buffer = ReflectionUtils.newInstance(definition.getDataClass(), conf);
        Counter counter = new Counter();
        ModelInput<T> input = createInput(context, split.group.containerPath, definition, counter, split.fragment);
        return new BridgeRecordReader<T>(input, buffer, counter, split.fragment.getSize());
    }
View Full Code Here
437
438
439
440
441
442
443
444
     */
    @Test
    public void move_simple() throws Exception {
        touch("src/a.csv");
        FileSystem fs = getTempFileSystem();
        HadoopDataSourceUtil.move(new Counter(), fs, getPath("src"), getPath("dst"));
        assertThat(collect(), is(path("dst/a.csv")));
    }
View Full Code Here
451
452
453
454
455
456
457
458
    public void move_multiple() throws Exception {
        touch("src/a.csv");
        touch("src/b.csv");
        touch("src/c.csv");
        FileSystem fs = getTempFileSystem();
        HadoopDataSourceUtil.move(new Counter(), fs, getPath("src"), getPath("dst"));
        assertThat(collect(), is(path("dst/a.csv", "dst/b.csv", "dst/c.csv")));
    }
View Full Code Here
465
466
467
468
469
470
471
472
    public void move_deep() throws Exception {
        touch("src/a.csv");
        touch("src/a/b.csv");
        touch("src/a/b/c.csv");
        FileSystem fs = getTempFileSystem();
        HadoopDataSourceUtil.move(new Counter(), fs, getPath("src"), getPath("dst"));
        assertThat(collect(), is(path("dst/a.csv", "dst/a/b.csv", "dst/a/b/c.csv")));
    }
View Full Code Here
TOP

Related Classes of com.asakusafw.runtime.directio.Counter

  • com.asakusafw.directio.hive.orc.OrcFileFormatTest
  • com.asakusafw.directio.hive.parquet.ParquetFileFormatTest
  • com.asakusafw.directio.tools.DirectIoCommandTestRoot
  • com.asakusafw.directio.tools.DirectIoDelete
  • com.asakusafw.directio.tools.DirectIoList
  • com.asakusafw.dmdl.directio.sequencefile.driver.SequenceFileFormatEmitterTest
  • com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditorTest
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCoreTest
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtilTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.