Package com.asakusafw.runtime.directio

Examples of com.asakusafw.runtime.directio.Counter

271
272
273
274
275
276
277
278
279
280
281
        if (description == null) {
            throw new IllegalArgumentException("description must not be null"); //$NON-NLS-1$
        }
        DataFormat<T> format = createFormat(dataType, description.getFormat());
        final DataDefinition<T> definition = SimpleDataDefinition.newInstance(dataType, format);
        final Counter counter = new Counter();
        try {
            FilePattern pattern = toInputPattern(description.getResourcePattern());
            LOG.info("Opening {}/{} for output (id={}, description={})", new Object[] {
                    fullPath,
                    pattern,
View Full Code Here
371
372
373
374
375
376
377
    }

    private OutputAttemptContext createOutputContext() {
        String tx = UUID.randomUUID().toString();
        String attempt = UUID.randomUUID().toString();
        return new OutputAttemptContext(tx, attempt, id, new Counter());
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
            repository = HadoopDataSourceUtil.loadRepository(getConf());
        }
        String basePath = repository.getComponentPath(path);
        DirectDataSource source = repository.getRelatedDataSource(path);
        for (FilePattern pattern : patterns) {
            source.delete(basePath, pattern, recursive, new Counter());
        }
        return 0;
    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
            repository = HadoopDataSourceUtil.loadRepository(getConf());
        }
        String basePath = repository.getComponentPath(path);
        DirectDataSource source = repository.getRelatedDataSource(path);
        for (FilePattern pattern : patterns) {
            List<ResourceInfo> list = source.list(basePath, pattern, new Counter());
            for (ResourceInfo info : list) {
                System.out.println(info.getPath());
            }
        }
        return 0;
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
        HadoopFileFormat<Object> unsafe = unsafe(support);

        model.set("value", new Text("Hello, world!"));

        ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(),  fs, path, new Counter());
        try {
            writer.write(model.unwrap());
        } finally {
            writer.close();
        }

        ModelInput<Object> reader = unsafe.createInput(
                model.unwrap().getClass(), fs, path, 0, fs.getFileStatus(path).getLen(), new Counter());
        try {
            Object buffer = loaded.newModel("Simple").unwrap();
            assertThat(reader.readTo(buffer), is(true));
            assertThat(buffer, is(buffer));
            assertThat(reader.readTo(buffer), is(false));
View Full Code Here
316
317
318
319
320
321
322
        }
        if (datasourceId == null) {
            throw new IllegalArgumentException("datasourceId must not be null"); //$NON-NLS-1$
        }
        String transactionId = getTransactionId(executionId);
        return new OutputTransactionContext(transactionId, datasourceId, new Counter());
    }
View Full Code Here
365
366
367
368
369
370
371
372
        if (context instanceof Progressable) {
            return new ProgressableCounter((Progressable) context);
        } else if (context instanceof org.apache.hadoop.mapred.JobContext) {
            return new ProgressableCounter(((org.apache.hadoop.mapred.JobContext) context).getProgressible());
        } else {
            return new Counter();
        }
    }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
                configure(context, group.getFormat()));
        DirectDataSource datasource = repository.getRelatedDataSource(path);
        String basePath = repository.getComponentPath(path);
        String resourcePath = variables.parse(group.getResourcePath());

        Counter counter = new Counter();
        ModelOutput output = datasource.openOutput(outputContext, definition, basePath, resourcePath, counter);

        long records = 0;
        try {
            for (Union union : values) {
                Object object = union.getObject();
                output.write(object);
                records++;
            }
        } finally {
            output.close();
        }
        recordCounter.increment(records);
        context.getCounter(COUNTER_GROUP, id + ".files").increment(1);
        context.getCounter(COUNTER_GROUP, id + ".records").increment(records);
        context.getCounter(COUNTER_GROUP, id + ".size").increment(counter.get());
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
    @Override
    public void setupAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException {
        if (context == null) {
            throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
        }
        Counter counter = context.getCounter();
        heartbeat.register(counter);
        try {
            entity.setupAttemptOutput(context);
        } finally {
            heartbeat.unregister(counter);
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
    @Override
    public void commitAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException {
        if (context == null) {
            throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
        }
        Counter counter = context.getCounter();
        heartbeat.register(counter);
        try {
            entity.commitAttemptOutput(context);
        } finally {
            heartbeat.unregister(counter);
View Full Code Here
TOP

Related Classes of com.asakusafw.runtime.directio.Counter

  • com.asakusafw.directio.hive.orc.OrcFileFormatTest
  • com.asakusafw.directio.hive.parquet.ParquetFileFormatTest
  • com.asakusafw.directio.tools.DirectIoCommandTestRoot
  • com.asakusafw.directio.tools.DirectIoDelete
  • com.asakusafw.directio.tools.DirectIoList
  • com.asakusafw.dmdl.directio.sequencefile.driver.SequenceFileFormatEmitterTest
  • com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditorTest
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCoreTest
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtilTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.