Examples of com.asakusafw.compiler.flow.jobflow.JobflowModel.Reduce

Package com.asakusafw.compiler.flow.jobflow.JobflowModel

Examples of com.asakusafw.compiler.flow.jobflow.JobflowModel.Reduce

com.asakusafw.compiler.flow.jobflow.JobflowModel.Reduce

        assert model != null;
        LOG.debug("{}を解析しています", model);
        List<Process> processes = analyzeProcesses(model);
        List<Delivery> deliveries = analyzeDeliveries(model);
        Set<SideData> sideData = analyzeSideData(model);
        Reduce reduce = analyzeReduce(model);
        Stage stage = new Stage(
                model,
                processes,
                deliveries,
                reduce,

View Full Code Here

            return null;
        }
        assert model.getReduceUnits().isEmpty() == false;
        CompiledShuffle shuffle = model.getShuffleModel().getCompiled();
        CompiledReduce reducer = model.getReduceUnits().get(0).getCompiled();
        return new Reduce(
                reducer.getReducerType().getQualifiedName(),
                reducer.getCombinerTypeOrNull() == null
                    ? null
                    : reducer.getCombinerTypeOrNull().getQualifiedName(),
                shuffle.getKeyTypeName(),

View Full Code Here

                    LOG.debug("Input: {} ({})", unit.getResolvedLocations(), unit.getDataType());
                }
                for (Delivery unit : stage.getDeliveries()) {
                    LOG.debug("Output: {} ({})", unit.getInputInfo().getLocations(), unit.getDataType());
                }
                Reduce reducer = stage.getReduceOrNull();
                if (reducer != null) {
                    LOG.debug("ShuffleKey: {}", reducer.getKeyTypeName().toNameString());
                    LOG.debug("ShuffleValue: {}", reducer.getValueTypeName().toNameString());
                    LOG.debug("Partitioner: {}", reducer.getPartitionerTypeName().toNameString());
                    LOG.debug("Grouping: {}", reducer.getGroupingComparatorTypeName().toNameString());
                    LOG.debug("Sort: {}", reducer.getSortComparatorTypeName().toNameString());
                    LOG.debug("Combiner: {}", reducer.getCombinerTypeNameOrNull() == null
                            ? "N/A" : reducer.getCombinerTypeNameOrNull().toNameString());
                    LOG.debug("Reducer: {}", reducer.getReducerTypeName().toNameString());
                }
                for (SideData data : stage.getSideData()) {
                    LOG.debug("SideData: {} ({})", data.getLocalName(), data.getClusterPaths());
                }
                LOG.debug("Upstreams: {}", getStageIds(graph.getConnected(stage)));

View Full Code Here

            for (Fragment fragment : unit.getFragments()) {
                String fragmentId = dumpFragment(context, fragment);
                context.connect(unitId, fragmentId);
            }
        }
        Reduce reducer = stage.getReduceOrNull();
        if (reducer != null) {
            String unitId = context.label(reducer, "Reducer", reducer.getReducerTypeName().toNameString());
            context.connect(id, unitId);
            for (ReduceUnit unit : stage.getModel().getReduceUnits()) {
                for (Fragment fragment : unit.getFragments()) {
                    String fragmentId = dumpFragment(context, fragment);
                    context.connect(unitId, fragmentId);

View Full Code Here

                    Collections.<FormalParameterDeclaration>emptyList(),
                    statements);
        }


        private List<MethodDeclaration> createShuffleMethods() {
            Reduce reduce = stage.getReduceOrNull();
            List<MethodDeclaration> results = Lists.create();
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_SHUFFLE_KEY_CLASS,
                    importer.toType(reduce.getKeyTypeName())));
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_SHUFFLE_VALUE_CLASS,
                    importer.toType(reduce.getValueTypeName())));
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_PARTITIONER_CLASS,
                    importer.toType(reduce.getPartitionerTypeName())));
            if (reduce.getCombinerTypeNameOrNull() != null) {
                results.add(createClassLiteralMethod(
                        AbstractStageClient.METHOD_COMBINER_CLASS,
                        importer.toType(reduce.getCombinerTypeNameOrNull())));
            }
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_SORT_COMPARATOR_CLASS,
                    importer.toType(reduce.getSortComparatorTypeName())));
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_GROUPING_COMPARATOR_CLASS,
                    importer.toType(reduce.getGroupingComparatorTypeName())));
            results.add(createClassLiteralMethod(
                    AbstractStageClient.METHOD_REDUCER_CLASS,
                    importer.toType(reduce.getReducerTypeName())));
            return results;
        }

View Full Code Here

TOP

Related Classes of com.asakusafw.compiler.flow.jobflow.JobflowModel.Reduce

com.asakusafw.compiler.flow.jobflow.JobflowAnalyzer

com.asakusafw.compiler.flow.jobflow.JobflowCompiler

com.asakusafw.compiler.flow.jobflow.StageClientEmitter$Engine

com.asakusafw.compiler.tool.analysis.VisualizeCompiledStructureProcessor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.