Package com.asakusafw.compiler.flow.stage

Examples of com.asakusafw.compiler.flow.stage.StageModel$ResourceFragment


     * @return 断片までの解析が終わったステージの構造
     * @throws IOException 出力に失敗した場合
     */
    protected StageModel compileFragments(StageBlock block) throws IOException {
        ShuffleModel shuffle = compileShuffle(block);
        StageModel stage = new StageAnalyzer(environment).analyze(block, shuffle);
        for (MapUnit unit : stage.getMapUnits()) {
            for (Fragment fragment : unit.getFragments()) {
                compile(fragment, stage);
            }
        }
        for (ReduceUnit unit : stage.getReduceUnits()) {
            for (Fragment fragment : unit.getFragments()) {
                compile(fragment, stage);
            }
        }
        return stage;
View Full Code Here


    }

    private void run(boolean combine) {
        environment.getOptions().setEnableCombiner(combine);
        List<StageModel> stages = compile(SummarizeFlowTrivial.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<ExSummarized> result = mapper.create("out");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        ExSummarized ex1 = new ExSummarized();
        ex1.setCount(1);
        f.begin();
View Full Code Here

        });
        return results;
    }

    private void writeStageBody(Context context, Stage stage) {
        StageModel model = stage.getModel();

        for (MapUnit unit : model.getMapUnits()) {
            context.put("mapper: {0}", name(unit.getCompiled().getQualifiedName()));
            context.push();
            writeFragments(context, unit.getFragments());
            context.pop();
        }
        if (stage.getReduceOrNull() != null) {
            context.put("reducer: {0}", name(stage.getReduceOrNull().getReducerTypeName()));
            context.push();
            for (ReduceUnit unit : model.getReduceUnits()) {
                writeFragments(context, unit.getFragments());
            }
            context.pop();
        }
    }
View Full Code Here

     * 単一グループでのテスト。
     */
    @Test
    public void op1() {
        List<StageModel> stages = compile(CoGroupFlowOp1.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> result = mapper.create("r1");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        ex1.setStringAsString("string");

View Full Code Here

     * 2グループでのテスト。
     */
    @Test
    public void op2() {
        List<StageModel> stages = compile(CoGroupFlowOp2.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> r1 = mapper.create("r1");
        MockResult<Ex2> r2 = mapper.create("r2");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment s1 = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        Segment s2 = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(1));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        Ex2 ex2 = new Ex2();
        ex1.setStringAsString("string");
View Full Code Here

     * 3グループでのテスト。
     */
    @Test
    public void op3() {
        List<StageModel> stages = compile(CoGroupFlowOp3.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> r1 = mapper.create("r1");
        MockResult<Ex1> r2 = mapper.create("r2");
        MockResult<Ex1> r3 = mapper.create("r3");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment s1 = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        Segment s2 = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(1));
        Segment s3 = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(2));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        ex1.setStringAsString("string");

View Full Code Here

     * パラメーター付きのテスト。
     */
    @Test
    public void withParameter() {
        List<StageModel> stages = compile(CoGroupFlowWithParameter.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> result = mapper.create("r1");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        ex1.setStringAsString("string");

View Full Code Here

     * スワップ付きバッファでのテスト。
     */
    @Test
    public void swap() {
        List<StageModel> stages = compile(CoGroupFlowSwap.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> result = mapper.create("r1");

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        ex1.setStringAsString("string");

View Full Code Here

     * グループごとの最大値を求めるテスト。
     */
    @Test
    public void max() {
        List<StageModel> stages = compile(GroupSortFlowMax.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> result = mapper.add("r1", new Ex1Copier());

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        f.begin();

View Full Code Here

     * グループごとの最小値を求めるテスト。
     */
    @Test
    public void min() {
        List<StageModel> stages = compile(GroupSortFlowMin.class);
        StageModel stage = stages.get(0);
        Assume.assumeThat(stage.getReduceUnits().size(), is(1));
        ReduceUnit reduce = stage.getReduceUnits().get(0);
        Fragment fragment = reduce.getFragments().get(0);
        Name name = fragment.getCompiled().getQualifiedName();

        ClassLoader loader = start();
        PortMapper mapper = new PortMapper(fragment);
        MockResult<Ex1> result = mapper.add("r1", new Ex1Copier());

        @SuppressWarnings("unchecked")
        Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());

        Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
        SegmentedWritable value = createShuffleValue(loader, stage);

        Ex1 ex1 = new Ex1();
        f.begin();

View Full Code Here

TOP

Related Classes of com.asakusafw.compiler.flow.stage.StageModel$ResourceFragment

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.