/**
* Copyright 2011-2014 Asakusa Framework Team.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.asakusafw.compiler.flow.processor;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.io.Writable;
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;
import com.asakusafw.compiler.flow.JobflowCompilerTestRoot;
import com.asakusafw.compiler.flow.processor.flow.SummarizeFlowKeyConflict;
import com.asakusafw.compiler.flow.processor.flow.SummarizeFlowRenameKey;
import com.asakusafw.compiler.flow.processor.flow.SummarizeFlowTrivial;
import com.asakusafw.compiler.flow.stage.ShuffleModel.Segment;
import com.asakusafw.compiler.flow.stage.StageModel;
import com.asakusafw.compiler.flow.stage.StageModel.Fragment;
import com.asakusafw.compiler.flow.stage.StageModel.ReduceUnit;
import com.asakusafw.compiler.flow.testing.model.Ex1;
import com.asakusafw.compiler.flow.testing.model.ExSummarized;
import com.asakusafw.compiler.flow.testing.model.ExSummarized2;
import com.asakusafw.compiler.flow.testing.model.KeyConflict;
import com.asakusafw.compiler.util.tester.CompilerTester;
import com.asakusafw.compiler.util.tester.CompilerTester.TestInput;
import com.asakusafw.compiler.util.tester.CompilerTester.TestOutput;
import com.asakusafw.runtime.flow.Rendezvous;
import com.asakusafw.runtime.flow.SegmentedWritable;
import com.asakusafw.runtime.testing.MockResult;
import com.asakusafw.utils.java.model.syntax.Name;
/**
* Test for {@link SummarizeFlowProcessor}.
*/
public class SummarizeFlowProcessorTest extends JobflowCompilerTestRoot {
/**
* テストヘルパ。
*/
@Rule
public CompilerTester tester = new CompilerTester();
/**
* 単純なテスト。
*/
@Test
public void trivial() {
run(false);
}
/**
* Combinerつきの単純なテスト。
*/
@Test
public void combine() {
run(true);
}
/**
* 単純なテスト。
* @throws Exception if test was failed
*/
@Test
public void renameKey() throws Exception {
runRenameKey(false);
}
/**
* Combinerつきの単純なテスト。
* @throws Exception if test was failed
*/
@Test
public void combineRenameKey() throws Exception {
runRenameKey(true);
}
/**
* Test for grouping key is conflict.
* @throws Exception if test was failed
*/
@Test
public void conflictKey() throws Exception {
runKeyConflict(false);
}
/**
* Test for grouping key is conflict with combiner.
* @throws Exception if test was failed
*/
@Test
public void combineConflictKey() throws Exception {
runKeyConflict(true);
}
private void run(boolean combine) {
environment.getOptions().setEnableCombiner(combine);
List<StageModel> stages = compile(SummarizeFlowTrivial.class);
StageModel stage = stages.get(0);
Assume.assumeThat(stage.getReduceUnits().size(), is(1));
ReduceUnit reduce = stage.getReduceUnits().get(0);
Fragment fragment = reduce.getFragments().get(0);
Name name = fragment.getCompiled().getQualifiedName();
ClassLoader loader = start();
PortMapper mapper = new PortMapper(fragment);
MockResult<ExSummarized> result = mapper.create("out");
@SuppressWarnings("unchecked")
Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());
Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
SegmentedWritable value = createShuffleValue(loader, stage);
ExSummarized ex1 = new ExSummarized();
ex1.setCount(1);
f.begin();
ex1.setValue(10);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(20);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(30);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(40);
setShuffleValue(segment, value, ex1);
f.process(value);
f.end();
assertThat(result.getResults().size(), is(1));
assertThat(result.getResults().get(0).getValue(), is(100L));
assertThat(result.getResults().get(0).getCount(), is(4L));
}
private void runRenameKey(boolean combine) throws Exception {
environment.getOptions().setEnableCombiner(combine);
TestInput<Ex1> in = tester.input(Ex1.class, "Ex1");
TestOutput<ExSummarized2> summarized = tester.output(ExSummarized2.class, "summarized");
Ex1 ex1 = new Ex1();
ex1.setStringAsString("a");
ex1.setSid(1);
ex1.setValue(10);
in.add(ex1);
ex1.setSid(2);
ex1.setValue(20);
in.add(ex1);
ex1.setStringAsString("b");
ex1.setSid(3);
ex1.setValue(30);
in.add(ex1);
ex1.setSid(4);
ex1.setValue(40);
in.add(ex1);
ex1.setSid(5);
ex1.setValue(50);
in.add(ex1);
assertThat(tester.runFlow(new SummarizeFlowRenameKey(
in.flow(), summarized.flow())), is(true));
List<ExSummarized2> results = summarized.toList(new Comparator<ExSummarized2>() {
@Override
public int compare(ExSummarized2 o1, ExSummarized2 o2) {
return o1.getKeyOption().compareTo(o2.getKeyOption());
}
});
assertThat(results.size(), is(2));
assertThat(results.get(0).getKeyAsString(), is("a"));
assertThat(results.get(0).getCount(), is(2L));
assertThat(results.get(0).getValue(), is(30L));
assertThat(results.get(1).getKeyAsString(), is("b"));
assertThat(results.get(1).getCount(), is(3L));
assertThat(results.get(1).getValue(), is(120L));
}
private void runKeyConflict(boolean combine) throws Exception {
environment.getOptions().setEnableCombiner(combine);
TestInput<Ex1> in = tester.input(Ex1.class, "Ex1");
TestOutput<KeyConflict> summarized = tester.output(KeyConflict.class, "summarized");
Ex1 ex1 = new Ex1();
ex1.setStringAsString("a");
ex1.setSid(1);
in.add(ex1);
ex1.setSid(2);
in.add(ex1);
ex1.setStringAsString("b");
ex1.setSid(3);
in.add(ex1);
ex1.setSid(4);
in.add(ex1);
ex1.setSid(5);
in.add(ex1);
assertThat(tester.runFlow(new SummarizeFlowKeyConflict(
in.flow(), summarized.flow())), is(true));
List<KeyConflict> results = summarized.toList(new Comparator<KeyConflict>() {
@Override
public int compare(KeyConflict o1, KeyConflict o2) {
return o1.getKeyOption().compareTo(o2.getKeyOption());
}
});
assertThat(results.size(), is(2));
assertThat(results.get(0).getKeyAsString(), is("a"));
assertThat(results.get(0).getCount(), is(2L));
assertThat(results.get(1).getKeyAsString(), is("b"));
assertThat(results.get(1).getCount(), is(3L));
}
}