Package org.apache.flink.runtime.operators.testutils

Examples of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator

279
280
281
282
283
284
285
286
287
288
289
  //
  //
 
  private final MutableObjectIterator<Record> getProbeInput(final int numKeys,
      final int probeValsPerKey, final int repeatedValue1, final int repeatedValue2) {
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(numKeys, probeValsPerKey, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(repeatedValue1, 17, 5);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(repeatedValue2, 23, 5);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here
361
362
363
364
365
366
367
368
369
370
371
372
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
   
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
   
    try {
      addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
      addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("The test caused an exception.");
    }
   
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
View Full Code Here
422
423
424
425
426
427
428
429
430
431
432
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    int valCnt = 20;
   
    this.outList = new ArrayList<Record>();
   
    try {
      InputFilePreparator.prepareInputFile(new UniformRecordGenerator(keyCnt, valCnt, false),
        this.tempTestPath, true);
    } catch (IOException e1) {
      Assert.fail("Unable to set-up test input file");
    }
   
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
    int valCnt = 10;
   
    this.outList = new NirvanaOutputList();
   
    try {
      InputFilePreparator.prepareInputFile(new UniformRecordGenerator(keyCnt, valCnt, false),
        this.tempTestPath, false);
    } catch (IOException e1) {
      Assert.fail("Unable to set-up test input file");
    }
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addOutput(new NirvanaOutputList());
   
    try {
      InputFilePreparator.prepareInputFile(new UniformRecordGenerator(keyCnt, valCnt, false),
        this.tempTestPath, false);
    } catch (IOException e1) {
      Assert.fail("Unable to set-up test input file");
    }
   
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    addDriverComparator(this.comparator);
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
   
    try {
      addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
     
      GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<Record, Record>();
     
      testDriver(testTask, MockReduceStub.class);
    } catch (Exception e) {
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  @Test
  public void testReduceTaskOnPreSortedInput() {
    final int keyCnt = 100;
    final int valCnt = 20;
   
    addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
    addDriverComparator(this.comparator);
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
   
    GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<Record, Record>();
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
   
    CombiningUnilateralSortMerger<Record> sorter = null;
    try {
      sorter = new CombiningUnilateralSortMerger<Record>(new MockCombiningReduceStub(),
        getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false),
        getOwningNepheleTask(), RecordSerializerFactory.get(), this.comparator.duplicate(), this.perSortFractionMem,
          4, 0.8f);
      addInput(sorter.getIterator());
     
      GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<Record, Record>();
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator

  • org.apache.flink.runtime.operators.CachedMatchTaskTest
  • org.apache.flink.runtime.operators.chaining.ChainTaskTest
  • org.apache.flink.runtime.operators.CoGroupTaskExternalITCase
  • org.apache.flink.runtime.operators.CoGroupTaskTest
  • org.apache.flink.runtime.operators.CombineTaskExternalITCase
  • org.apache.flink.runtime.operators.CombineTaskTest
  • org.apache.flink.runtime.operators.CrossTaskExternalITCase
  • org.apache.flink.runtime.operators.CrossTaskTest
  • org.apache.flink.runtime.operators.DataSinkTaskTest
  • org.apache.flink.runtime.operators.DataSourceTaskTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.