Package org.apache.flink.runtime.operators.testutils

Examples of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator

142
143
144
145
146
147
148
149
150
151
152
153
154
155
    final int NUM_KEYS = 100000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probeInput = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probeInput = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
255
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probeInput = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
    final int NUM_BUILD_KEYS = 1000000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 20;
    final int NUM_PROBE_VALS = 1;

    MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(
        NUM_BUILD_KEYS, NUM_BUILD_VALS, false);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 128);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }

    final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(
        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor,
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformRecordGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));

    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
        * NUM_PROBE_VALS;

    final Record recordReuse = new Record();
View Full Code Here
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
    final int NUM_BUILD_KEYS = 500000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 10;
    final int NUM_PROBE_VALS = 1;
   
    MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(NUM_BUILD_KEYS, NUM_BUILD_VALS, false);
   
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 85);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }   
       
    final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(
        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor,
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformRecordGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));
   
    final Record recordReuse = new Record();
    int numRecordsInJoinResult = 0;
   
    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
 
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
   
    try {
   
      // environment
      initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
      addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
      addOutput(this.outList);
     
      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
    final double memoryFraction = 1.0;
   
    try {
      // environment
      initEnvironment(memorySize, bufferSize);
      addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
      addOutput(this.outList);
 
      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator

  • org.apache.flink.runtime.operators.CachedMatchTaskTest
  • org.apache.flink.runtime.operators.chaining.ChainTaskTest
  • org.apache.flink.runtime.operators.CoGroupTaskExternalITCase
  • org.apache.flink.runtime.operators.CoGroupTaskTest
  • org.apache.flink.runtime.operators.CombineTaskExternalITCase
  • org.apache.flink.runtime.operators.CombineTaskTest
  • org.apache.flink.runtime.operators.CrossTaskExternalITCase
  • org.apache.flink.runtime.operators.CrossTaskTest
  • org.apache.flink.runtime.operators.DataSinkTaskTest
  • org.apache.flink.runtime.operators.DataSourceTaskTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.