Package org.apache.flink.runtime.operators.testutils

Examples of org.apache.flink.runtime.operators.testutils.UniformIntPairGenerator


    final int NUM_KEYS = 100000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here


    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
 
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);
View Full Code Here

    final int NUM_BUILD_KEYS = 1000000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 20;
    final int NUM_PROBE_VALS = 1;

    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_BUILD_KEYS, NUM_BUILD_VALS, false);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 128);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }

    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
        this.pairBuildSideAccesssor, this.pairProbeSideAccesssor,
        this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformIntPairGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));

    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
        * NUM_PROBE_VALS;

    final IntPair recordReuse = new IntPair();
View Full Code Here

    final int NUM_BUILD_KEYS = 500000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 10;
    final int NUM_PROBE_VALS = 1;
   
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_BUILD_KEYS, NUM_BUILD_VALS, false);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 85);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }
       
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
        this.pairBuildSideAccesssor, this.pairProbeSideAccesssor,
        this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformIntPairGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));
   
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
   
    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
 
View Full Code Here

    final int NUM_KEYS = 100000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }
   
    // ----------------------------------------------------------------------------------------
   
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
        this.pairBuildSideAccesssor, this.pairProbeSideAccesssor,
        this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
      memSegments, ioManager);
    join.open(buildInput, probeInput);
   
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
   
    while (join.nextRecord()) {
      HashBucketIterator<IntPair, IntPair> buildSide = join.getBuildSideIterator();
      while (buildSide.next(recordReuse) != null) {
        numRecordsInJoinResult++;
      }
    }
    Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
   
    join.close();

    // ----------------------------------------------------------------------------------------
    // recreate the inputs
   
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

    // create a probe input that gives 10 million pairs with 10 values sharing a key
    probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

    join.open(buildInput, probeInput);
   
    numRecordsInJoinResult = 0;
   
View Full Code Here

  public void testCompactingHashMapPerformance() {
   
    try {
      final int NUM_MEM_PAGES = SIZE * NUM_PAIRS / PAGE_SIZE;
     
      MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
     
      MutableObjectIterator<IntPair> probeTester = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
     
      MutableObjectIterator<IntPair> updater = new UniformIntPairGenerator(NUM_PAIRS, 1, false);

      MutableObjectIterator<IntPair> updateTester = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
     
      long start = 0L;
      long end = 0L;
     
      long first = System.currentTimeMillis();
     
      System.out.println("Creating and filling CompactingHashMap...");
      start = System.currentTimeMillis();
      AbstractMutableHashTable<IntPair> table = new CompactingHashTable<IntPair>(serializer, comparator, getMemory(NUM_MEM_PAGES, PAGE_SIZE));
      table.open();
     
      IntPair target = new IntPair();
      while(buildInput.next(target) != null) {
        table.insert(target);
      }
      end = System.currentTimeMillis();
      System.out.println("HashMap ready. Time: " + (end-start) + " ms");
     
      System.out.println("Starting first probing run...");
      start = System.currentTimeMillis();
     
      AbstractHashTableProber<IntPair, IntPair> prober = table.getProber(comparator, pairComparator);
      IntPair temp = new IntPair();
      while(probeTester.next(target) != null) {
        assertNotNull(prober.getMatchFor(target, temp));
        assertEquals(temp.getValue(), target.getValue());
      }
      end = System.currentTimeMillis();
      System.out.println("Probing done. Time: " + (end-start) + " ms");
     
      System.out.println("Starting update...");
      start = System.currentTimeMillis();
      while(updater.next(target) != null) {
        target.setValue(target.getValue()*-1);
        table.insertOrReplaceRecord(target, temp);
      }
      end = System.currentTimeMillis();
      System.out.println("Update done. Time: " + (end-start) + " ms");
     
      System.out.println("Starting second probing run...");
      start = System.currentTimeMillis();
      while (updateTester.next(target) != null) {
        assertNotNull(prober.getMatchFor(target, temp));
        assertEquals(target.getValue(), temp.getValue());
      }
      end = System.currentTimeMillis();
      System.out.println("Probing done. Time: " + (end-start) + " ms");
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.operators.testutils.UniformIntPairGenerator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.