Examples of eu.stratosphere.pact.runtime.test.util.UniformIntPairGenerator

Package eu.stratosphere.pact.runtime.test.util

Examples of eu.stratosphere.pact.runtime.test.util.UniformIntPairGenerator

eu.stratosphere.pact.runtime.test.util.UniformIntPairGenerator

  public void testCompare() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, true);
    
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
      generator.next(record);
      num++;
    }
    while (sorter.write(record) && num < 3354624);
    
    // compare random elements

View Full Code Here

  }
  
  @Test
  public void testBuildFirstWithMixedDataTypes() {
    try {
      MutableObjectIterator<IntPair> input1 = new UniformIntPairGenerator(500, 40, false);
      
      final Generator generator2 = new Generator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
      final TestData.GeneratorIterator input2 = new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);
      
      // collect expected data
      final Map<TestData.Key, Collection<RecordIntPairMatch>> expectedMatchesMap = matchRecordIntPairValues(
        collectIntPairData(input1),
        collectRecordData(input2));
      
      final GenericJoiner<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
      final Collector<Record> collector = new DiscardingOutputCollector<Record>();
  
      // reset the generators
      input1 = new UniformIntPairGenerator(500, 40, false);
      generator2.reset();
      input2.reset();
  
      // compare with iterator values
      BuildSecondHashMatchIterator<IntPair, Record, Record> iterator =

View Full Code Here

  }
  
  @Test
  public void testBuildSecondWithMixedDataTypes() {
    try {
      MutableObjectIterator<IntPair> input1 = new UniformIntPairGenerator(500, 40, false);
      
      final Generator generator2 = new Generator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
      final TestData.GeneratorIterator input2 = new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);
      
      // collect expected data
      final Map<TestData.Key, Collection<RecordIntPairMatch>> expectedMatchesMap = matchRecordIntPairValues(
        collectIntPairData(input1),
        collectRecordData(input2));
      
      final GenericJoiner<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
      final Collector<Record> collector = new DiscardingOutputCollector<Record>();
  
      // reset the generators
      input1 = new UniformIntPairGenerator(500, 40, false);
      generator2.reset();
      input2.reset();
  
      // compare with iterator values
      BuildFirstHashMatchIterator<IntPair, Record, Record> iterator =

View Full Code Here

    final int NUM_KEYS = 100000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);


    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);


    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);

View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);


    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);


    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);

View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);


    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);


    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 896);

View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
  
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);

View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
  
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);

View Full Code Here

    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
  
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);

View Full Code Here

    final int NUM_BUILD_KEYS = 1000000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 20;
    final int NUM_PROBE_VALS = 1;


    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_BUILD_KEYS, NUM_BUILD_VALS, false);


    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
      memSegments = this.memManager.allocatePages(MEM_OWNER, 128);
    }
    catch (MemoryAllocationException maex) {
      fail("Memory for the Join could not be provided.");
      return;
    }


    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
        this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, 
        this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformIntPairGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));


    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
        * NUM_PROBE_VALS;


    final IntPair recordReuse = new IntPair();

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.pact.runtime.test.util.UniformIntPairGenerator

eu.stratosphere.pact.runtime.hash.HashMatchIteratorITCase

eu.stratosphere.pact.runtime.hash.HashTableITCase

eu.stratosphere.pact.runtime.hash.HashTablePerformanceComparison

eu.stratosphere.pact.runtime.sort.FixedLengthRecordSorterTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.