Examples of org.apache.flink.runtime.operators.shipping.RecordOutputEmitter

Package org.apache.flink.runtime.operators.shipping

Examples of org.apache.flink.runtime.operators.shipping.RecordOutputEmitter

org.apache.flink.runtime.operators.shipping.RecordOutputEmitter

  @Test
  public void testPartitionHash() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, intComp);


    int numChans = 100;
    int numRecs = 50000;
    int[] hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      int[] chans = oe1.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] > 0);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecs);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final RecordComparator stringComp = new RecordComparator(new int[] {0}, new Class[] {StringValue.class});
    ChannelSelector<Record> oe2 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, stringComp);


    numChans = 100;
    numRecs = 10000;
    
    hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
        
      int[] chans = oe2.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

View Full Code Here

  @Test
  public void testForward() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.FORWARD, intComp);


    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      int[] chans = oe1.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] == (numRecords/numChannels) || hit[i] == (numRecords/numChannels)-1);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecords);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final RecordComparator stringComp = new RecordComparator(new int[] {0}, new Class[] {StringValue.class});
    final ChannelSelector<Record> oe2 = new RecordOutputEmitter(ShipStrategyType.FORWARD, stringComp);


    numChannels = 100;
    numRecords = 10000;
    
    hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
        
      int[] chans = oe2.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

View Full Code Here

  @Test
  public void testBroadcast() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.BROADCAST, intComp);


    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      int[] chans = oe1.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i]+"", hit[i] == numRecords);
    }
    
    // Test for StringValue
    @SuppressWarnings("unchecked")
    final RecordComparator stringComp = new RecordComparator(new int[] {0}, new Class[] {StringValue.class});
    final ChannelSelector<Record> oe2 = new RecordOutputEmitter(ShipStrategyType.BROADCAST, stringComp);


    numChannels = 100;
    numRecords = 5000;
    
    hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
        
      int[] chans = oe2.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

View Full Code Here

  
  @Test
  public void testMultiKeys() {
    @SuppressWarnings("unchecked")
    final RecordComparator multiComp = new RecordComparator(new int[] {0,1,3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, multiComp);


    int numChannels = 100;
    int numRecords = 5000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      Record rec = new Record(4);
      rec.setField(0, new IntValue(i));
      rec.setField(1, new StringValue("AB"+i+"CD"+i));
      rec.setField(3, new DoubleValue(i*3.141d));
      
      int[] chans = oe1.selectChannels(rec, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

View Full Code Here

  @Test
  public void testMissingKey() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {1}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, intComp);


    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    
    try {
      oe1.selectChannels(rec, 100);
    } catch (KeyFieldOutOfBoundsException re) {
      Assert.assertEquals(1, re.getFieldNumber());
      return;
    }
    Assert.fail("Expected a KeyFieldOutOfBoundsException.");

View Full Code Here

  @Test
  public void testNullKey() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, intComp);


    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));


    try {
      oe1.selectChannels(rec, 100);
    } catch (NullKeyFieldException re) {
      Assert.assertEquals(0, re.getFieldNumber());
      return;
    }
    Assert.fail("Expected a NullKeyFieldException.");

View Full Code Here

  public void testWrongKeyClass() {
    
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final RecordComparator doubleComp = new RecordComparator(new int[] {0}, new Class[] {DoubleValue.class});
    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, doubleComp);


    PipedInputStream pipedInput = new PipedInputStream(1024*1024);
    DataInputStream in = new DataInputStream(pipedInput);
    DataOutputStream out;
    Record rec = null;
    
    try {
      out = new DataOutputStream(new PipedOutputStream(pipedInput));
      
      rec = new Record(1);
      rec.setField(0, new IntValue());
      
      rec.write(new OutputViewDataOutputStreamWrapper(out));
      rec = new Record();
      rec.read(new InputViewDataInputStreamWrapper(in));
    
    } catch (IOException e) {
      fail("Test erroneous");
    }


    try {
      oe1.selectChannels(rec, 100);
    } catch (DeserializationException re) {
      return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
  }

View Full Code Here

    
    final DataDistribution distri = new UniformIntegerDistribution(DISTR_MIN, DISTR_MAX);
    
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe = new RecordOutputEmitter(ShipStrategyType.PARTITION_RANGE, intComp, distri);
    
    final IntValue integer = new IntValue();
    final Record rec = new Record();
    
    for (int i = 0; i < NUM_ELEMENTS; i++) {
      final int nextValue = rnd.nextInt(DISTR_RANGE) + DISTR_MIN;
      integer.setValue(nextValue);
      rec.setField(0, integer);
      
      final int[] channels = oe.selectChannels(rec, NUM_BUCKETS);
      if (channels.length != 1) {
        Assert.fail("Resulting channels array has more than one channel.");
      }
      
      final int bucket = channels[0];

View Full Code Here

      // create a writer for each output
      for (int i = 0; i < numOutputs; i++) {
        // create the OutputEmitter from output ship strategy
        final ShipStrategyType strategy = config.getOutputShipStrategy(i);
        final TypeComparatorFactory<?> compFact = config.getOutputComparator(i, cl);
        final RecordOutputEmitter oe;
        if (compFact == null) {
          oe = new RecordOutputEmitter(strategy);
        } else {
          @SuppressWarnings("unchecked")
          TypeComparator<Record> comparator = (TypeComparator<Record>) compFact.createComparator();
          if (!comparator.supportsCompareAgainstReference()) {
            throw new Exception("Incompatibe serializer-/comparator factories.");
          }
          final DataDistribution distribution = config.getOutputDataDistribution(i, cl);
          oe = new RecordOutputEmitter(strategy, comparator, distribution);
        }


        writers.add(new RecordWriter<Record>(task, oe));
      }
      if (eventualOutputs != null) {

View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.operators.shipping.RecordOutputEmitter

org.apache.flink.runtime.operators.RegularPactTask

org.apache.flink.runtime.operators.util.RecordOutputEmitterTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.