Examples of org.apache.flink.api.common.distributions.SimpleDistribution

Package org.apache.flink.api.common.distributions

Examples of org.apache.flink.api.common.distributions.SimpleDistribution

org.apache.flink.api.common.distributions.SimpleDistribution

        .lenient(true)
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);
      
      Ordering ordering = new Ordering(0, StringValue.class, Order.DESCENDING);
      out.setGlobalOrder(ordering, new SimpleDistribution(new StringValue[] {new StringValue("N")}));
      
      Plan p = new Plan(out, "WordCount Example");
      p.setDefaultParallelism(DEFAULT_PARALLELISM);
  
      OptimizedPlan plan;

View Full Code Here

  @Test
  public void testConstructorSingleKey() {


    // check correct data distribution
    try {
      SimpleDistribution dd = new SimpleDistribution(new Key[] {new IntValue(1), new IntValue(2), new IntValue(3)});
      Assert.assertEquals(1, dd.getNumberOfFields());
    }
    catch (Throwable t) {
      Assert.fail();
    }
    
    // check incorrect key types
    try {
      new SimpleDistribution(new Key[] {new IntValue(1), new StringValue("ABC"), new IntValue(3)});
      Assert.fail("Data distribution accepts inconsistent key types");
    } catch(IllegalArgumentException iae) {
      // do nothing
    }
    
    // check inconsistent number of keys
    try {
      new SimpleDistribution(new Key[][] {{new IntValue(1)}, {new IntValue(2), new IntValue(2)}, {new IntValue(3)}});
      Assert.fail("Data distribution accepts inconsistent many keys");
    } catch(IllegalArgumentException iae) {
      // do nothing
    }
  }

View Full Code Here

  
  @Test 
  public void testConstructorMultiKey() {
    
    // check correct data distribution
    SimpleDistribution dd = new SimpleDistribution(
        new Key[][] {{new IntValue(1), new StringValue("A"), new IntValue(1)}, 
              {new IntValue(2), new StringValue("A"), new IntValue(1)}, 
              {new IntValue(3), new StringValue("A"), new IntValue(1)}});
    Assert.assertEquals(3, dd.getNumberOfFields());
    
    // check inconsistent key types
    try {
      new SimpleDistribution( 
          new Key[][] {{new IntValue(1), new StringValue("A"), new DoubleValue(1.3d)}, 
                {new IntValue(2), new StringValue("B"), new IntValue(1)}});
      Assert.fail("Data distribution accepts incorrect key types");
    } catch(IllegalArgumentException iae) {
      // do nothing
    }
    
    // check inconsistent number of keys
    try {
      dd = new SimpleDistribution(
          new Key[][] {{new IntValue(1), new IntValue(2)}, 
                {new IntValue(2), new IntValue(2)}, 
                {new IntValue(3)}});
      Assert.fail("Data distribution accepts bucket boundaries with inconsistent many keys");
    } catch(IllegalArgumentException iae) {

View Full Code Here

  }
  
  @Test
  public void testWriteRead() {
    
    SimpleDistribution ddWrite = new SimpleDistribution(
        new Key[][] {{new IntValue(1), new StringValue("A"), new IntValue(1)}, 
              {new IntValue(2), new StringValue("A"), new IntValue(1)}, 
              {new IntValue(2), new StringValue("B"), new IntValue(4)},
              {new IntValue(2), new StringValue("B"), new IntValue(3)},
              {new IntValue(2), new StringValue("B"), new IntValue(2)}});
    Assert.assertEquals(3, ddWrite.getNumberOfFields());
    
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final DataOutputStream dos = new DataOutputStream(baos);
    try {
      ddWrite.write(new OutputViewDataOutputStreamWrapper(dos));
    } catch (IOException e) {
      Assert.fail("Error serializing the DataDistribution: " + e.getMessage());
    }


    byte[] seralizedDD = baos.toByteArray();
    
    final ByteArrayInputStream bais = new ByteArrayInputStream(seralizedDD);
    final DataInputStream in = new DataInputStream(bais);
    
    SimpleDistribution ddRead = new SimpleDistribution();
    
    try {
      ddRead.read(new InputViewDataInputStreamWrapper(in));
    } catch (Exception ex) {
      Assert.fail("The deserialization of the encoded data distribution caused an error");
    }
    
    Assert.assertEquals(3, ddRead.getNumberOfFields());
    
    // compare written and read distributions
    for(int i=0;i<6;i++) {
      Key<?>[] recW = ddWrite.getBucketBoundary(0, 6);
      Key<?>[] recR = ddWrite.getBucketBoundary(0, 6);

View Full Code Here

  }
  
  @Test
  public void testGetBucketBoundary() {
    
    SimpleDistribution dd = new SimpleDistribution(
        new Key[][] {{new IntValue(1), new StringValue("A")}, 
              {new IntValue(2), new StringValue("B")}, 
              {new IntValue(3), new StringValue("C")},
              {new IntValue(4), new StringValue("D")},
              {new IntValue(5), new StringValue("E")},
              {new IntValue(6), new StringValue("F")},
              {new IntValue(7), new StringValue("G")}});
    
    Key<?>[] boundRec = dd.getBucketBoundary(0, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 1);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("A"));
    
    boundRec = dd.getBucketBoundary(1, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 2);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("B"));
    
    boundRec = dd.getBucketBoundary(2, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 3);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("C"));
    
    boundRec = dd.getBucketBoundary(3, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 4);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("D"));
    
    boundRec = dd.getBucketBoundary(4, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 5);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("E"));
    
    boundRec = dd.getBucketBoundary(5, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 6);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("F"));
    
    boundRec = dd.getBucketBoundary(6, 8);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 7);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("G"));
    
    boundRec = dd.getBucketBoundary(0, 4);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 2);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("B"));
    
    boundRec = dd.getBucketBoundary(1, 4);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 4);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("D"));
    
    boundRec = dd.getBucketBoundary(2, 4);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 6);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("F"));
    
    boundRec = dd.getBucketBoundary(0, 2);
    Assert.assertEquals(((IntValue) boundRec[0]).getValue(), 4);
    Assert.assertTrue(((StringValue) boundRec[1]).getValue().equals("D"));
    
    try {
      boundRec = dd.getBucketBoundary(0, 7);
      Assert.fail();
    } catch(IllegalArgumentException iae) {
      // nothing to do
    }
    
    try {
      boundRec = dd.getBucketBoundary(3, 4);
      Assert.fail();
    } catch(IllegalArgumentException iae) {
      // nothing to do
    }
    
    try {
      boundRec = dd.getBucketBoundary(-1, 4);
      Assert.fail();
    } catch(IllegalArgumentException iae) {
      // nothing to do
    }
    
    try {
      boundRec = dd.getBucketBoundary(0, 0);
      Assert.fail();
    } catch(IllegalArgumentException iae) {
      // nothing to do
    }
  }

View Full Code Here

TOP

Related Classes of org.apache.flink.api.common.distributions.SimpleDistribution

org.apache.flink.api.distributions.SimpleDataDistributionTest

org.apache.flink.test.compiler.examples.WordCountCompilerTest

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.