Examples of org.apache.flink.api.java.ExecutionEnvironment.generateSequence()

Class org.apache.flink.api.java.ExecutionEnvironment

Examples of org.apache.flink.api.java.ExecutionEnvironment.generateSequence()

org.apache.flink.api.java.ExecutionEnvironment.generateSequence()
Creates a new data set that contains a sequence of numbers. The data set will be created in parallel, so there is no guarantee about the oder of the elements. @param from The number to start at (inclusive). @param to The number to stop at (inclusive). @return A DataSet, containing all number in the {@code [from, to]} interval.

  
  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
    DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));
    
    DataSet<Tuple2<Long, Long>> edges = edgeString.map(new EdgeParser());
    
    DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());

View Full Code Here

  @Override
  protected void testProgram() throws Exception {
    
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20).map(new Dupl());
        
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 20, 0);
    iter.closeWith(iter.getWorkset(), iter.getWorkset())
      .output(new LocalCollectionOutputFormat<Tuple2<Long, Long>>(result));

View Full Code Here

  
  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    IterativeDataSet<Long> iteration = env.generateSequence(1, 10).iterate(100);
    iteration.closeWith(iteration)
      .output(new LocalCollectionOutputFormat<Long>(result));
    
    env.execute();
  }

View Full Code Here

  public void testPipelineBreakerWithBroadcastVariable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(64);
      
      DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
      
      DataSet<Long> result = source.map(new IdentityMapper<Long>())
                    .map(new IdentityMapper<Long>())
                      .withBroadcastSet(source, "bc");

View Full Code Here

  public void testPipelineBreakerBroadcastedAllReduce() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(64);
      
      DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
      
      DataSet<Long> bcInput1 = sourceWithMapper
                    .map(new IdentityMapper<Long>())
                    .reduce(new SelectOneReducer<Long>());
      DataSet<Long> bcInput2 = env.generateSequence(1, 10);

View Full Code Here

      DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
      
      DataSet<Long> bcInput1 = sourceWithMapper
                    .map(new IdentityMapper<Long>())
                    .reduce(new SelectOneReducer<Long>());
      DataSet<Long> bcInput2 = env.generateSequence(1, 10);
      
      DataSet<Long> result = sourceWithMapper
          .map(new IdentityMapper<Long>())
              .withBroadcastSet(bcInput1, "bc1")
              .withBroadcastSet(bcInput2, "bc2");

View Full Code Here

    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(64);
      
      
      DataSet<Long> initialSource = env.generateSequence(1, 10);
      IterativeDataSet<Long> iteration = initialSource.iterate(100);
      
      
      DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());

View Full Code Here

      
      DataSet<Long> initialSource = env.generateSequence(1, 10);
      IterativeDataSet<Long> iteration = initialSource.iterate(100);
      
      
      DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
      
      DataSet<Long> bcInput1 = sourceWithMapper
                    .map(new IdentityMapper<Long>())
                    .reduce(new SelectOneReducer<Long>());

View Full Code Here

    try {
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
        
        DataSet<Long> initialSource = env.generateSequence(1, 10);
        
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST);
        initialSource
          .map(new IdentityMapper<Long>())

View Full Code Here

      
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
        
        DataSet<Long> initialSource = env.generateSequence(1, 10);
        
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.