Examples of org.apache.flink.runtime.jobgraph.IntermediateDataSet

Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.IntermediateDataSet

org.apache.flink.runtime.jobgraph.IntermediateDataSet
An intermediate data set is the data set produced by an operator - either a source or any intermediate operation. Intermediate data sets may be read by other operators, materialized, or discarded.

    
    // this creates an intermediate result for v1
    v2.connectNewDataSetAsInput(v1, DistributionPattern.BIPARTITE);
    
    // create results for v2 and v3
    IntermediateDataSet v2result = v2.createAndAddResultDataSet();
    IntermediateDataSet v3result_1 = v3.createAndAddResultDataSet();
    IntermediateDataSet v3result_2 = v3.createAndAddResultDataSet();
    
    
    List<AbstractJobVertex> ordered = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3));


    ExecutionGraph eg = new ExecutionGraph(jobId, jobName, cfg);

View Full Code Here

    
    // this creates an intermediate result for v1
    v2.connectNewDataSetAsInput(v1, DistributionPattern.BIPARTITE);
    
    // create results for v2 and v3
    IntermediateDataSet v2result = v2.createAndAddResultDataSet();
    IntermediateDataSet v3result_1 = v3.createAndAddResultDataSet();
    IntermediateDataSet v3result_2 = v3.createAndAddResultDataSet();
    
    
    List<AbstractJobVertex> ordered = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3));


    ExecutionGraph eg = new ExecutionGraph(jobId, jobName, cfg);
    try {
      eg.attachJobGraph(ordered);
    }
    catch (JobException e) {
      e.printStackTrace();
      fail("Job failed with exception: " + e.getMessage());
    }
    
    // attach the second part of the graph
    
    AbstractJobVertex v4 = new AbstractJobVertex("vertex4");
    AbstractJobVertex v5 = new AbstractJobVertex("vertex5");
    v4.setParallelism(11);
    v5.setParallelism(4);
    
    v4.connectIdInput(v2result.getId(), DistributionPattern.BIPARTITE);
    v4.connectIdInput(v3result_1.getId(), DistributionPattern.BIPARTITE);
    v5.connectNewDataSetAsInput(v4, DistributionPattern.BIPARTITE);
    v5.connectIdInput(v3result_2.getId(), DistributionPattern.BIPARTITE);
    
    List<AbstractJobVertex> ordered2 = new ArrayList<AbstractJobVertex>(Arrays.asList(v4, v5));
    
    try {
      eg.attachJobGraph(ordered2);

View Full Code Here

      
      v1.setParallelism(5);
      v2.setParallelism(7);
      v3.setParallelism(2);


      IntermediateDataSet result = v1.createAndAddResultDataSet();
      v2.connectDataSetAsInput(result, DistributionPattern.BIPARTITE);
      v3.connectDataSetAsInput(result, DistributionPattern.BIPARTITE);
      
      List<AbstractJobVertex> ordered = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3));

View Full Code Here

    }
    
    // create the intermediate results
    this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()];
    for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) {
      IntermediateDataSet set = jobVertex.getProducedDataSets().get(i);
      this.producedDataSets[i] = new IntermediateResult(set.getId(), this, numTaskVertices);
    }
    
    // create all task vertices
    for (int i = 0; i < numTaskVertices; i++) {
      ExecutionVertex vertex = new ExecutionVertex(this, i, this.producedDataSets, createTimestamp);

View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.jobgraph.IntermediateDataSet

org.apache.flink.runtime.executiongraph.ExecutionGraphConstructionTest

org.apache.flink.runtime.executiongraph.ExecutionJobVertex

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.