Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.FileDataSource

115
116
117
118
119
120
121
122
123
124
125
    // parse job parameters
    int numSubTasks   = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
    String dataInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");

    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    }
  }

  @Override
  protected JobGraph getJobGraph() throws Exception {
    FileDataSource input = new FileDataSource(
        new ContractITCaseInputFormat(), inPath);
    DelimitedInputFormat.configureDelimitedFormat(input)
      .recordDelimiter('\n');
    input.setDegreeOfParallelism(config.getInteger("ReduceTest#NoSubtasks", 1));

    ReduceOperator testReducer = ReduceOperator.builder(new TestReducer(), StringValue.class, 0)
      .build();
    testReducer.setDegreeOfParallelism(config.getInteger("ReduceTest#NoSubtasks", 1));
    testReducer.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  @SuppressWarnings("unchecked")
  private static Plan getPlanForWorksetConnectedComponentsWithSolutionSetAsFirstInput(
      int numSubTasks, String verticesInput, String edgeInput, String output, int maxIterations)
  {
    // data source for initial vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
   
    // create DataSourceContract for the edges
    FileDataSource edges = new FileDataSource(new CsvInputFormat(' ', LongValue.class, LongValue.class), edgeInput, "Edges");

    // create CrossOperator for distance computation
    JoinOperator joinWithNeighbors = JoinOperator.builder(new NeighborWithComponentIDJoin(), LongValue.class, 0, 0)
        .input1(iteration.getWorkset())
        .input2(edges)
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
      // parse program parameters
      int numSubtasks       = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
      String recordsPath    = (args.length > 1 ? args[1] : "");
      String output        = (args.length > 2 ? args[2] : "");
     
      FileDataSource source = new FileDataSource(CsvInputFormat.class, recordsPath);
      source.setDegreeOfParallelism(numSubtasks);
      CsvInputFormat.configureRecordFormat(source)
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .field(IntValue.class, 0);
     
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    final String ordersPath    = (args.length > 1 ? args[1] : "");
    final String lineitemsPath = (args.length > 2 ? args[2] : "");
    final String output        = (args.length > 3 ? args[3] : "");

    // create DataSourceContract for Orders input
    FileDataSource orders = new FileDataSource(new CsvInputFormat(), ordersPath, "Orders");
    CsvInputFormat.configureRecordFormat(orders)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .field(LongValue.class, 0)    // order id
      .field(IntValue.class, 7)     // ship prio
      .field(StringValue.class, 2, 2)  // order status
      .field(StringValue.class, 4, 10)  // order date
      .field(StringValue.class, 5, 8);  // order prio

    // create DataSourceContract for LineItems input
    FileDataSource lineitems = new FileDataSource(new CsvInputFormat(), lineitemsPath, "LineItems");
    CsvInputFormat.configureRecordFormat(lineitems)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .field(LongValue.class, 0)    // order id
      .field(DoubleValue.class, 5);  // extended price
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    String clusterInput = (args.length > 2 ? args[2] : "");
    String output = (args.length > 3 ? args[3] : "");
    int numIterations = (args.length > 4 ? Integer.parseInt(args[4]) : 2);

    // data source data point input
    @SuppressWarnings("unchecked")
    FileDataSource pointsSource = new FileDataSource(new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class), dataPointInput, "Data Points");

    // data source for cluster center input
    @SuppressWarnings("unchecked")
    FileDataSource clustersSource = new FileDataSource(new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class), clusterInput, "Centers");
   
    MapOperator dataPoints = MapOperator.builder(new PointBuilder()).name("Build data points").input(pointsSource).build();
   
    MapOperator clusterPoints = MapOperator.builder(new PointBuilder()).name("Build cluster points").input(clustersSource).build();
   
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
208
209
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
 
  private Plan getRecordTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
    FileDataSource solutionSetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Solution Set");
    FileDataSource worksetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Workset");
   
    FileDataSource invariantInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Invariant Input");
   
    DeltaIteration iteration = new DeltaIteration(0, ITERATION_NAME);
    iteration.setInitialSolutionSet(solutionSetInput);
    iteration.setInitialWorkset(worksetInput);
    iteration.setMaximumNumberOfIterations(100);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
public class AdditionalOperatorsTest extends CompilerTestBase {

  @Test
  public void testCrossWithSmall() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
  }
 
  @Test
  public void testCrossWithLarge() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
  @Test
  public void checkPropertyHandlingWithIncreasingGlobalParallelism1() {
    final int degOfPar = DEFAULT_PARALLELISM;
   
    // construct the plan
    FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
    source.setDegreeOfParallelism(degOfPar);
   
    MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
    map1.setDegreeOfParallelism(degOfPar);
    map1.setInput(source);
   
View Full Code Here
TOP

Related Classes of org.apache.flink.api.java.record.operators.FileDataSource

  • org.apache.flink.compiler.AdditionalOperatorsTest
  • org.apache.flink.compiler.BranchingPlansCompilerTest
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.HardPlansCompilationTest
  • org.apache.flink.compiler.ReduceAllTest
  • org.apache.flink.compiler.UnionPropertyPropagationTest
  • org.apache.flink.compiler.WorksetIterationsRecordApiCompilerTest
  • org.apache.flink.test.accumulators.AccumulatorIterativeITCase
  • org.apache.flink.test.broadcastvars.BroadcastBranchingITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.