Package org.apache.flink.compiler.util

Examples of org.apache.flink.compiler.util.IdentityReduce

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    // construct the plan

    FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
    FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
   
    ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceA)
      .build();
    ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceB)
      .build();
   
    ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
    globalRed.addInput(redA);
    globalRed.addInput(redB);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
   
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
  @Test
  public void testReduceWithGroupOrder() {
    // construct the plan
    FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
   
    ReduceOperator reduce = ReduceOperator.builder(new IdentityReduce()).keyField(IntValue.class, 2).name("Reduce").input(source).build();
    Ordering groupOrder = new Ordering(5, StringValue.class, Order.DESCENDING);
    reduce.setGroupOrder(groupOrder);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, reduce, "Sink");
   
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
        .input1(iteration.getSolutionSet())
        .input2(joinWithInvariant)
        .name(JOIN_WITH_SOLUTION_SET)
        .build();
   
    ReduceOperator nextWorkset = ReduceOperator.builder(new IdentityReduce(), LongValue.class, 0)
        .input(joinWithSolutionSet)
        .name(NEXT_WORKSET_REDUCER_NAME)
        .build();
   
    if (mapBeforeSolutionDelta) {
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
        .build();
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat1)
        .build();
      ReduceOperator r = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma)
        .build();
      CrossOperator c = CrossOperator.builder(new DummyCrossStub())
        .input1(r)
        .input2(mat2)
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
     
      MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
     
      ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(map1)
        .name("Reduce 1")
        .build();
     
      JoinOperator match1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
View Full Code Here
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
        .name("Match 1")
        .build();
     
      MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
     
      ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma1)
        .name("Reduce 1")
        .build();
     
      ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(mat1)
        .name("Reduce 2")
        .build();
     
      MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
   
    MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
    map1.setDegreeOfParallelism(degOfPar);
    map1.setInput(source);
   
    ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
    reduce1.setDegreeOfParallelism(degOfPar);
    reduce1.setInput(map1);
   
    MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
    map2.setDegreeOfParallelism(degOfPar * 2);
    map2.setInput(reduce1);
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
 
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
   
    MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
    map1.setDegreeOfParallelism(degOfPar);
    map1.setInput(source);
   
    ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
    reduce1.setDegreeOfParallelism(degOfPar);
    reduce1.setInput(map1);
   
    MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
    map2.setDegreeOfParallelism(degOfPar);
    map2.setInput(reduce1);
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
 
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
   
    MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
    map1.setDegreeOfParallelism(degOfPar);
    map1.setInput(source);
   
    ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
    reduce1.setDegreeOfParallelism(degOfPar);
    reduce1.setInput(map1);
   
    MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
    map2.setDegreeOfParallelism(degOfPar * 2);
    map2.setInput(reduce1);
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
 
View Full Code Here
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
   
    MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
    map1.setDegreeOfParallelism(degOfPar * 2);
    map1.setInput(source);
   
    ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
    reduce1.setDegreeOfParallelism(degOfPar * 2);
    reduce1.setInput(map1);
   
    MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
    map2.setDegreeOfParallelism(degOfPar);
    map2.setInput(reduce1);
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar);
View Full Code Here
TOP

Related Classes of org.apache.flink.compiler.util.IdentityReduce

  • org.apache.flink.compiler.BranchingPlansCompilerTest
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.HardPlansCompilationTest
  • org.apache.flink.compiler.ReduceAllTest
  • org.apache.flink.compiler.UnionPropertyPropagationTest
  • org.apache.flink.compiler.WorksetIterationsRecordApiCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.