Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.FileDataSink

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    testReducer.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("ReduceTest#LocalStrategy", ""));
    testReducer.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
        config.getString("ReduceTest#ShipStrategy", ""));

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testReducer);
    testReducer.setInput(input);

    Plan plan = new Plan(output);

    PactCompiler pc = new PactCompiler(new DataStatistics());
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
      CsvInputFormat.configureRecordFormat(source)
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .field(IntValue.class, 0);
     
      FileDataSink sink =
        new FileDataSink(CsvOutputFormat.class, output);
      sink.setDegreeOfParallelism(numSubtasks);
      CsvOutputFormat.configureRecordFormat(sink)
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .lenient(true)
        .field(IntValue.class, 0);
     
      sink.setGlobalOrder(new Ordering(0, IntValue.class, Order.ASCENDING), new UniformIntegerDistribution(Integer.MIN_VALUE, Integer.MAX_VALUE));
      sink.setInput(source);
     
      return new Plan(sink);
    }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
   
    iteration.setNextWorkset(updateComponentId);
    iteration.setSolutionSetDelta(updateComponentId);

    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
      .input(joinLiO)
      .name("AggLio")
      .build();

    // create DataSinkContract for writing the result
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, aggLiO, "Output");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .lenient(true)
      .field(LongValue.class, 0)
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
    iter.setNextPartialSolution(recomputeClusterCenter);

    // ---------------------- End K-Means Loop ---------------------
   
    // create DataSinkContract for writing the new cluster positions
    FileDataSink newClusterPoints = new FileDataSink(new PointOutFormat(), output, iter, "New Center Positions");

    Plan plan = new Plan(newClusterPoints, "K-Means");
    plan.setDefaultParallelism(degreeOfParallelism);
    return plan;
  }
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
      iteration.setSolutionSetDelta(joinWithSolutionSet);
    }
   
    iteration.setNextWorkset(nextWorkset);

    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, iteration, "Sink");
   
    Plan plan = new Plan(sink);
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
    return plan;
  }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
   
    CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
   
    Plan plan = new Plan(sink);
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
   
   
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
   
    CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
   
    Plan plan = new Plan(sink);
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
   
   
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
View Full Code Here
TOP

Related Classes of org.apache.flink.api.java.record.operators.FileDataSink

  • org.apache.flink.addons.hbase.example.HBaseReadExample
  • org.apache.flink.api.common.typeinfo.NothingTypeInfo
  • org.apache.flink.api.java.typeutils.RecordTypeInfo
  • org.apache.flink.compiler.AdditionalOperatorsTest
  • org.apache.flink.compiler.BranchingPlansCompilerTest
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.HardPlansCompilationTest
  • org.apache.flink.compiler.ReduceAllTest
  • org.apache.flink.compiler.UnionPropertyPropagationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.