Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.FileDataSink

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
   
    FileDataSource lineItems =
      new FileDataSource(new IntTupleDataInFormat(), this.lineItemInputPath, "LineItems");
    lineItems.setDegreeOfParallelism(this.degreeOfParallelism);
   
    FileDataSink result =
      new FileDataSink(new StringTupleDataOutFormat(), this.outputPath, "Output");
    result.setDegreeOfParallelism(this.degreeOfParallelism);
   
    MapOperator lineItemFilter =
      MapOperator.builder(new LineItemFilter())
      .name("LineItem Filter")
      .build();
    lineItemFilter.setDegreeOfParallelism(this.degreeOfParallelism);
   
    ReduceOperator groupByReturnFlag =
      ReduceOperator.builder(new GroupByReturnFlag(), StringValue.class, 0)
      .name("groupyBy")
      .build();
   
    lineItemFilter.setInput(lineItems);
    groupByReturnFlag.setInput(lineItemFilter);
    result.setInput(groupByReturnFlag);
   
    return new Plan(result, "TPC-H 1");
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    CollectionDataSource source2 = new CollectionDataSource(tmp, "test_collection");

    JoinOperator join = JoinOperator.builder(Join.class, IntValue.class, 0, 0)
      .input1(source).input2(source2).build();

    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, join, "Collection Join");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(IntValue.class, 1);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    input2.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1));
   
    MapOperator testMapper = MapOperator.builder(new TestMapper()).build();
    testMapper.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1));

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testMapper);

    testMapper.addInput(input1);
    testMapper.addInput(input2);

    return new Plan(output);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
        .name("Compute termination criterion (Map)")
        .build();
   
    iteration.setTerminationCriterion(terminationMapper);

    FileDataSink finalResult = new FileDataSink(CsvOutputFormat.class, output, iteration, "Output");
    CsvOutputFormat.configureRecordFormat(finalResult)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0);
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
    MapOperator mapper = MapOperator.builder(new TokenizeLine()).input(source).name("Tokenize Lines").build();
   
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0).input(mapper)
        .name("Count Words").build();
   
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
   
    CsvOutputFormat.configureRecordFormat(out).recordDelimiter('\n')
        .fieldDelimiter(' ').field(StringValue.class, 0)
        .field(IntValue.class, 1);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
      .input(source)
      .name("Ordered Reducer")
      .build();
    reducer.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));
   
    FileDataSink sink = new FileDataSink(CsvOutputFormat.class, this.resultPath, reducer, "Sink");
    CsvOutputFormat.configureRecordFormat(sink)
      .recordDelimiter('\n')
      .fieldDelimiter(',')
      .field(IntValue.class, 0)
      .field(IntValue.class, 1);
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    input.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));

    MapOperator testMapper = MapOperator.builder(new TestMapper()).build();
    testMapper.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testMapper);
    testMapper.setInput(input);

    return new Plan(output);
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
        .name("Compute sum (Reduce)")
        .build();
   
    iteration.setNextPartialSolution(sumReduce);

    FileDataSink finalResult = new FileDataSink(CsvOutputFormat.class, output, iteration, "Output");
    CsvOutputFormat.configureRecordFormat(finalResult)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0);
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
        .input(computeFinalDistance)
        .name("Find Nearest Final Centers")
        .build();

    // create DataSinkContract for writing the new cluster positions
    FileDataSink finalClusters = new FileDataSink(new PointOutFormat(), output+"/centers", iteration, "Cluster Positions");

    // write assigned clusters
    FileDataSink clusterAssignments = new FileDataSink(new PointOutFormat(), output+"/points", findNearestFinalCluster, "Cluster Assignments");
   
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(finalClusters);
    sinks.add(clusterAssignments);
   
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
   
    @SuppressWarnings("unchecked")
    FileDataSink out = new FileDataSink(new CsvOutputFormat("\n", " ", StringValue.class, IntValue.class), output, reducer, "Word Counts");
   
    Plan plan = new Plan(out, "WordCount Example");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }
View Full Code Here
TOP

Related Classes of org.apache.flink.api.java.record.operators.FileDataSink

  • org.apache.flink.addons.hbase.example.HBaseReadExample
  • org.apache.flink.api.common.typeinfo.NothingTypeInfo
  • org.apache.flink.api.java.typeutils.RecordTypeInfo
  • org.apache.flink.compiler.AdditionalOperatorsTest
  • org.apache.flink.compiler.BranchingPlansCompilerTest
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.HardPlansCompilationTest
  • org.apache.flink.compiler.ReduceAllTest
  • org.apache.flink.compiler.UnionPropertyPropagationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.