Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.MapOperator

681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(source2);
    iteration.setMaximumNumberOfIterations(10);
   
    MapOperator inMap = MapOperator.builder(new IdentityMap())
                               .input(source1)
                               .name("In Iteration Map")
                               .setBroadcastVariable("BC", iteration.getPartialSolution())
                               .build();
   
    iteration.setNextPartialSolution(inMap);
   
    MapOperator postMap = MapOperator.builder(new IdentityMap())
                     .input(source1)
                     .name("Post Iteration Map")
                     .setBroadcastVariable("BC", iteration)
                     .build();
   
View Full Code Here
833
834
835
836
837
838
839
840
841
842
843
   */
  @Test
  public void testDeltaIterationWithStaticInput() {
    FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");

    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();

    ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
View Full Code Here
896
897
898
899
900
901
902
903
904
905
906
   */
  @Test
  public void testIterationWithStaticInput() {
    FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");

    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();

    ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
   

    FileDataSource edges = new FileDataSource(new EdgeInputFormat(), edgeInput, "Input Edges");
    edges.setParameter(EdgeInputFormat.ID_DELIMITER_CHAR, delimiter);
   
    MapOperator projectEdge = MapOperator.builder(new ProjectEdge())
      .input(edges).name("Project Edge").build();
   
    ReduceOperator edgeCounter = ReduceOperator.builder(new CountEdges(), IntValue.class, 0)
      .input(projectEdge).name("Count Edges for Vertex").build();
   
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
 
  static Plan getTestPlanPlan(int numSubTasks, String input, String output) {
   
    FileDataSource source = new FileDataSource(new TextInputFormat(), input, "Input Lines");
    source.setParameter(TextInputFormat.CHARSET_NAME, "ASCII");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    final int maxIterations = (args.length > 4 ? Integer.parseInt(args[4]) : 1);

    // data source for initial vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    // data source for cluster center input
    @SuppressWarnings("unchecked")
    FileDataSource clustersSource = new FileDataSource(new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class), clusterInput, "Centers");
   
    MapOperator dataPoints = MapOperator.builder(new PointBuilder()).name("Build data points").input(pointsSource).build();
   
    MapOperator clusterPoints = MapOperator.builder(new PointBuilder()).name("Build cluster points").input(clustersSource).build();
   
    // ---------------------- Begin K-Means Loop ---------------------
   
    BulkIteration iter = new BulkIteration("k-means loop");
    iter.setInput(clusterPoints);
    iter.setMaximumNumberOfIterations(numIterations);

    // compute the distances and select the closest center
    MapOperator findNearestClusterCenters = MapOperator.builder(new SelectNearestCenter())
      .setBroadcastVariable("centers", iter.getPartialSolution())
      .input(dataPoints)
      .name("Find Nearest Centers")
      .build();
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
    final int maxIterations = (args.length > 4 ? Integer.parseInt(args[4]) : 1);

    // data source for initial vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    // the loop takes the vertices as the solution set and changed vertices as the workset
    // initially, all vertices are changed
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
   
    // init data source
    FileDataSource input = new FileDataSource(new ContractITCaseInputFormat(), inputPath);

    // init failing map task
    MapOperator testMapper = MapOperator.builder(FailingMapper.class).build();

    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);

    // compose failing program
    output.setInput(testMapper);
    testMapper.setInput(input);

    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
   
    // init data source
    FileDataSource input = new FileDataSource(new ContractITCaseInputFormat(), inputPath);

    // init (working) map task
    MapOperator testMapper = MapOperator.builder(TestMapper.class).build();

    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);

    // compose working program
    output.setInput(testMapper);
    testMapper.setInput(input);

    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.MapOperator

  • eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.hadoopcompatibility.example.WordCountWithHadoopOutputFormat
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
  • eu.stratosphere.test.accumulators.AccumulatorITCase
  • eu.stratosphere.test.broadcastvars.BroadcastBranchingITCase
  • eu.stratosphere.test.cancelling.MapCancelingITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.