Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.ReduceOperator

114
115
116
117
118
119
120
121
122
123
124
    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
   
    @SuppressWarnings("unchecked")
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      FileDataSource sourceNode = new FileDataSource(new TextInputFormat(), IN_FILE, "Input Lines");
      MapOperator mapNode = MapOperator.builder(new TokenizeLine())
        .input(sourceNode)
        .name("Tokenize Lines")
        .build();
      ReduceOperator reduceNode = ReduceOperator.builder(new CountWords(), StringValue.class, 0)
        .input(mapNode)
        .name("Count Words")
        .build();
      FileDataSink out = new FileDataSink(new CsvOutputFormat(), OUT_FILE, reduceNode, "Word Counts");
      CsvOutputFormat.configureRecordFormat(out)
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();

    // create ReduceOperator for finding the nearest cluster centers
    ReduceOperator minCandidateId = ReduceOperator.builder(new MinimumComponentIDReduce(), LongValue.class, 0)
        .input(joinWithNeighbors)
        .name("Find Minimum Candidate Id")
        .build();
   
    // create CrossOperator for distance computation
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    // construct the plan

    FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
    FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
   
    ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceA)
      .build();
    ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceB)
      .build();
   
    ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
    globalRed.addInput(redA);
    globalRed.addInput(redB);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
   
    // return the plan
    Plan plan = new Plan(sink, "Union Property Propagation");
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
        .input(source)
        .name("Tokenize Lines")
        .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
        .input(mapper)
        .name("Count Words")
        .build();
    HadoopDataSink<Text, IntWritable> out = new HadoopDataSink<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(),new JobConf(), "Hadoop TextOutputFormat", reducer, Text.class, IntWritable.class);
    TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
   
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
    CsvOutputFormat.configureRecordFormat(out)
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
        .build();
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat1)
        .build();
      ReduceOperator r = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma)
        .build();
      CrossOperator c = CrossOperator.builder(new DummyCrossStub())
        .input1(r)
        .input2(mat2)
View Full Code Here
363
364
365
366
367
368
369
370
371
372
373
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
     
      MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
     
      ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(map1)
        .name("Reduce 1")
        .build();
     
      @SuppressWarnings("unchecked")
View Full Code Here
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
        .name("Match 1")
        .build();
     
      MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
     
      ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma1)
        .name("Reduce 1")
        .build();
     
      ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(mat1)
        .name("Reduce 2")
        .build();
     
      MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
View Full Code Here
838
839
840
841
842
843
844
845
846
847
848
    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();

    ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
        input(source).
        name("Identity reduce source").
        build();

    DeltaIteration iteration = new DeltaIteration(0,"Loop");
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.ReduceOperator

  • eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.hadoopcompatibility.example.WordCountWithHadoopOutputFormat
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.