Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.CrossOperator

308
309
310
311
312
313
314
315
316
317
318
        .input2(mat1)
        .build();
      ReduceOperator r = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma)
        .build();
      CrossOperator c = CrossOperator.builder(new DummyCrossStub())
        .input1(r)
        .input2(mat2)
        .build();
     
      FileDataSink sinkA = new FileDataSink(new DummyOutputFormat(), out1Path, c);
View Full Code Here
381
382
383
384
385
386
387
388
389
390
391
        .input1(sourceA)
        .input2(sourceB)
        .name("CoGroup 1")
        .build();
     
      CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub())
        .input1(reduce1)
        .input2(cogroup1)
        .name("Cross 1")
        .build();
     
View Full Code Here
732
733
734
735
736
737
738
739
740
741
742
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(sourceA);
    iteration.setMaximumNumberOfIterations(10);

    CrossOperator stepFunction = CrossOperator.builder(DummyCrossStub.class).name("StepFunction").
        input1(iteration.getPartialSolution()).
        input2(sourceB).
        build();

    iteration.setNextPartialSolution(stepFunction);
View Full Code Here
781
782
783
784
785
786
787
788
789
790
791
    DeltaIteration iteration = new DeltaIteration(0, "Loop");
    iteration.setInitialSolutionSet(sourceA);
    iteration.setInitialWorkset(sourceB);
    iteration.setMaximumNumberOfIterations(10);

    CrossOperator nextWorkset = CrossOperator.builder(DummyCrossStub.class).name("Next workset").
        input1(iteration.getWorkset()).
        input2(sourceC).
        build();

    JoinOperator solutionSetDelta = JoinOperator.builder(DummyMatchStub.class, LongValue.class,0,0).
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
   
    // create DataSourceContract for data point input
    FileDataSource dataPoints = new FileDataSource(new PointInFormat(), dataPointInput, "Data Points");

    // create CrossOperator for distance computation
    CrossOperator computeDistance = CrossOperator.builder(new ComputeDistance())
        .input1(dataPoints)
        .input2(iteration.getPartialSolution())
        .name("Compute Distances")
        .build();

    // create ReduceOperator for finding the nearest cluster centers
    ReduceOperator findNearestClusterCenters = ReduceOperator.builder(new FindNearestCenter(), IntValue.class, 0)
        .input(computeDistance)
        .name("Find Nearest Centers")
        .build();

    // create ReduceOperator for computing new cluster positions
    ReduceOperator recomputeClusterCenter = ReduceOperator.builder(new RecomputeClusterCenter(), IntValue.class, 0)
        .input(findNearestClusterCenters)
        .name("Recompute Center Positions")
        .build();
    iteration.setNextPartialSolution(recomputeClusterCenter);
   
    // create DataSourceContract for data point input
    FileDataSource dataPoints2 = new FileDataSource(new PointInFormat(), dataPointInput, "Data Points 2");
   
    // compute distance of points to final clusters
    CrossOperator computeFinalDistance = CrossOperator.builder(new ComputeDistance())
        .input1(dataPoints2)
        .input2(iteration)
        .name("Compute Final Distances")
        .build();
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
   
    MapOperator map = MapOperator.builder(new IdentityMap()).name("Map1").input(source).build();
   
    ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce1").input(map).build();
   
    CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub()).name("Cross1").input1(reduce1).input2(source).build();
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce2").input(cross1).build();
   
    CrossOperator cross2 = CrossOperator.builder(new DummyCrossStub()).name("Cross2").input1(reduce2).input2(source).build();
   
    ReduceOperator reduce3 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce3").input(cross2).build();
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setInput(reduce3);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        new ContractITCaseInputFormat(), rightInPath);
    DelimitedInputFormat.configureDelimitedFormat(input_right)
      .recordDelimiter('\n');
    input_right.setDegreeOfParallelism(config.getInteger("CrossTest#NoSubtasks", 1));

    CrossOperator testCross = CrossOperator.builder(new TestCross()).build();
    testCross.setDegreeOfParallelism(config.getInteger("CrossTest#NoSubtasks", 1));
    testCross.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("CrossTest#LocalStrategy", ""));
    if (config.getString("CrossTest#ShipStrategy", "").equals("BROADCAST_FIRST")) {
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
    } else if (config.getString("CrossTest#ShipStrategy", "").equals("BROADCAST_SECOND")) {
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
    } else {
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
          config.getString("CrossTest#ShipStrategy", ""));
    }

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testCross);
    testCross.setFirstInput(input_left);
    testCross.setSecondInput(input_right);

    return new Plan(output);
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
  public void testCrossWithSmall() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
   
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
  public void testCrossWithLarge() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
   
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.CrossOperator

  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.test.operators.CrossITCase
  • eu.stratosphere.test.recordJobs.kmeans.KMeansCross
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.