Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSink

88
89
90
91
92
93
94
95
96
97
98
    MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build();

    // Mp2 filters out all p values which can be divided by z
    MapOperator mp2 = MapOperator.builder(Mp2.class).setBroadcastVariable("z", mp1).input(jn2).build();

    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);
    output.setInput(mp2);

    return new Plan(output);
  }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("CoGroupTest#LocalStrategy", ""));
    testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
        config.getString("CoGroupTest#ShipStrategy", ""));

    FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testCoGrouper);
    testCoGrouper.setFirstInput(input_left);
    testCoGrouper.setSecondInput(input_right);

    return new Plan(output);
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    @SuppressWarnings("unchecked")
    MapOperator map2 = MapOperator.builder(new IdentityMapper()).input(iteration.getPartialSolution(), iteration.getPartialSolution()).name("map").build();
   
    iteration.setNextPartialSolution(map2);

    FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");

    Plan plan = new Plan(finalResult, "Iteration with union test");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    FileDataSource input2 = new FileDataSource(new ContractITCaseInputFormat(), path2);
   
    MapOperator testMapper1 = MapOperator.builder(new TestMapper()).build();
    MapOperator testMapper2 = MapOperator.builder(new TestMapper()).build();

    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultDir);

    testMapper1.setInput(input1);
    testMapper2.setInput(input2);

    output.addInput(testMapper1);
    output.addInput(testMapper2);
   
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);

    PactCompiler pc = new PactCompiler(new DataStatistics());
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
    } else {
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
          config.getString("MatchTest#ShipStrategy", ""));
    }

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testMatcher);
    testMatcher.setFirstInput(input_left);
    testMatcher.setSecondInput(input_right);

    return new Plan(output);
  }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
        .build();
      ReduceOperator reduceNode = ReduceOperator.builder(new CountWords(), StringValue.class, 0)
        .input(mapNode)
        .name("Count Words")
        .build();
      FileDataSink out = new FileDataSink(new CsvOutputFormat(), OUT_FILE, reduceNode, "Word Counts");
      CsvOutputFormat.configureRecordFormat(out)
        .recordDelimiter('\n')
        .fieldDelimiter(' ')
        .lenient(true)
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);
     
      Ordering ordering = new Ordering(0, StringValue.class, Order.DESCENDING);
      out.setGlobalOrder(ordering, new SimpleDistribution(new StringValue[] {new StringValue("N")}));
     
      Plan p = new Plan(out, "WordCount Example");
      p.setDefaultParallelism(DEFAULT_PARALLELISM);
 
      OptimizedPlan plan;
View Full Code Here
289
290
291
292
293
294
295
296
297
298
299
   
    iteration.setNextWorkset(updateComponentId);
    iteration.setSolutionSetDelta(updateComponentId);

    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
   
    ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
    globalRed.addInput(redA);
    globalRed.addInput(redB);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
   
    // return the plan
    Plan plan = new Plan(sink, "Union Property Propagation");
   
    OptimizedPlan oPlan = compileNoStats(plan);
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(IntValue.class, 1);
View Full Code Here
486
487
488
489
490
491
492
493
494
495
496
        .input2(ma2)
        .name("Match 2")
        .build();
      mat2.setParameter(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_MERGE);
     
      FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat2);
     
     
      // return the PACT plan
      Plan plan = new Plan(sink, "Branching Union");
     
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSink

  • eu.stratosphere.addons.hbase.example.HBaseReadExample
  • eu.stratosphere.api.java.typeutils.RecordTypeInfo
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.