Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.MapOperator

83
84
85
86
87
88
89
90
91
92
93
94
95
96
    // Jn2 matches polynomial and arguments by id, computes p = min(P(x),P(y)) and emits (id, p) tuples
    JoinOperator jn2 = JoinOperator.builder(Jn2.class, StringValue.class, 0, 0).input1(jn1).input2(sc1).build();

    // Mp1 selects (id, x, y) triples where x = y and broadcasts z (=x=y) to Mp2
    MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build();

    // Mp2 filters out all p values which can be divided by z
    MapOperator mp2 = MapOperator.builder(Mp2.class).setBroadcastVariable("z", mp1).input(jn2).build();

    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);
    output.setInput(mp2);
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    int numSubTasks   = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
    String dataInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");

    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
 
//  @Test
  public void testMapCancelling() throws Exception {
    GenericDataSource<InfiniteIntegerInputFormat> source = new GenericDataSource<InfiniteIntegerInputFormat>(
                                    new InfiniteIntegerInputFormat(), "Source");
    MapOperator mapper = MapOperator.builder(IdentityMapper.class)
      .input(source)
      .name("Identity Mapper")
      .build();
    GenericDataSink sink = new GenericDataSink(new DiscardingOutputFormat(), mapper, "Sink");
   
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
 
//  @Test
  public void testSlowMapCancelling() throws Exception {
    GenericDataSource<InfiniteIntegerInputFormat> source = new GenericDataSource<InfiniteIntegerInputFormat>(
                                    new InfiniteIntegerInputFormat(), "Source");
    MapOperator mapper = MapOperator.builder(DelayingIdentityMapper.class)
      .input(source)
      .name("Delay Mapper")
      .build();
    GenericDataSink sink = new GenericDataSink(new DiscardingOutputFormat(), mapper, "Sink");
   
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
 
//  @Test
  public void testMapWithLongCancellingResponse() throws Exception {
    GenericDataSource<InfiniteIntegerInputFormat> source = new GenericDataSource<InfiniteIntegerInputFormat>(
                                    new InfiniteIntegerInputFormat(), "Source");
    MapOperator mapper = MapOperator.builder(LongCancelTimeIdentityMapper.class)
      .input(source)
      .name("Long Cancelling Time Mapper")
      .build();
    GenericDataSink sink = new GenericDataSink(new DiscardingOutputFormat(), mapper, "Sink");
   
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
 
//  @Test
  public void testMapPriorToFirstRecordReading() throws Exception {
    GenericDataSource<InfiniteIntegerInputFormat> source = new GenericDataSource<InfiniteIntegerInputFormat>(
                                    new InfiniteIntegerInputFormat(), "Source");
    MapOperator mapper = MapOperator.builder(StuckInOpenIdentityMapper.class)
      .input(source)
      .name("Stuck-In-Open Mapper")
      .build();
    GenericDataSink sink = new GenericDataSink(new DiscardingOutputFormat(), mapper, "Sink");
   
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(initialInput);
    iteration.setMaximumNumberOfIterations(2);

    @SuppressWarnings("unchecked")
    MapOperator map2 = MapOperator.builder(new IdentityMapper()).input(iteration.getPartialSolution(), iteration.getPartialSolution()).name("map").build();
   
    iteration.setNextPartialSolution(map2);

    FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    String path2 = config.getBoolean("input2PathHasData", false) ? textInput : emptyInput;
   
    FileDataSource input1 = new FileDataSource(new ContractITCaseInputFormat(), path1);
    FileDataSource input2 = new FileDataSource(new ContractITCaseInputFormat(), path2);
   
    MapOperator testMapper1 = MapOperator.builder(new TestMapper()).build();
    MapOperator testMapper2 = MapOperator.builder(new TestMapper()).build();

    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultDir);

    testMapper1.setInput(input1);
    testMapper2.setInput(input2);

    output.addInput(testMapper1);
    output.addInput(testMapper2);
   
    Plan plan = new Plan(output);
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
  }
 
  private void checkWordCountWithSortedSink(boolean estimates) {
    try {
      FileDataSource sourceNode = new FileDataSource(new TextInputFormat(), IN_FILE, "Input Lines");
      MapOperator mapNode = MapOperator.builder(new TokenizeLine())
        .input(sourceNode)
        .name("Tokenize Lines")
        .build();
      ReduceOperator reduceNode = ReduceOperator.builder(new CountWords(), StringValue.class, 0)
        .input(mapNode)
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    try {
      // set statistics
      OperatorResolver cr = getContractResolver(p);
      FileDataSource ordersSource = cr.getNode(ORDERS);
      FileDataSource lineItemSource = cr.getNode(LINEITEM);
      MapOperator mapper = cr.getNode(MAPPER_NAME);
      JoinOperator joiner = cr.getNode(JOIN_NAME);
      setSourceStatistics(ordersSource, orderSize, 100f);
      setSourceStatistics(lineItemSource, lineitemSize, 140f);
      mapper.getCompilerHints().setAvgOutputRecordSize(16f);
      mapper.getCompilerHints().setFilterFactor(orderSelectivity);
      joiner.getCompilerHints().setFilterFactor(joinSelectivity);
     
      // compile
      final OptimizedPlan plan = compileWithStats(p);
      final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.MapOperator

  • eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.hadoopcompatibility.example.WordCountWithHadoopOutputFormat
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
  • eu.stratosphere.test.accumulators.AccumulatorITCase
  • eu.stratosphere.test.broadcastvars.BroadcastBranchingITCase
  • eu.stratosphere.test.cancelling.MapCancelingITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.