Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.MapOperator


      int numSubTasks, String verticesInput, String edgeInput, String output, int maxIterations)
  {
    // create DataSourceContract for the vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
View Full Code Here


    HadoopDataSource<LongWritable, Text> source = new HadoopDataSource<LongWritable, Text>(
        new TextInputFormat(), new JobConf(), "Input Lines");
    TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));


    MapOperator mapper = MapOperator.builder(new TokenizeLine())
        .input(source)
        .name("Tokenize Lines")
        .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
        .input(mapper)
View Full Code Here

    // Example with Wrapper Converter
    HadoopDataSource<LongWritable,Text> sourceHadoopType = new HadoopDataSource<LongWritable, Text>(
        new TextInputFormat(), new JobConf(), "Input Lines", new WritableWrapperConverter<LongWritable, Text>());
    TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
   
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
View Full Code Here

      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
 
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
 
      MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
 
      FileDataSink[] sinkA = new FileDataSink[SINKS];
      FileDataSink[] sinkB = new FileDataSink[SINKS];
      for (int sink = 0; sink < SINKS; sink++) {
        sinkA[sink] = new FileDataSink(DummyOutputFormat.class, out1Path, mapA, "Sink A:" + sink);
View Full Code Here

      final String out2Path = "file:///test/2";
      final String out3Path = "file:///test/3";
 
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);

      MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
      MapOperator mapB = MapOperator.builder(IdentityMap.class).input(mapA).name("Map B").build();
      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
     
      FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, mapB, "Sink A");
      FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, mapC, "Sink B");
      FileDataSink sinkC = new FileDataSink(DummyOutputFormat.class, out3Path, mapC, "Sink C");
     
View Full Code Here

      JoinOperator mat5 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(mat4)
        .build();
     
      MapOperator ma = MapOperator.builder(new IdentityMap()).input(sourceA).build();
     
      JoinOperator mat6 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(ma)
        .build();
View Full Code Here

     
      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .build();
      MapOperator ma = MapOperator.builder(new IdentityMap()).input(co).build();
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceB)
        .input2(sourceC)
        .build();
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
View Full Code Here

      // construct the plan
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), "file:///test/file1", "Source A");
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
     
      MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
     
      ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(map1)
        .name("Reduce 1")
        .build();
     
      @SuppressWarnings("unchecked")
      JoinOperator match1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceB, sourceB, sourceC)
        .input2(sourceC)
        .name("Match 1")
        .build();
      ;
      CoGroupOperator cogroup1 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .name("CoGroup 1")
        .build();
     
      CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub())
        .input1(reduce1)
        .input2(cogroup1)
        .name("Cross 1")
        .build();
     
     
      CoGroupOperator cogroup2 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(cross1)
        .input2(cross1)
        .name("CoGroup 2")
        .build();
     
      CoGroupOperator cogroup3 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(map1)
        .input2(match1)
        .name("CoGroup 3")
        .build();
     
     
      MapOperator map2 = MapOperator.builder(new IdentityMap()).input(cogroup3).name("Map 2").build();
     
      CoGroupOperator cogroup4 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(map2)
        .input2(match1)
        .name("CoGroup 4")
View Full Code Here

        .input1(source1)
        .input2(source2)
        .name("Match 1")
        .build();
     
      MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
     
      ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma1)
        .name("Reduce 1")
        .build();
     
      ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(mat1)
        .name("Reduce 2")
        .build();
     
      MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
     
      MapOperator ma3 = MapOperator.builder(new IdentityMap()).input(ma2).name("Map 3").build();
     
      @SuppressWarnings("unchecked")
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(r1, r2, ma2, ma3)
        .input2(ma2)
View Full Code Here

   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(sourceA);
    iteration.setMaximumNumberOfIterations(10);
   
    MapOperator mapper = MapOperator.builder(IdentityMap.class).name("Mapper").input(iteration.getPartialSolution()).build();
    iteration.setNextPartialSolution(mapper);
   
    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 1");
   
    MapOperator postMap = MapOperator.builder(IdentityMap.class).name("Post Iteration Mapper")
        .input(iteration).build();
   
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink 2");
   
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
View Full Code Here

TOP

Related Classes of eu.stratosphere.api.java.record.operators.MapOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.