Examples of CoGroupOperator

eu.stratosphere.api.java.record.operators.CoGroupOperator
CoGroupOperator that applies a {@link CoGroupFunction} to groups of records sharingthe same key (one group per input). @see CoGroupFunction
org.apache.flink.api.java.record.operators.CoGroupOperator
s.apache.org/jira/browse/FLINK-1106">FLINK-1106 for more details. @see CoGroupFunction

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

    FileDataSource input_right =  new FileDataSource(new CoGroupTestInFormat(), rightInPath);
    DelimitedInputFormat.configureDelimitedFormat(input_right)
      .recordDelimiter('\n');
    input_right.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1));


    CoGroupOperator testCoGrouper = CoGroupOperator.builder(new TestCoGrouper(), StringValue.class, 0, 0)
      .build();
    testCoGrouper.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1));
    testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("CoGroupTest#LocalStrategy", ""));
    testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
        config.getString("CoGroupTest#ShipStrategy", ""));


    FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testCoGrouper);
    testCoGrouper.setFirstInput(input_left);
    testCoGrouper.setSecondInput(input_right);


    return new Plan(output);
  }

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

    GenericDataSource<RandomInputFormat> bookSource = new GenericDataSource<RandomInputFormat>(
      new RandomInputFormat(true));
    GenericDataSource<RandomInputFormat> authorSource = new GenericDataSource<RandomInputFormat>(
      new RandomInputFormat(false));


    CoGroupOperator coGroupOperator = CoGroupOperator.builder(MyCoGrouper.class, LongValue.class, 0, 0)
      .input1(bookSource).input2(authorSource).name("CoGrouper Test").build();


    GenericDataSink sink = new GenericDataSink(PrintingOutputFormat.class, coGroupOperator);


    Plan plan = new Plan(sink, "CoGroper Test Plan");

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

      JoinOperator mat10 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat9)
        .build();
      
      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0, 0)
        .input1(mat5)
        .input2(mat10)
        .build();
  
      FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, co);

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

  
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), IN_FILE);
      
      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .build();
      MapOperator ma = MapOperator.builder(new IdentityMap()).input(co).build();
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

        .input1(sourceB, sourceB, sourceC)
        .input2(sourceC)
        .name("Match 1")
        .build();
      ;
      CoGroupOperator cogroup1 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .name("CoGroup 1")
        .build();
      
      CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub())
        .input1(reduce1)
        .input2(cogroup1)
        .name("Cross 1")
        .build();
      
      
      CoGroupOperator cogroup2 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(cross1)
        .input2(cross1)
        .name("CoGroup 2")
        .build();
      
      CoGroupOperator cogroup3 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(map1)
        .input2(match1)
        .name("CoGroup 3")
        .build();
      
      
      MapOperator map2 = MapOperator.builder(new IdentityMap()).input(cogroup3).name("Map 2").build();
      
      CoGroupOperator cogroup4 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(map2)
        .input2(match1)
        .name("CoGroup 4")
        .build();
      
      CoGroupOperator cogroup5 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(cogroup2)
        .input2(cogroup1)
        .name("CoGroup 5")
        .build();
      
      CoGroupOperator cogroup6 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(reduce1)
        .input2(cogroup4)
        .name("CoGroup 6")
        .build();
      
      CoGroupOperator cogroup7 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(cogroup5)
        .input2(cogroup6)
        .name("CoGroup 7")
        .build();

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

        .input1(iteration.getWorkset())
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();


    CoGroupOperator minAndUpdate = CoGroupOperator.builder(new MinIdAndUpdate(), LongValue.class, 0, 0)
        .input1(joinWithNeighbors)
        .input2(iteration.getSolutionSet())
        .name("Min Id and Update")
        .build();

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

        .input1(iteration.getPartialSolution())
        .input2(adjacencyListInput)
        .name("Join with Edges")
        .build();
    
    CoGroupOperator rankAggregation = CoGroupOperator.builder(new DotProductCoGroup(), LongValue.class, 0, 0)
        .input1(iteration.getPartialSolution())
        .input2(join)
        .name("Rank Aggregation")
        .build();
    rankAggregation.getParameters().setLong(DotProductCoGroup.NUM_VERTICES_PARAMETER, numVertices);
    rankAggregation.getParameters().setLong(DotProductCoGroup.NUM_DANGLING_VERTICES_PARAMETER, numDanglingVertices);
    
    iteration.setNextPartialSolution(rankAggregation);
    iteration.setMaximumNumberOfIterations(numIterations);
    iteration.getAggregators().registerAggregationConvergenceCriterion(DotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class, DiffL1NormConvergenceCriterion.class);

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

      .name("Join Docs Ranks")
      .build();


    // Create CoGroupOperator to realize a anti join between the joined
    // documents and ranks relation and the filtered visits relation
    CoGroupOperator antiJoinVisits = CoGroupOperator.builder(new AntiJoinVisits(), StringValue.class, 0, 0)
      .input1(joinDocsRanks)
      .input2(filterVisits)
      .name("Antijoin DocsVisits")
      .build();

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

  public void testCoGroupWithGroupOrder() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source2");
    
    CoGroupOperator coGroup = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 3, 6)
        .keyField(LongValue.class, 0, 0)
        .name("CoGroup").input1(source1).input2(source2).build();
    
    Ordering groupOrder1 = new Ordering(5, StringValue.class, Order.DESCENDING);
    Ordering groupOrder2 = new Ordering(1, StringValue.class, Order.DESCENDING);
    groupOrder2.appendOrdering(4, DoubleValue.class, Order.ASCENDING);
    coGroup.setGroupOrderForInputOne(groupOrder1);
    coGroup.setGroupOrderForInputTwo(groupOrder2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, coGroup, "Sink");
    
    Plan plan = new Plan(sink, "Reduce Group Order Test");
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);

View Full Code Here

Examples of eu.stratosphere.api.java.record.operators.CoGroupOperator

        .input1(iteration.getWorkset())
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();


    CoGroupOperator minAndUpdate = CoGroupOperator.builder(new MinIdAndUpdate(), LongValue.class, 0, 0)
        .input1(joinWithNeighbors)
        .input2(iteration.getSolutionSet())
        .name("Min Id and Update")
        .build();

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.