Examples of ReduceOperator

eu.stratosphere.api.java.record.operators.ReduceOperator
ReduceOperator evaluating a {@link ReduceFunction} over each group of records that share the same key. @see ReduceFunction
org.apache.flink.api.java.record.operators.ReduceOperator
ReduceOperator evaluating a {@link ReduceFunction} over each group of records that share the same key. @see ReduceFunction

Examples of eu.stratosphere.api.java.record.operators.ReduceOperator

    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
    
    @SuppressWarnings("unchecked")

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

    // construct the plan


    FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
    FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
    
    ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceA)
      .build();
    ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceB)
      .build();
    
    ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
    globalRed.addInput(redA);
    globalRed.addInput(redB);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
    
    // return the plan
    Plan plan = new Plan(sink, "Union Property Propagation");

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

    // =========================== Vertex Degree ============================
    
    MapOperator projectEdge = MapOperator.builder(new ProjectEdge())
        .input(edges).name("Project Edge").build();
    
    ReduceOperator edgeCounter = ReduceOperator.builder(new CountEdges(), IntValue.class, 0)
        .input(projectEdge).name("Count Edges for Vertex").build();
    
    ReduceOperator countJoiner = ReduceOperator.builder(new JoinCountsAndUniquify(), IntValue.class, 0)
        .keyField(IntValue.class, 1)
        .input(edgeCounter).name("Join Counts").build();
    
    
    // =========================== Triangle Enumeration ============================
    
    MapOperator toLowerDegreeEdge = MapOperator.builder(new ProjectToLowerDegreeVertex())
        .input(countJoiner).name("Select lower-degree Edge").build();
    
    MapOperator projectOutCounts = MapOperator.builder(new ProjectOutCounts())
        .input(countJoiner).name("Project out Counts").build();


    ReduceOperator buildTriads = ReduceOperator.builder(new BuildTriads(), IntValue.class, 0)
        .input(toLowerDegreeEdge).name("Build Triads").build();


    JoinOperator closeTriads = JoinOperator.builder(new CloseTriads(), IntValue.class, 1, 0)
        .keyField(IntValue.class, 2, 1)
        .input1(buildTriads).input2(projectOutCounts)

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

      JoinOperator.builder(PartListJoin.class, IntValue.class , 0, 0)
      .name("partlistJoin")
      .build();


    /* Aggregate sum(amount) by (nation,year): */
    ReduceOperator sumAmountAggregate =
      ReduceOperator.builder(AmountAggregate.class, StringIntPair.class, 0)
      .name("groupyBy")
      .build();


    /* Connect input filters: */
    filterPart.setInput(partInput);
    mapPartsupp.setInput(partSuppInput);
    mapOrder.setInput(ordersInput);
    mapLineItem.setInput(lineItemInput);
    mapSupplier.setInput(supplierInput);


    /* Connect equijoins: */
    partsJoin.setFirstInput(filterPart);
    partsJoin.setSecondInput(mapPartsupp);
    orderedPartsJoin.setFirstInput(mapOrder);
    orderedPartsJoin.setSecondInput(mapLineItem);
    suppliersJoin.setFirstInput(mapSupplier);
    suppliersJoin.setSecondInput(nationInput);
    filteredPartsJoin.setFirstInput(partsJoin);
    filteredPartsJoin.setSecondInput(orderedPartsJoin);
    partListJoin.setFirstInput(filteredPartsJoin);
    partListJoin.setSecondInput(suppliersJoin);


    /* Connect aggregate: */
    sumAmountAggregate.setInput(partListJoin);


    /* Connect sink: */
    FileDataSink result = new FileDataSink(new StringIntPairStringDataOutFormat(), this.outputPath, "Results sink");
    result.setInput(sumAmountAggregate);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

    String edgeInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");


    FileDataSource edges = new FileDataSource(new EdgeInFormat(), edgeInput, "BTC Edges");
    
    ReduceOperator buildTriads = ReduceOperator.builder(new BuildTriads(), StringValue.class, 0)
      .name("Build Triads")
      .build();


    JoinOperator closeTriads = JoinOperator.builder(new CloseTriads(), StringValue.class, 1, 0)
      .keyField(StringValue.class, 2, 1)
      .name("Close Triads")
      .build();
    closeTriads.setParameter("INPUT_LEFT_SHIP_STRATEGY", "SHIP_REPARTITION_HASH");
    closeTriads.setParameter("INPUT_RIGHT_SHIP_STRATEGY", "SHIP_REPARTITION_HASH");
    closeTriads.setParameter("LOCAL_STRATEGY", "LOCAL_STRATEGY_HASH_BUILD_SECOND");


    FileDataSink triangles = new FileDataSink(new CsvOutputFormat(), output, "Output");
    CsvOutputFormat.configureRecordFormat(triangles)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(StringValue.class, 1)
      .field(StringValue.class, 2);


    triangles.setInput(closeTriads);
    closeTriads.setSecondInput(edges);
    closeTriads.setFirstInput(buildTriads);
    buildTriads.setInput(edges);


    Plan plan = new Plan(triangles, "Enumerate Triangles");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

    // create DataSourceContract for Orders input
    @SuppressWarnings("unchecked")
    CsvInputFormat format1 = new CsvInputFormat('|', IntValue.class, IntValue.class);
    FileDataSource input1 = new FileDataSource(format1, input1Path, "Input 1");
    
    ReduceOperator aggInput1 = ReduceOperator.builder(DummyReduce.class, IntValue.class, 0)
      .input(input1)
      .name("AggOrders")
      .build();


    
    // create DataSourceContract for Orders input
    @SuppressWarnings("unchecked")
    CsvInputFormat format2 = new CsvInputFormat('|', IntValue.class, IntValue.class);
    FileDataSource input2 = new FileDataSource(format2, input2Path, "Input 2");
    input2.setDegreeOfParallelism(numSubtasksInput2);


    ReduceOperator aggInput2 = ReduceOperator.builder(DummyReduce.class, IntValue.class, 0)
      .input(input2)
      .name("AggLines")
      .build();
    aggInput2.setDegreeOfParallelism(numSubtasksInput2);
    
    // create JoinOperator for joining Orders and LineItems
    JoinOperator joinLiO = JoinOperator.builder(JoinInputs.class, IntValue.class, 0, 0)
      .input1(aggInput1)
      .input2(aggInput2)

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

        .input1(iteration.getWorkset())
        .input2(dependencySet)
        .name("calculate dependencies")
        .build();
    
    ReduceOperator updateRanks = ReduceOperator.builder(UpdateRankReduceDelta.class, LongValue.class, 0)
        .input(dependenciesMatch)
        .name("update ranks")
        .build();
    
    JoinOperator oldRankComparison = JoinOperator.builder(RankComparisonMatch.class, LongValue.class, 0, 0)

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

    @SuppressWarnings("unchecked")
    CsvInputFormat format = new CsvInputFormat(' ', IntValue.class, IntValue.class);
    FileDataSource input = new FileDataSource(format, dataInput, "Input");
    
    // create the reduce contract and sets the key to the first field
    ReduceOperator sorter = ReduceOperator.builder(new IdentityReducer(), IntValue.class, 0)
      .input(input)
      .name("Reducer")
      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));


    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator

      .field(IntValue.class, 1)
      .field(DoubleValue.class, 2);
    
    // create ReduceOperator for aggregating the result
    // the reducer has a composite key, consisting of the fields 0 and 1
    @SuppressWarnings("unchecked")
    ReduceOperator aggLiO = ReduceOperator.builder(new AggLiO())
      .keyField(LongValue.class, 0)
      .keyField(StringValue.class, 1)
      .input(joinLiO, partJoin2, partJoin1)
      .name("AggLio")

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.ReduceOperator


    JoinOperator joinNCOL = JoinOperator.builder(JoinNCOL.class, IntValue.class, 4, 0)
      .name("JoinNCOL")
      .build();


    ReduceOperator reduce = ReduceOperator.builder(Sum.class)
      .keyField(IntValue.class, 0) 
      .keyField(StringValue.class, 1)
      .keyField(StringValue.class, 3)
      .keyField(StringValue.class, 4)
      .keyField(StringValue.class, 5)
      .keyField(StringValue.class, 6)
      .keyField(StringValue.class, 7)
      .name("Reduce")
      .build();


    FileDataSink result = new FileDataSink(new TupleOutputFormat(), resultPath, "Output");


    result.setInput(reduce);
    
    reduce.setInput(joinNCOL);
    
    joinNCOL.setFirstInput(joinCOL);
    joinNCOL.setSecondInput(projectN);
    
    joinCOL.setFirstInput(projectC);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.