Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.MapOperator


                .input(iteration.getPartialSolution())
                .name("Reduce something")
                .build();


        MapOperator dummyMap = MapOperator.builder(new IdentityMapper()).input(dummyReduce).build();
        iteration.setNextPartialSolution(dummyMap);

        FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");

        Plan plan = new Plan(finalResult, "Iteration with chained map test");
View Full Code Here


    FileDataSource customers = new FileDataSource(new IntTupleDataInFormat(), customersPath, "Customers");

    FileDataSource nations = new FileDataSource(new IntTupleDataInFormat(), nationsPath, "Nations");


    MapOperator mapO = MapOperator.builder(FilterO.class)
      .name("FilterO")
      .build();

    MapOperator mapLi = MapOperator.builder(FilterLI.class)
      .name("FilterLi")
      .build();

    MapOperator projectC = MapOperator.builder(ProjectC.class)
      .name("ProjectC")
      .build();

    MapOperator projectN = MapOperator.builder(ProjectN.class)
      .name("ProjectN")
      .build();

    JoinOperator joinOL = JoinOperator.builder(JoinOL.class, IntValue.class, 0, 0)
      .name("JoinOL")
      .build();

    JoinOperator joinCOL = JoinOperator.builder(JoinCOL.class, IntValue.class, 0, 0)
      .name("JoinCOL")
      .build();

    JoinOperator joinNCOL = JoinOperator.builder(JoinNCOL.class, IntValue.class, 4, 0)
      .name("JoinNCOL")
      .build();

    ReduceOperator reduce = ReduceOperator.builder(Sum.class)
      .keyField(IntValue.class, 0)
      .keyField(StringValue.class, 1)
      .keyField(StringValue.class, 3)
      .keyField(StringValue.class, 4)
      .keyField(StringValue.class, 5)
      .keyField(StringValue.class, 6)
      .keyField(StringValue.class, 7)
      .name("Reduce")
      .build();

    FileDataSink result = new FileDataSink(new TupleOutputFormat(), resultPath, "Output");

    result.setInput(reduce);
   
    reduce.setInput(joinNCOL);
   
    joinNCOL.setFirstInput(joinCOL);
    joinNCOL.setSecondInput(projectN);
   
    joinCOL.setFirstInput(projectC);
    joinCOL.setSecondInput(joinOL);
   
    joinOL.setFirstInput(mapO);
    joinOL.setSecondInput(mapLi);
   
    projectC.setInput(customers);
    projectN.setInput(nations);
    mapLi.setInput(lineitems);
    mapO.setInput(orders);

    // return the PACT plan
    Plan p = new Plan(result, "TPCH Q10");
View Full Code Here

  public static Plan getPlan(int numSubTasks, String verticesInput, String edgeInput, String output, int maxIterations, boolean extraMap) {

    // data source for initial vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    // the loop takes the vertices as the solution set and changed vertices as the workset
    // initially, all vertices are changed
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
   
    // data source for the edges
    FileDataSource edges = new FileDataSource(new CsvInputFormat(' ', LongValue.class, LongValue.class), edgeInput, "Edges");

    // join workset (changed vertices) with the edges to propagate changes to neighbors
    JoinOperator joinWithNeighbors = JoinOperator.builder(new NeighborWithComponentIDJoin(), LongValue.class, 0, 0)
        .input1(iteration.getWorkset())
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();

    // find for each neighbor the smallest of all candidates
    ReduceOperator minCandidateId = ReduceOperator.builder(new MinimumComponentIDReduce(), LongValue.class, 0)
        .input(joinWithNeighbors)
        .name("Find Minimum Candidate Id")
        .build();
   
    // join candidates with the solution set and update if the candidate component-id is smaller
    JoinOperator updateComponentId = JoinOperator.builder(new UpdateComponentIdMatchNonPreserving(), LongValue.class, 0, 0)
        .input1(minCandidateId)
        .input2(iteration.getSolutionSet())
        .name("Update Component Id")
        .build();
   
    if (extraMap) {
      MapOperator mapper = MapOperator.builder(IdentityMap.class).input(updateComponentId).name("idmap").build();
      iteration.setSolutionSetDelta(mapper);
    } else {
      iteration.setSolutionSetDelta(updateComponentId);
    }
   
View Full Code Here

        new ContractITCaseInputFormat(), inPath);
    DelimitedInputFormat.configureDelimitedFormat(input)
      .recordDelimiter('\n');
    input.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));

    MapOperator testMapper = MapOperator.builder(new TestMapper()).build();
    testMapper.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testMapper);
    testMapper.setInput(input);

    return new Plan(output);
  }
View Full Code Here

    String dataInput = (args.length > 1 ? args[1] : "");
    String output = (args.length > 2 ? args[2] : "");

    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");

    MapOperator mapper = MapOperator.builder(new TokenizeLine()).input(source).name("Tokenize Lines").build();
   
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0).input(mapper)
        .name("Count Words").build();
   
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
View Full Code Here

      new IntTupleDataInFormat(), this.nationInputPath, "\"nation\" source");
    //nationInput.setOutputContract(UniqueKey.class);
//    nationInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* Filter on part's name, project values to NULL: */
    MapOperator filterPart = MapOperator.builder(PartFilter.class)
      .name("filterParts")
      .build();

    /* Map to change the key element of partsupp, project value to (supplycost, suppkey): */
    MapOperator mapPartsupp = MapOperator.builder(PartsuppMap.class)
      .name("mapPartsupp")
      .build();

    /* Map to extract the year from order: */
    MapOperator mapOrder = MapOperator.builder(OrderMap.class)
      .name("mapOrder")
      .build();

    /* Project value to (partkey, suppkey, quantity, price = extendedprice*(1-discount)): */
    MapOperator mapLineItem = MapOperator.builder(LineItemMap.class)
      .name("proj.Partsupp")
      .build();

    /* - change the key of supplier to nationkey, project value to suppkey */
    MapOperator mapSupplier = MapOperator.builder(SupplierMap.class)
      .name("proj.Partsupp")
      .build();

    /* Equijoin on partkey of part and partsupp: */
    JoinOperator partsJoin = JoinOperator.builder(PartJoin.class, IntValue.class, 0, 0)
      .name("partsJoin")
      .build();

    /* Equijoin on orderkey of orders and lineitem: */
    JoinOperator orderedPartsJoin =
      JoinOperator.builder(OrderedPartsJoin.class, IntValue.class, 0, 0)
      .name("orderedPartsJoin")
      .build();

    /* Equijoin on nationkey of supplier and nation: */
    JoinOperator suppliersJoin =
      JoinOperator.builder(SuppliersJoin.class, IntValue.class, 0, 0)
      .name("suppliersJoin")
      .build();

    /* Equijoin on (partkey,suppkey) of parts and orderedParts: */
    JoinOperator filteredPartsJoin =
      JoinOperator.builder(FilteredPartsJoin.class, IntPair.class, 0, 0)
      .name("filteredPartsJoin")
      .build();

    /* Equijoin on suppkey of filteredParts and suppliers: */
    JoinOperator partListJoin =
      JoinOperator.builder(PartListJoin.class, IntValue.class , 0, 0)
      .name("partlistJoin")
      .build();

    /* Aggregate sum(amount) by (nation,year): */
    ReduceOperator sumAmountAggregate =
      ReduceOperator.builder(AmountAggregate.class, StringIntPair.class, 0)
      .name("groupyBy")
      .build();

    /* Connect input filters: */
    filterPart.setInput(partInput);
    mapPartsupp.setInput(partSuppInput);
    mapOrder.setInput(ordersInput);
    mapLineItem.setInput(lineItemInput);
    mapSupplier.setInput(supplierInput);

    /* Connect equijoins: */
    partsJoin.setFirstInput(filterPart);
    partsJoin.setSecondInput(mapPartsupp);
    orderedPartsJoin.setFirstInput(mapOrder);
 
View Full Code Here

        .input(joinWithSolutionSet)
        .name(NEXT_WORKSET_REDUCER_NAME)
        .build();
   
    if (mapBeforeSolutionDelta) {
      MapOperator mapper = MapOperator.builder(new IdentityMap())
        .input(joinWithSolutionSet)
        .name(SOLUTION_DELTA_MAPPER_NAME)
        .build();
      iteration.setSolutionSetDelta(mapper);
    } else {
View Full Code Here

 
  public static void main(String[] args) throws Exception {
   
    GenericDataSource<UserGeneratingInputFormat> source = new GenericDataSource<UserGeneratingInputFormat>(UserGeneratingInputFormat.class);
   
    MapOperator mapper = MapOperator.builder(new NumberExtractingMapper())
        .input(source).name("le mapper").build();
   
    ReduceOperator reducer = ReduceOperator.builder(new ConcatenatingReducer(), IntValue.class, 1)
        .input(mapper).name("le reducer").build();
   
View Full Code Here

TOP

Related Classes of eu.stratosphere.api.java.record.operators.MapOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.