Examples of eu.stratosphere.api.java.record.operators.FileDataSink

Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.api.java.record.operators.FileDataSink
Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

    input2.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1));
    
    MapOperator testMapper = MapOperator.builder(new TestMapper()).build();
    testMapper.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1));


    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testMapper);


    testMapper.addInput(input1);
    testMapper.addInput(input2);


    return new Plan(output);

View Full Code Here

      .input(joinLiO, partJoin2, partJoin1)
      .name("AggLio")
      .build();


    // create DataSinkContract for writing the result
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, aggLiO, "Output");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .lenient(true)
      .field(LongValue.class, 0)

View Full Code Here

      .keyField(StringValue.class, 1, 1)
      .name("Find Shortest Paths")
      .build();
    findShortestPaths.setDegreeOfParallelism(numSubTasks);


    FileDataSink result = new FileDataSink(new PathOutFormat(),output, "New Paths");
    result.setDegreeOfParallelism(numSubTasks);


    result.setInput(findShortestPaths);
    findShortestPaths.setFirstInput(pathsInput);
    findShortestPaths.setSecondInput(concatPaths);
    concatPaths.setFirstInput(pathsInput);
    concatPaths.setSecondInput(pathsInput);

View Full Code Here




        MapOperator dummyMap = MapOperator.builder(new IdentityMapper()).input(dummyReduce).build();
        iteration.setNextPartialSolution(dummyMap);


        FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");


        Plan plan = new Plan(finalResult, "Iteration with chained map test");
        plan.setDefaultParallelism(numSubTasks);
        return plan;
    }

View Full Code Here

      .keyField(StringValue.class, 6)
      .keyField(StringValue.class, 7)
      .name("Reduce")
      .build();


    FileDataSink result = new FileDataSink(new TupleOutputFormat(), resultPath, "Output");


    result.setInput(reduce);
    
    reduce.setInput(joinNCOL);
    
    joinNCOL.setFirstInput(joinCOL);
    joinNCOL.setSecondInput(projectN);

View Full Code Here

    }
    
    iteration.setNextWorkset(updateComponentId);


    // sink is the iteration result
    FileDataSink result = new FileDataSink(new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class), output, iteration, "Result");


    // return the PACT plan
    Plan plan = new Plan(result, "Workset Connected Components");
    plan.setDefaultParallelism(numSubTasks);
    return plan;

View Full Code Here

    input.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));


    MapOperator testMapper = MapOperator.builder(new TestMapper()).build();
    testMapper.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1));


    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testMapper);
    testMapper.setInput(input);


    return new Plan(output);
  }

View Full Code Here

    MapOperator mapper = MapOperator.builder(new TokenizeLine()).input(source).name("Tokenize Lines").build();
    
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0).input(mapper)
        .name("Count Words").build();
    
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
    
    CsvOutputFormat.configureRecordFormat(out).recordDelimiter('\n')
        .fieldDelimiter(' ').field(StringValue.class, 0)
        .field(IntValue.class, 1);

View Full Code Here

    testReducer.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("ReduceTest#LocalStrategy", ""));
    testReducer.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
        config.getString("ReduceTest#ShipStrategy", ""));


    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testReducer);
    testReducer.setInput(input);


    Plan plan = new Plan(output);


    PactCompiler pc = new PactCompiler(new DataStatistics());

View Full Code Here


    /* Connect aggregate: */
    sumAmountAggregate.setInput(partListJoin);


    /* Connect sink: */
    FileDataSink result = new FileDataSink(new StringIntPairStringDataOutFormat(), this.outputPath, "Results sink");
    result.setInput(sumAmountAggregate);


    Plan p = new Plan(result, "TPC-H query 9");
    p.setDefaultParallelism(this.degreeOfParallelism);
    return p;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.addons.hbase.example.HBaseReadExample

eu.stratosphere.api.java.typeutils.RecordTypeInfo

eu.stratosphere.hadoopcompatibility.example.WordCount

eu.stratosphere.pact.compiler.AdditionalOperatorsTest

eu.stratosphere.pact.compiler.BranchingPlansCompilerTest

eu.stratosphere.pact.compiler.DOPChangeTest

eu.stratosphere.pact.compiler.GroupOrderTest

eu.stratosphere.pact.compiler.HardPlansCompilationTest

eu.stratosphere.pact.compiler.ReduceAllTest

eu.stratosphere.pact.compiler.UnionPropertyPropagationTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.