Examples of eu.stratosphere.api.java.record.operators.FileDataSink

Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.api.java.record.operators.FileDataSink
Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

    
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
    
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
    
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

    
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
    
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
    
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here


    // init failing map task
    MapOperator testMapper = MapOperator.builder(FailingMapper.class).build();


    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);


    // compose failing program
    output.setInput(testMapper);
    testMapper.setInput(input);


    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);

View Full Code Here

    
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar);
    reduce2.setInput(map2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar);
    sink.setInput(reduce2);
    
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
    
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

      CsvInputFormat.configureRecordFormat(source)
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .field(IntValue.class, 0);
      
      FileDataSink sink =
        new FileDataSink(CsvOutputFormat.class, output);
      sink.setDegreeOfParallelism(numSubtasks);
      CsvOutputFormat.configureRecordFormat(sink)
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .lenient(true)
        .field(IntValue.class, 0);
      
      sink.setGlobalOrder(new Ordering(0, IntValue.class, Order.ASCENDING), new UniformIntegerDistribution(Integer.MIN_VALUE, Integer.MAX_VALUE));
      sink.setInput(source);
      
      return new Plan(sink);
    }

View Full Code Here


    // init (working) map task
    MapOperator testMapper = MapOperator.builder(TestMapper.class).build();


    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);


    // compose working program
    output.setInput(testMapper);
    testMapper.setInput(input);


    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);

View Full Code Here

    JoinOperator mat = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
      .input1(redA)
      .input2(redB)
      .build();
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat);
    
    sourceA.setDegreeOfParallelism(5);
    sourceB.setDegreeOfParallelism(7);
    redA.setDegreeOfParallelism(5);
    redB.setDegreeOfParallelism(7);
    
    mat.setDegreeOfParallelism(5);
    
    sink.setDegreeOfParallelism(5);
    
    
    // return the PACT plan
    Plan plan = new Plan(sink, "Partition on DoP Change");

View Full Code Here

        .name("Compute sum (Reduce)")
        .build();
    
    iteration.setNextPartialSolution(sumReduce);


    @SuppressWarnings("unchecked")
    FileDataSink finalResult = new FileDataSink(new CsvOutputFormat("\n",  " ", StringValue.class), output, iteration, "Output");


    Plan plan = new Plan(finalResult, "Iteration with AllReducer (keyless Reducer)");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }

View Full Code Here

    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();


    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, mapper, "Selection");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(IntValue.class, 1);

View Full Code Here


    iteration.setNextWorkset(updateRanks);
    iteration.setSolutionSetDelta(oldRankComparison);
    
    // create DataSinkContract for writing the final ranks
    FileDataSink result = new FileDataSink(CsvOutputFormat.class, output, iteration, "Final Ranks");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(DoubleValue.class, 1);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.addons.hbase.example.HBaseReadExample

eu.stratosphere.api.java.typeutils.RecordTypeInfo

eu.stratosphere.hadoopcompatibility.example.WordCount

eu.stratosphere.pact.compiler.AdditionalOperatorsTest

eu.stratosphere.pact.compiler.BranchingPlansCompilerTest

eu.stratosphere.pact.compiler.DOPChangeTest

eu.stratosphere.pact.compiler.GroupOrderTest

eu.stratosphere.pact.compiler.HardPlansCompilationTest

eu.stratosphere.pact.compiler.ReduceAllTest

eu.stratosphere.pact.compiler.UnionPropertyPropagationTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.