Examples of eu.stratosphere.api.java.record.operators.FileDataSink

Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.api.java.record.operators.FileDataSink
Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
  
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
      
      FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
      FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceA);
      
      List<FileDataSink> sinks = new ArrayList<FileDataSink>();
      sinks.add(sinkA);
      sinks.add(sinkB);

View Full Code Here

    final String out2Path = "file:///test/2";


    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
    
    FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
    FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB);
    
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sinkA);
    sinks.add(sinkB);

View Full Code Here

    String output    = (args.length > 1 ? args[1] : "");


    GenericDataSource<TableInputFormat> source = new GenericDataSource<TableInputFormat>(new MyTableInputFormat(), "HBase Input");
    source.setParameter(TableInputFormat.INPUT_TABLE, "twitter");
    source.setParameter(TableInputFormat.CONFIG_LOCATION, "/etc/hbase/conf/hbase-site.xml");
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, source, "HBase String dump");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(StringValue.class, 1)

View Full Code Here

    final String out4Path = "file:///test/4";


    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
    
    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA, "1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB, "2");
    FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, out3Path, sourceA, "3");
    FileDataSink sink4 = new FileDataSink(DummyOutputFormat.class, out4Path, sourceB, "4");
    
    
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink1);
    sinks.add(sink2);

View Full Code Here

    iteration.setMaximumNumberOfIterations(10);
    
    MapOperator mapper = MapOperator.builder(IdentityMap.class).name("Mapper").input(iteration.getPartialSolution()).build();
    iteration.setNextPartialSolution(mapper);
    
    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 1");
    
    MapOperator postMap = MapOperator.builder(IdentityMap.class).name("Post Iteration Mapper")
        .input(iteration).build();
    
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink 2");
    
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink1);
    sinks.add(sink2);

View Full Code Here

                     .input(source1)
                     .name("Post Iteration Map")
                     .setBroadcastVariable("BC", iteration)
                     .build();
    
    FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink");
    
    Plan plan = new Plan(sink);
    
    try {
      compileNoStats(plan);

View Full Code Here

  @Test
  public void testClosure() {
    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");


    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceB, "Sink 2");


    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(sourceA);
    iteration.setMaximumNumberOfIterations(10);


    CrossOperator stepFunction = CrossOperator.builder(DummyCrossStub.class).name("StepFunction").
        input1(iteration.getPartialSolution()).
        input2(sourceB).
        build();


    iteration.setNextPartialSolution(stepFunction);


    FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");


    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink1);
    sinks.add(sink2);
    sinks.add(sink3);

View Full Code Here

  public void testClosureDeltaIteration() {
    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
    FileDataSource sourceC = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 3");


    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceC, "Sink 2");


    DeltaIteration iteration = new DeltaIteration(0, "Loop");
    iteration.setInitialSolutionSet(sourceA);
    iteration.setInitialWorkset(sourceB);
    iteration.setMaximumNumberOfIterations(10);


    CrossOperator nextWorkset = CrossOperator.builder(DummyCrossStub.class).name("Next workset").
        input1(iteration.getWorkset()).
        input2(sourceC).
        build();


    JoinOperator solutionSetDelta = JoinOperator.builder(DummyMatchStub.class, LongValue.class,0,0).
        name("Next solution set.").
        input1(nextWorkset).
        input2(iteration.getSolutionSet()).
        build();


    iteration.setNextWorkset(nextWorkset);
    iteration.setSolutionSetDelta(solutionSetDelta);


    FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");


    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink1);
    sinks.add(sink2);
    sinks.add(sink3);

View Full Code Here

        build();


    iteration.setNextWorkset(nextWorkset);
    iteration.setSolutionSetDelta(solutionSetDelta);


    FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Iteration sink");
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink);


    Plan plan = new Plan(sinks);

View Full Code Here

        name("Next partial solution").
        build();


    iteration.setNextPartialSolution(nextPartialSolution);


    FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Iteration sink");
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(sink);


    Plan plan = new Plan(sinks);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSink

eu.stratosphere.addons.hbase.example.HBaseReadExample

eu.stratosphere.api.java.typeutils.RecordTypeInfo

eu.stratosphere.hadoopcompatibility.example.WordCount

eu.stratosphere.pact.compiler.AdditionalOperatorsTest

eu.stratosphere.pact.compiler.BranchingPlansCompilerTest

eu.stratosphere.pact.compiler.DOPChangeTest

eu.stratosphere.pact.compiler.GroupOrderTest

eu.stratosphere.pact.compiler.HardPlansCompilationTest

eu.stratosphere.pact.compiler.ReduceAllTest

eu.stratosphere.pact.compiler.UnionPropertyPropagationTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.