Examples of org.apache.flink.api.java.record.operators.FileDataSink

Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.FileDataSink

org.apache.flink.api.java.record.operators.FileDataSink
s.apache.org/jira/browse/FLINK-1106">FLINK-1106 for more details. Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

    
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
    
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
    
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

    
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar);
    reduce2.setInput(map2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar);
    sink.setInput(reduce2);
    
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
    
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

    JoinOperator mat = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
      .input1(redA)
      .input2(redB)
      .build();
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat);
    
    sourceA.setDegreeOfParallelism(5);
    sourceB.setDegreeOfParallelism(7);
    redA.setDegreeOfParallelism(5);
    redB.setDegreeOfParallelism(7);
    
    mat.setDegreeOfParallelism(5);
    
    sink.setDegreeOfParallelism(5);
    
    
    // return the PACT plan
    Plan plan = new Plan(sink, "Partition on DoP Change");

View Full Code Here

    
    ReduceOperator reduce = ReduceOperator.builder(new IdentityReduce()).keyField(IntValue.class, 2).name("Reduce").input(source).build();
    Ordering groupOrder = new Ordering(5, StringValue.class, Order.DESCENDING);
    reduce.setGroupOrder(groupOrder);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, reduce, "Sink");
    
    
    Plan plan = new Plan(sink, "Test Temp Task");
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);

View Full Code Here

    Ordering groupOrder2 = new Ordering(1, StringValue.class, Order.DESCENDING);
    groupOrder2.appendOrdering(4, DoubleValue.class, Order.ASCENDING);
    coGroup.setGroupOrderForInputOne(groupOrder1);
    coGroup.setGroupOrderForInputTwo(groupOrder2);
    
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, coGroup, "Sink");
    
    Plan plan = new Plan(sink, "Reduce Group Order Test");
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
    
    OptimizedPlan oPlan;

View Full Code Here

      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
  
      FileDataSink[] sinkA = new FileDataSink[SINKS];
      FileDataSink[] sinkB = new FileDataSink[SINKS];
      for (int sink = 0; sink < SINKS; sink++) {
        sinkA[sink] = new FileDataSink(DummyOutputFormat.class, out1Path, mapA, "Sink A:" + sink);
        sinks.add(sinkA[sink]);
  
        sinkB[sink] = new FileDataSink(DummyOutputFormat.class, out2Path, mapC, "Sink B:" + sink);
        sinks.add(sinkB[sink]);
      }
  
      // return the PACT plan
      Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");

View Full Code Here


      MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
      MapOperator mapB = MapOperator.builder(IdentityMap.class).input(mapA).name("Map B").build();
      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
      
      FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, mapB, "Sink A");
      FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, mapC, "Sink B");
      FileDataSink sinkC = new FileDataSink(DummyOutputFormat.class, out3Path, mapC, "Sink C");
      
      List<FileDataSink> sinks = new ArrayList<FileDataSink>();
      sinks.add(sinkA);
      sinks.add(sinkB);
      sinks.add(sinkC);

View Full Code Here

      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0, 0)
        .input1(mat5)
        .input2(mat10)
        .build();
  
      FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, co);
      
      // return the PACT plan
      Plan plan = new Plan(sink, "Branching Source Multiple Times");
      
      OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

      CrossOperator c = CrossOperator.builder(new DummyCrossStub())
        .input1(r)
        .input2(mat2)
        .build();
      
      FileDataSink sinkA = new FileDataSink(new DummyOutputFormat(), out1Path, c);
      FileDataSink sinkB = new FileDataSink(new DummyOutputFormat(), out2Path, mat2);
      FileDataSink sinkC = new FileDataSink(new DummyOutputFormat(), out3Path, mat2);
      
      List<FileDataSink> sinks = new ArrayList<FileDataSink>();
      sinks.add(sinkA);
      sinks.add(sinkB);
      sinks.add(sinkC);

View Full Code Here

        .input1(cogroup5)
        .input2(cogroup6)
        .name("CoGroup 7")
        .build();
      
      FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cogroup7);
      sink.addInput(sourceA);
      sink.addInput(cogroup3);
      sink.addInput(cogroup4);
      sink.addInput(cogroup1);
      
      // return the PACT plan
      Plan plan = new Plan(sink, "Branching of each contract type");
      
      OptimizedPlan oPlan = compileNoStats(plan);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.flink.api.java.record.operators.FileDataSink

org.apache.flink.addons.hbase.example.HBaseReadExample

org.apache.flink.api.common.typeinfo.NothingTypeInfo

org.apache.flink.api.java.typeutils.RecordTypeInfo

org.apache.flink.compiler.AdditionalOperatorsTest

org.apache.flink.compiler.BranchingPlansCompilerTest

org.apache.flink.compiler.DOPChangeTest

org.apache.flink.compiler.GroupOrderTest

org.apache.flink.compiler.HardPlansCompilationTest

org.apache.flink.compiler.ReduceAllTest

org.apache.flink.compiler.UnionPropertyPropagationTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.