Examples of FileDataSink

eu.stratosphere.api.java.record.operators.FileDataSink
Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat
org.apache.flink.api.java.record.operators.FileDataSink
s.apache.org/jira/browse/FLINK-1106">FLINK-1106 for more details. Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      ReduceOperator counts = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
        .input(wordsFirstInput, wordsSecondInput)
        .name("Word Counts")
        .build();


      FileDataSink sink = new FileDataSink(CsvOutputFormat.class, outputPath, counts);
      CsvOutputFormat.configureRecordFormat(sink)
        .recordDelimiter('\n')
        .fieldDelimiter(' ')
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      .input(findNearestClusterCenters)
      .name("Recompute Center Positions")
      .build();


    // create DataSinkContract for writing the new cluster positions
    FileDataSink newClusterPoints = new FileDataSink(new PointOutFormat(), output, recomputeClusterCenter, "New Center Positions");


    // return the plan
    Plan plan = new Plan(newClusterPoints, "KMeans Iteration");
    plan.setDefaultParallelism(numSubTasks);
    return plan;

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    
    FileDataSource lineItems =
      new FileDataSource(new IntTupleDataInFormat(), this.lineItemInputPath, "LineItems");
    lineItems.setDegreeOfParallelism(this.degreeOfParallelism);
    
    FileDataSink result = 
        new FileDataSink(new StringTupleDataOutFormat(), this.outputPath, "Output");
    result.setDegreeOfParallelism(degreeOfParallelism);
    
    MapOperator lineFilter = 
        MapOperator.builder(LiFilter.class)
      .name("LineItemFilter")
      .build();
    lineFilter.setDegreeOfParallelism(degreeOfParallelism);
    
    MapOperator ordersFilter = 
        MapOperator.builder(OFilter.class)
      .name("OrdersFilter")
      .build();
    ordersFilter.setDegreeOfParallelism(degreeOfParallelism);
    
    JoinOperator join = 
        JoinOperator.builder(JoinLiO.class, IntValue.class, 0, 0)
      .name("OrdersLineitemsJoin")
      .build();
      join.setDegreeOfParallelism(degreeOfParallelism);
    
    ReduceOperator aggregation = 
        ReduceOperator.builder(CountAgg.class, StringValue.class, 0)
      .name("AggregateGroupBy")
      .build();
    aggregation.setDegreeOfParallelism(this.degreeOfParallelism);
    
    lineFilter.setInput(lineItems);
    ordersFilter.setInput(orders);
    join.setFirstInput(ordersFilter);
    join.setSecondInput(lineFilter);
    aggregation.setInput(join);
    result.setInput(aggregation);
    
      
    return new Plan(result, "TPC-H 4");
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    FileDataSource input2 = new FileDataSource(new ContractITCaseInputFormat(), path2);
    
    MapOperator testMapper1 = MapOperator.builder(new TestMapper()).build();
    MapOperator testMapper2 = MapOperator.builder(new TestMapper()).build();


    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultDir);


    testMapper1.setInput(input1);
    testMapper2.setInput(input2);


    output.addInput(testMapper1);
    output.addInput(testMapper2);
    
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(DOP);


    PactCompiler pc = new PactCompiler(new DataStatistics());

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      final String output      = (args.length > 2 ? args[2] : "");


      @SuppressWarnings("unchecked")
      FileDataSource source = new FileDataSource(new CsvInputFormat(',', IntValue.class, IntValue.class, IntValue.class), recordsPath);


      FileDataSink sink = new FileDataSink(CsvOutputFormat.class, output);
      CsvOutputFormat.configureRecordFormat(sink)
        .recordDelimiter('\n')
        .fieldDelimiter(',')
        .lenient(true)
        .field(IntValue.class, 0)
        .field(IntValue.class, 1)
        .field(IntValue.class, 2);


      sink.setGlobalOrder(
        new Ordering(0, IntValue.class, Order.DESCENDING)
          .appendOrdering(1, IntValue.class, Order.ASCENDING)
          .appendOrdering(2, IntValue.class, Order.DESCENDING),
        new TripleIntDistribution(Order.DESCENDING, Order.ASCENDING, Order.DESCENDING));
      sink.setInput(source);


      Plan p = new Plan(sink);
      p.setDefaultParallelism(numSubtasks);
      return p;
    }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

        .name("Reduce something").build();


    MapOperator dummyMap = MapOperator.builder(new IdentityMapper()).input(dummyReduce).build();
    iteration.setNextPartialSolution(dummyMap);


    FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");


    Plan plan = new Plan(finalResult, "Iteration with chained map test");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    
    iteration.setNextWorkset(minAndUpdate);
    iteration.setSolutionSetDelta(minAndUpdate);


    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      .name("AggCo")
      .build();
    aggCO.setDegreeOfParallelism(numSubtasks);


    // create DataSinkContract for writing the result
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, "Output");
    result.setDegreeOfParallelism(numSubtasks);
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .field(IntValue.class, 0)
      .field(StringValue.class, 1);


    // assemble the plan
    result.setInput(aggCO);
    aggCO.setInput(joinCO);
    joinCO.setFirstInput(orders);
    joinCO.setSecondInput(customers);


    return new Plan(result, "TPCH Asterix");

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    } else {
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
          config.getString("MatchTest#ShipStrategy", ""));
    }


    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testMatcher);
    testMatcher.setFirstInput(input_left);
    testMatcher.setSecondInput(input_right);


    return new Plan(output);
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    }
    
    iteration.setNextWorkset(updateComponentId);


    // sink is the iteration result
    FileDataSink result = new FileDataSink(new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class), output, iteration, "Result");


    // return the PACT plan
    Plan plan = new Plan(result, "Workset Connected Components");
    plan.setDefaultParallelism(numSubTasks);
    return plan;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.