Examples of org.apache.flink.api.java.ExecutionEnvironment.readTextFile()

Class org.apache.flink.api.java.ExecutionEnvironment

Examples of org.apache.flink.api.java.ExecutionEnvironment.readTextFile()

org.apache.flink.api.java.ExecutionEnvironment.readTextFile()
ocal/file" or "hdfs://host:port/file/path"). @return A DataSet that represents the data read from the given file as text lines.

    
    // construct the plan it will be multiple flat maps, all unioned
    // and the "unioned" dataSet will be grouped
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> source = env.readTextFile(IN_FILE);
    DataSet<Tuple2<String, Integer>> lastUnion = source.flatMap(new DummyFlatMap());
  
    for (int i = 1; i< NUM_INPUTS; i++){
      lastUnion = lastUnion.union(source.flatMap(new DummyFlatMap()));
    }

View Full Code Here

  
  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


    DataSet<String> text = env.readTextFile(textPath);


    DataSet<WC> counts = text
        .flatMap(new Tokenizer())
        .groupBy("word")
        .reduce(new ReduceFunction<WC>() {

View Full Code Here

  }
  
  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> text = env.readTextFile(textPath);


    DataSet<WC> counts = text
        .flatMap(new Tokenizer())
        .groupBy("complex.someTest")
        .reduce(new ReduceFunction<WC>() {

View Full Code Here


  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(dataPath); 
    
    input.flatMap(new TokenizeLine())
      .groupBy(0)
      .reduceGroup(new CountWords())
      .writeAsCsv(resultPath, "\n", " ");

View Full Code Here

  
  @Test
  public void testBranchingBroadcastVariable() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
    DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
    DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
    
    DataSet<String> result1 = input1
        .map(new IdentityMapper<String>())

View Full Code Here

  @Test
  public void testBranchingBroadcastVariable() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
    DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
    DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
    
    DataSet<String> result1 = input1
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>())

View Full Code Here

  public void testBranchingBroadcastVariable() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
    DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
    DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
    
    DataSet<String> result1 = input1
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>())
          .withBroadcastSet(input3, "bc");

View Full Code Here

  
  @Test
  public void testBCVariableClosure() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
    
    DataSet<String> reduced = input
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>());

View Full Code Here

  
  @Test
  public void testMultipleIterations() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
    
    DataSet<String> reduced = input
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>());

View Full Code Here

  
  @Test
  public void testMultipleIterationsWithClosueBCVars() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
      
    IterativeDataSet<String> iteration1 = input.iterate(100);
    IterativeDataSet<String> iteration2 = input.iterate(20);
    IterativeDataSet<String> iteration3 = input.iterate(17);

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.