Examples of FileDataSink

eu.stratosphere.api.java.record.operators.FileDataSink
Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat
org.apache.flink.api.java.record.operators.FileDataSink
s.apache.org/jira/browse/FLINK-1106">FLINK-1106 for more details. Operator for nodes which act as data sinks, storing the data they receive in a file instead of sending it to another contract. The encoding of the data in the file is handled by the {@link FileOutputFormat}. @see FileOutputFormat

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));


    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(IntValue.class, 0)
      .field(IntValue.class, 1);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    iteration.setNextPartialSolution(rankAggregation);
    iteration.setMaximumNumberOfIterations(numIterations);
    iteration.getAggregators().registerAggregationConvergenceCriterion(DotProductCoGroup.AGGREGATOR_NAME, new PageRankStatsAggregator(), 
        new DiffL1NormConvergenceCriterion());
    
    FileDataSink out = new FileDataSink(new PageWithRankOutFormat(), outputPath, iteration, "Final Ranks");


    Plan p = new Plan(out, "Dangling PageRank");
    p.setDefaultParallelism(dop);
    return p;
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    final FileDataSource source = 
        new FileDataSource(new TeraInputFormat(), input, "Data Source");
    source.setDegreeOfParallelism(numSubTasks);


    // This task writes the sorted data back to disk
    final FileDataSink sink = 
        new FileDataSink(new TeraOutputFormat(), output, "Data Sink");
    sink.setDegreeOfParallelism(numSubTasks);
    sink.setGlobalOrder(new Ordering(0, TeraKey.class, Order.ASCENDING), new TeraDistribution());


    sink.setInput(source);


    return new Plan(sink, "TeraSort");
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

      .keyField(StringValue.class, 6)
      .keyField(StringValue.class, 7)
      .name("Reduce")
      .build();


    FileDataSink result = new FileDataSink(new TupleOutputFormat(), resultPath, "Output");


    result.setInput(reduce);
    
    reduce.setInput(joinNCOL);
    
    joinNCOL.setFirstInput(joinCOL);
    joinNCOL.setSecondInput(projectN);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

        .name("Join Old and New")
        .build();
    
    iteration.setTerminationCriterion(termination);
    
    FileDataSink out = new FileDataSink(new PageWithRankOutFormat(), outputPath, iteration, "Final Ranks");


    Plan p = new Plan(out, "Simple PageRank");
    p.setDefaultParallelism(dop);
    return p;
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

        .name("Close Triads")
        .build();
    closeTriads.setParameter("INPUT_SHIP_STRATEGY", "SHIP_REPARTITION_HASH");
    closeTriads.setParameter("LOCAL_STRATEGY", "LOCAL_STRATEGY_HASH_BUILD_SECOND");


    FileDataSink triangles = new FileDataSink(new TriangleOutputFormat(), output, closeTriads, "Triangles");


    Plan p = new Plan(triangles, "Enumerate Triangles");
    p.setDefaultParallelism(numSubTasks);
    return p;
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

        .build();
      ReduceOperator reduceNode = ReduceOperator.builder(new CountWords(), StringValue.class, 0)
        .input(mapNode)
        .name("Count Words")
        .build();
      FileDataSink out = new FileDataSink(new CsvOutputFormat(), OUT_FILE, reduceNode, "Word Counts");
      CsvOutputFormat.configureRecordFormat(out)
        .recordDelimiter('\n')
        .fieldDelimiter(' ')
        .lenient(true)
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);
      
      Ordering ordering = new Ordering(0, StringValue.class, Order.DESCENDING);
      out.setGlobalOrder(ordering, new SimpleDistribution(new StringValue[] {new StringValue("N")}));
      
      Plan p = new Plan(out, "WordCount Example");
      p.setDefaultParallelism(DEFAULT_PARALLELISM);
  
      OptimizedPlan plan;

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build();


    // Mp2 filters out all p values which can be divided by z
    MapOperator mp2 = MapOperator.builder(Mp2.class).setBroadcastVariable("z", mp1).input(jn2).build();


    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);
    output.setInput(mp2);


    return new Plan(output);
  }

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    
    iteration.setNextWorkset(updateComponentId);
    iteration.setSolutionSetDelta(updateComponentId);


    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.FileDataSink

    String output    = (args.length > 1 ? args[1] : "");


    GenericDataSource<TableInputFormat> source = new GenericDataSource<TableInputFormat>(new MyTableInputFormat(), "HBase Input");
    source.setParameter(TableInputFormat.INPUT_TABLE, "twitter");
    source.setParameter(TableInputFormat.CONFIG_LOCATION, "/etc/hbase/conf/hbase-site.xml");
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, source, "HBase String dump");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0)
      .field(StringValue.class, 1)

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.