Examples of eu.stratosphere.api.java.ExecutionEnvironment.fromElements()

Class eu.stratosphere.api.java.ExecutionEnvironment

Examples of eu.stratosphere.api.java.ExecutionEnvironment.fromElements()

eu.stratosphere.api.java.ExecutionEnvironment.fromElements()
Creates a new data set that contains the given elements. The elements must all be of the same type, for example, all of the {@link String} or {@link Integer}. The sequence of elements must not be empty. Furthermore, the elements must be serializable (as defined in {@link java.io.Serializable}, because the execution environment may ship the elements into the cluster.
The framework will try and determine the exact type from the collection elements. In case of generic elements, it may be necessary to manually supply the type information via {@link #fromCollection(Collection,TypeInformation)}.
Note that this operation will result in a non-parallel data source, i.e. a data source with a degree of parallelism of one. @param data The elements to make up the data set. @return A DataSet representing the given list of elements.

  public void translateUnaryFunctionAnnotationTuples() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      
      @SuppressWarnings("unchecked")
      DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(new Tuple3<Long, String, Integer>(3l, "test", 42));
      input.map(new IndividualConstantMapper<Long, String, Integer>()).print();
      
      Plan plan = env.createProgramPlan();
      
      GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();

View Full Code Here

  public void translateBinaryFunctionAnnotationTuples() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      
      @SuppressWarnings("unchecked")
      DataSet<Tuple2<Long, String>> input1 = env.fromElements(new Tuple2<Long, String>(3l, "test"));
      @SuppressWarnings("unchecked")
      DataSet<Tuple2<Long, Double>> input2 = env.fromElements(new Tuple2<Long, Double>(3l, 3.1415));
      
      input1.join(input2).where(0).equalTo(0).with(new ForwardingTupleJoin<Long, String, Long, Double>())
        .print();

View Full Code Here

      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      
      @SuppressWarnings("unchecked")
      DataSet<Tuple2<Long, String>> input1 = env.fromElements(new Tuple2<Long, String>(3l, "test"));
      @SuppressWarnings("unchecked")
      DataSet<Tuple2<Long, Double>> input2 = env.fromElements(new Tuple2<Long, Double>(3l, 3.1415));
      
      input1.join(input2).where(0).equalTo(0).with(new ForwardingTupleJoin<Long, String, Long, Double>())
        .print();
      
      Plan plan = env.createProgramPlan();

View Full Code Here

  public void testAllGroupReduceNoCombiner() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
      
      data.reduceGroup(new GroupReduceFunction<Double, Double>() {
        public void reduce(Iterator<Double> values, Collector<Double> out) {}
      }).name("reducer")
      .print().name("sink");

View Full Code Here

      // compose test program
      {
        DataSet<Long> vertexIds = env.generateSequence(1, 2);
        
        @SuppressWarnings("unchecked")
        DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
        
        DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
        DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
        
        result.print();

View Full Code Here

      
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
      // compose test program
      {
        DataSet<Long> bcVar = env.fromElements(1L);
        
        DataSet<Long> vertexIds = env.generateSequence(1, 2);
        
        @SuppressWarnings("unchecked")
        DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));

View Full Code Here

        DataSet<Long> bcVar = env.fromElements(1L);
        
        DataSet<Long> vertexIds = env.generateSequence(1, 2);
        
        @SuppressWarnings("unchecked")
        DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
        
        DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
        
        VertexCentricIteration<Long, Long, Long, ?> vcIter = VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100);
        vcIter.addBroadcastSetForMessagingFunction(BC_VAR_NAME, bcVar);

View Full Code Here

      final int DOP = 8;
      ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(DOP);
      
      @SuppressWarnings("unchecked")
      DataSet<Tuple3<Double, StringValue, Long>> initialData = 
          env.fromElements(new Tuple3<Double, StringValue, Long>(3.141592, new StringValue("foobar"), new Long(77)));
      
      initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).print();
      
      Plan p = env.createProgramPlan();

View Full Code Here


  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


    DataSet<String> text = env.fromElements(WordCountData.TEXT);
    DataSet<Tuple2<String, Integer>> words = text.flatMap(new WordCount.Tokenizer());
    DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1);


    result.output(new LocalCollectionOutputFormat<Tuple2<String, Integer>>(resultsCollected));
    env.execute("Word Count Collection");

View Full Code Here


  public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Long> vertexIds = env.generateSequence(0, 10);
    DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(0L, 2L), new Tuple2<Long, Long>(2L, 4L), new Tuple2<Long, Long>(4L, 8L),
                              new Tuple2<Long, Long>(1L, 5L), new Tuple2<Long, Long>(3L, 7L), new Tuple2<Long, Long>(3L, 9L));
    
    DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
    
    DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.