Package org.apache.spark.api.java

Examples of org.apache.spark.api.java.JavaRDD

56
57
58
59
60
61
62
63
64
65
66
    JavaPairRDD[] rdds = new JavaPairRDD[parents.size()];
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        rdds[i] = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      } else {
        JavaRDD rdd = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = rdd.mapPartitionsToPair(new PairFlatMapDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      }
    }
    return runtime.getSparkContext().union(rdds);
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
        long messages = data.filter(new Function<Tuple2<Text, MapWritable>, Boolean>() {
            public Boolean call(Tuple2<Text, MapWritable> t) { return t._2.containsKey(new Text("message")); }
        }).count();

        JavaRDD map = data.map(new Function<Tuple2<Text, MapWritable>, Map<String, Object>>() {
            public Map<String, Object> call(Tuple2<Text, MapWritable> v1) throws Exception {
                return (Map<String, Object>) WritableUtils.fromWritable(v1._2);
            }
        });

        JavaRDD fooBar = data.map(new Function<Tuple2<Text, MapWritable>, String>() {
            public String call(Tuple2<Text, MapWritable> v1) throws Exception {
                return v1._1.toString();
            }
        });

        assertThat((int) data.count(), is(2));
        System.out.println(data.take(10));
        System.out.println(messages);
        System.out.println(fooBar.take(2));
        System.out.println(map.take(10));
    }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    JavaPairRDD[] rdds = new JavaPairRDD[parents.size()];
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        rdds[i] = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      } else {
        JavaRDD rdd = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = rdd.mapPartitions(new PairFlatMapDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      }
    }
    return runtime.getSparkContext().union(rdds);
  }
View Full Code Here
224
225
226
227
228
229
230
231
232
233
    }

    @Test
    public void testJavaFunctions5() throws Exception {
        RDD rdd = mock(RDD.class);
        JavaRDD jrdd = mock(JavaRDD.class);
        when(jrdd.rdd()).thenReturn(rdd);
        RDDJavaFunctions rddjf = javaFunctions(jrdd);
        assertThat(rddjf.rdd, is(rdd));
    }
View Full Code Here
TOP

Related Classes of org.apache.spark.api.java.JavaRDD

  • com.datastax.spark.connector.CassandraJavaUtilTest
  • org.apache.crunch.impl.spark.collect.UnionTable
  • org.elasticsearch.spark.integration.AbstractHadoopBasicSparkTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.