Package org.apache.spark.api.java

Examples of org.apache.spark.api.java.JavaDoubleRDD

557
558
559
560
561
562
563
564
565
566
567
568
569
  }

  @SuppressWarnings("unchecked")
  @Test
  public void persist() {
    JavaDoubleRDD doubleRDD = sc.parallelizeDoubles(Arrays.asList(1.0, 1.0, 2.0, 3.0, 5.0, 8.0));
    doubleRDD = doubleRDD.persist(StorageLevel.DISK_ONLY());
    Assert.assertEquals(20, doubleRDD.sum(), 0.1);

    List<Tuple2<Integer, String>> pairs = Arrays.asList(
      new Tuple2<Integer, String>(1, "a"),
      new Tuple2<Integer, String>(2, "aa"),
      new Tuple2<Integer, String>(3, "aaa")
View Full Code Here
831
832
833
834
835
836
837
838
839
840
841
  }

  @Test
  public void zip() {
    JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
    JavaDoubleRDD doubles = rdd.mapToDouble(new DoubleFunction<Integer>() {
      @Override
      public double call(Integer x) {
        return 1.0 * x;
      }
    });
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
  }

  @Test
  public void map() {
    JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
    JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache();
    doubles.collect();
    JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x))
      .cache();
    pairs.collect();
    JavaRDD<String> strings = rdd.map(x -> x.toString()).cache();
    strings.collect();
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    });

    Assert.assertEquals(new Tuple2<>("Hello", "Hello"), pairs.first());
    Assert.assertEquals(11, pairs.count());

    JavaDoubleRDD doubles = rdd.flatMapToDouble(s -> {
      List<Double> lengths = new LinkedList<>();
      for (String word : s.split(" ")) lengths.add(word.length() * 1.0);
      return lengths;
    });

    Double x = doubles.first();
    Assert.assertEquals(5.0, doubles.first(), 0.01);
    Assert.assertEquals(11, pairs.count());
  }
View Full Code Here
268
269
270
271
272
273
274
275
276
  }

  @Test
  public void zip() {
    JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
    JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x);
    JavaPairRDD<Integer, Double> zipped = rdd.zip(doubles);
    zipped.count();
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
  @Test
  public void testUniformRDD() {
    long m = 1000L;
    int p = 2;
    long seed = 1L;
    JavaDoubleRDD rdd1 = uniformJavaRDD(sc, m);
    JavaDoubleRDD rdd2 = uniformJavaRDD(sc, m, p);
    JavaDoubleRDD rdd3 = uniformJavaRDD(sc, m, p, seed);
    for (JavaDoubleRDD rdd: Lists.newArrayList(rdd1, rdd2, rdd3)) {
      Assert.assertEquals(m, rdd.count());
    }
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
  @Test
  public void testNormalRDD() {
    long m = 1000L;
    int p = 2;
    long seed = 1L;
    JavaDoubleRDD rdd1 = normalJavaRDD(sc, m);
    JavaDoubleRDD rdd2 = normalJavaRDD(sc, m, p);
    JavaDoubleRDD rdd3 = normalJavaRDD(sc, m, p, seed);
    for (JavaDoubleRDD rdd: Lists.newArrayList(rdd1, rdd2, rdd3)) {
      Assert.assertEquals(m, rdd.count());
    }
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
  public void testPoissonRDD() {
    double mean = 2.0;
    long m = 1000L;
    int p = 2;
    long seed = 1L;
    JavaDoubleRDD rdd1 = poissonJavaRDD(sc, mean, m);
    JavaDoubleRDD rdd2 = poissonJavaRDD(sc, mean, m, p);
    JavaDoubleRDD rdd3 = poissonJavaRDD(sc, mean, m, p, seed);
    for (JavaDoubleRDD rdd: Lists.newArrayList(rdd1, rdd2, rdd3)) {
      Assert.assertEquals(m, rdd.count());
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.spark.api.java.JavaDoubleRDD

  • org.apache.spark.Java8APISuite
  • org.apache.spark.JavaAPISuite
  • org.apache.spark.mllib.random.JavaRandomRDDsSuite
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.