Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline.emptyPCollection()

44
45
46
47
48
49
50
51
52
53
54
  public TemporaryPath tempDir = new TemporaryPath();

  @Test
  public void testEmptyMR() throws Exception {
    Pipeline p = new SparkPipeline("local", "empty");
    assertTrue(Iterables.isEmpty(p.emptyPCollection(Writables.strings())
        .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs()))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
  }

  @Test
  public void testUnionWithEmptyMR() throws Exception {
    Pipeline p = new SparkPipeline("local", "empty");
    assertFalse(Iterables.isEmpty(p.emptyPCollection(Writables.strings())
        .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs()))
        .union(
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.