Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline.run()

93
94
95
96
97
98
99
100
101
102
103
    TableSourceTarget<String, Long> inter = At.sequenceFile(output, Writables.strings(), Writables.longs());
    set1Lengths.write(inter);
    set2Counts.write(inter, Target.WriteMode.APPEND);

    pipeline.run();

    PTable<String, Long> in = pipeline.read(inter);
    Set<Pair<String, Long>> values = Sets.newHashSet(in.materialize());
    assertEquals(7, values.size());
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
    String out = tempDir.getFileName("out");
    SparkPipeline pipeline = new SparkPipeline("local", "skipptypes");
    PCollection<String> shakes = pipeline.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")));
    PTable<String, Long> wcnt = shakes.count();
    wcnt.write(new MySeqFileTableSourceTarget(out, ptt));
    pipeline.run();

    PTable<Text, LongWritable> wcntIn = pipeline.read(new MySeqFileTableSourceTarget(out, ptt));
    assertEquals(new LongWritable(1L), wcntIn.materialize().iterator().next().second());
    pipeline.done();
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    TableSourceTarget<String, Long> inter = At.sequenceFile(output, Writables.strings(), Writables.longs());
    set1Lengths.write(inter);
    set2Counts.write(inter, Target.WriteMode.APPEND);

    pipeline.run();

    PTable<String, Long> in = pipeline.read(inter);
    Set<Pair<String, Long>> values = Sets.newHashSet(in.materialize());
    assertEquals(7, values.size());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.