Examples of readTextFile()

Examples of com.cloudera.crunch.Pipeline.readTextFile()

23
24
25
26
27
28
29
30
31
32
33
public class SortCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(SortCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("sort/A");
   
    PCollection<Pair<Integer, Integer>> pairs = records.parallelDo(new DoFn<String, Pair<Integer, Integer>>() {
      @Override
      public void process(String input, Emitter<Pair<Integer, Integer>> emitter) {
        Iterator<String> split = Splitter.on('\t').split(input).iterator();
View Full Code Here

Examples of com.cloudera.crunch.Pipeline.readTextFile()

21
22
23
24
25
26
27
28
29
30
31
  private static final int MISSING = 9999;
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(MaxTemperatureCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("input");
   
    PTable<String, Integer> maxTemps = records
      .parallelDo(toYearTempPairsFn(), tableOf(strings(), ints()))
      .groupByKey()
      .combineValues(CombineFn.<String> MAX_INTS());
View Full Code Here

Examples of com.cloudera.crunch.Pipeline.readTextFile()

21
22
23
24
25
26
27
28
29
30
31
public class JoinCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(JoinCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
View Full Code Here

Examples of com.cloudera.crunch.Pipeline.readTextFile()

22
23
24
25
26
27
28
29
30
31
32
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(JoinCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
View Full Code Here

Examples of com.cloudera.crunch.Pipeline.readTextFile()

23
24
25
26
27
28
29
30
31
32
33
public class CogroupCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(CogroupCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
View Full Code Here

Examples of com.cloudera.crunch.Pipeline.readTextFile()

24
25
26
27
28
29
30
31
32
33
34
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(CogroupCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
View Full Code Here

Examples of com.cloudera.crunch.impl.mr.MRPipeline.readTextFile()

23
24
25
26
27
28
29
30
31
32
33
public class SortCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(SortCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("sort/A");
   
    PCollection<Pair<Integer, Integer>> pairs = records.parallelDo(new DoFn<String, Pair<Integer, Integer>>() {
      @Override
      public void process(String input, Emitter<Pair<Integer, Integer>> emitter) {
        Iterator<String> split = Splitter.on('\t').split(input).iterator();
View Full Code Here

Examples of com.cloudera.crunch.impl.mr.MRPipeline.readTextFile()

21
22
23
24
25
26
27
28
29
30
31
  private static final int MISSING = 9999;
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(MaxTemperatureCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("input");
   
    PTable<String, Integer> maxTemps = records
      .parallelDo(toYearTempPairsFn(), tableOf(strings(), ints()))
      .groupByKey()
      .combineValues(CombineFn.<String> MAX_INTS());
View Full Code Here

Examples of com.cloudera.crunch.impl.mr.MRPipeline.readTextFile()

21
22
23
24
25
26
27
28
29
30
31
public class JoinCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(JoinCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
View Full Code Here

Examples of com.cloudera.crunch.impl.mr.MRPipeline.readTextFile()

22
23
24
25
26
27
28
29
30
31
32
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(JoinCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.