Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

110
111
112
113
114
115
116
117
118
119
120
      }
    }, Writables.tableOf(Writables.writables(IntWritable.class), Writables.writables(Text.class)));
   
    PTable<Text, LongWritable> out = Mapreduce.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(19, sr.getCounters().findCounter("words", "where").getValue());
  }
}
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
      }
    }, Writables.tableOf(Writables.writables(IntWritable.class), Writables.writables(Text.class)));
   
    PTable<Text, LongWritable> out = Mapred.map(two, TestMapper.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
 
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
      }
    }, Writables.tableOf(Writables.writables(IntWritable.class), Writables.writables(Text.class)));
   
    PTable<Text, LongWritable> out = Mapred.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(108, sr.getCounters().findCounter("thou", "count").getValue());
  }
}
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    String urlsFile = tmpDir.copyResourceFileName("urls.txt");
    String out = tmpDir.getFileName("out");
    MRPipeline p = new MRPipeline(CompressIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> in = p.readTextFile(urlsFile);
    in.write(Compress.gzip(To.textFile(out)));
    p.done();
    assertTrue(checkDirContainsExt(out, ".gz"));
  }

  @Test
  public void testCompressAvro() throws Exception {
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    assertTrue(Iterables.isEmpty(p.emptyPCollection(Writables.strings())
        .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs()))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }

  @Test
  public void testUnionWithEmptyMR() throws Exception {
    MRPipeline p = new MRPipeline(EmptyPCollectionIT.class, tempDir.getDefaultConfiguration());
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    String urlsFile = tmpDir.copyResourceFileName("urls.txt");
    String out = tmpDir.getFileName("out");
    MRPipeline p = new MRPipeline(CompressIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> in = p.read(From.textFile(urlsFile, Avros.strings()));
    in.write(Compress.snappy(To.avroFile(out)));
    p.done();

    FileSystem fs = FileSystem.get(tmpDir.getDefaultConfiguration());
    FileStatus fstat = fs.getFileStatus(new Path(out, "part-m-00000.avro"));
    assertEquals(176, fstat.getLen());
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }

  @Test
  public void testUnionTableWithEmptyMR() throws Exception {
    MRPipeline p = new MRPipeline(EmptyPCollectionIT.class, tempDir.getDefaultConfiguration());
View Full Code Here
76
77
78
79
80
81
82
83
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }
}
View Full Code Here
46
47
48
49
50
51
52
53
54
55
    Files.copy(src2, new File(srcFiles, "src2.txt"));

    MRPipeline p = new MRPipeline(CombineFileIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> in = p.readTextFile(srcFiles.getAbsolutePath());
    in.write(To.textFile(outputFiles.getAbsolutePath()));
    p.done();
    assertEquals(4, outputFiles.listFiles().length);
  }

}
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
  public void testWritables() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, WritableTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.