Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline

34
35
36
37
38
39
40
41
42
  @Rule
  public TemporaryPath tmpDir = TemporaryPaths.create();
 
  @Test
  public void testMR() throws Exception {
    run(new MRPipeline(LongPipelinePlannerIT.class, tmpDir.getDefaultConfiguration()),
        tmpDir.copyResourceFileName("shakes.txt"),
        tmpDir.getFileName("output"));
  }
View Full Code Here
50
51
52
53
54
55
56
  // total number of documents, should calculate
  protected static final double N = 2;

  @Test
  public void testWritablesSingleRun() throws IOException {
    run(new MRPipeline(TfIdfIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance(), true);
  }
View Full Code Here
55
56
57
58
59
60
61
    run(new MRPipeline(TfIdfIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance(), true);
  }

  @Test
  public void testWritablesMultiRun() throws IOException {
    run(new MRPipeline(TfIdfIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance(), false);
  }
View Full Code Here
204
205
206
207
208
209
210
    jos.closeEntry();
  }

  @Test
  public void testWordCount() throws IOException {
    run(new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration()));
  }
View Full Code Here
270
271
272
273
274
275
276
277
278
279
280
      pipeline.done();

      //verify HBaseTarget supports deletes.
      Scan clearScan = new Scan();
      clearScan.addFamily(COUNTS_COLFAM);
      pipeline = new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration());
      HBaseSourceTarget clearSource = new HBaseSourceTarget(outputTableName, clearScan);
      PTable<ImmutableBytesWritable, Result> counts = pipeline.read(clearSource);
      pipeline.write(clearCounts(counts), new HBaseTarget(outputTableName));
      pipeline.done();
     
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
  public TemporaryPath tmpDir = TemporaryPaths.create();
 
  @Test
  public void testCheckpoints() throws Exception {
    String inputPath = tmpDir.copyResourceFileName("shakes.txt");
    Pipeline p = new MRPipeline(CheckpointIT.class);
    String inter = tmpDir.getFileName("intermediate");
    PipelineResult one = run(p, tmpDir, inputPath, inter, false);
    assertTrue(one.succeeded());
    assertEquals(2, one.getStageResults().size());
    PipelineResult two = run(p, tmpDir, inputPath, inter, false);
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
  }
 
  @Test
  public void testUnsuccessfulCheckpoint() throws Exception {
    String inputPath = tmpDir.copyResourceFileName("shakes.txt");
    Pipeline p = new MRPipeline(CheckpointIT.class);
    String inter = tmpDir.getFileName("intermediate");
    PipelineResult one = run(p, tmpDir, inputPath, inter, true);
    assertFalse(one.succeeded());
    PipelineResult two = run(p, tmpDir, inputPath, inter, false);
    assertTrue(two.succeeded());
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
  }
 
  @Test
  public void testModifiedFileCheckpoint() throws Exception {
    String inputPath = tmpDir.copyResourceFileName("shakes.txt");
    Pipeline p = new MRPipeline(CheckpointIT.class);
    Path inter = tmpDir.getPath("intermediate");
    PipelineResult one = run(p, tmpDir, inputPath, inter.toString(), false);
    assertTrue(one.succeeded());
    assertEquals(2, one.getStageResults().size());
    // Update the input path
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    List<KeyValue> kvs = generateKeyValues(100);
    Path inputPath = tmpDir.getPath("in");
    Path outputPath = tmpDir.getPath("out");
    writeKeyValuesToHFile(inputPath, kvs);

    Pipeline pipeline = new MRPipeline(HFileSourceIT.class, conf);
    PCollection<KeyValue> in = pipeline.read(FromHBase.hfile(inputPath));
    PCollection<String> texts = in.parallelDo(new MapFn<KeyValue, String>() {
      @Override
      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
View Full Code Here
267
268
269
270
271
272
273
274
275
  private List<Result> doTestScanHFiles(List<KeyValue> kvs, Scan scan) throws IOException {
    Path inputPath = tmpDir.getPath("in");
    writeKeyValuesToHFile(inputPath, kvs);

    Pipeline pipeline = new MRPipeline(HFileSourceIT.class, conf);
    PCollection<Result> results = HFileUtils.scanHFiles(pipeline, inputPath, scan);
    return ImmutableList.copyOf(results.materialize());
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.mr.MRPipeline

  • com.cloudera.cdk.data.crunch.TestCrunchDatasets
  • org.apache.bigtop.bigpetstore.etl.CrunchETL
  • org.apache.crunch.Breakpoint2IT
  • org.apache.crunch.BreakpointIT
  • org.apache.crunch.CancelJobsIT
  • org.apache.crunch.CheckpointIT
  • org.apache.crunch.CleanTextIT
  • org.apache.crunch.CollectionPObjectIT
  • org.apache.crunch.CollectionsIT
  • org.apache.crunch.CollectionsLengthIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.