Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
   * get failed. The rest of running jobs should be killed.
   */
  @Test
  public void testStopPipelineImmediatelyOnJobFailure() throws Exception {
    String inPath = tmpDir.copyResourceFileName("shakes.txt");
    MRPipeline pipeline = new MRPipeline(MRExecutorIT.class);

    // Issue two jobs that sleep forever.
    PCollection<String> in = pipeline.read(From.textFile(inPath));
    for (int i = 0; i < 2; i++) {
      in.count()
          .values()
          .parallelDo(new SleepForeverFn(), longs())
          .write(To.textFile(tmpDir.getPath("out_" + i)));
    }
    MRPipelineExecution exec = pipeline.runAsync();

    // Wait until both of the two jobs are submitted.
    List<MRJob> jobs = exec.getJobs();
    assertEquals(2, jobs.size());
    StopWatch watch = new StopWatch();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    List<KeyValue> kvs = generateKeyValues(100);
    Path inputPath = tmpDir.getPath("in");
    Path outputPath = tmpDir.getPath("out");
    writeKeyValuesToHFile(inputPath, kvs);

    Pipeline pipeline = new MRPipeline(HFileSourceIT.class, conf);
    PCollection<KeyValue> in = pipeline.read(FromHBase.hfile(inputPath));
    PCollection<String> texts = in.parallelDo(new MapFn<KeyValue, String>() {
      @Override
      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
View Full Code Here
268
269
270
271
272
273
274
275
276
  private List<Result> doTestScanHFiles(List<KeyValue> kvs, Scan scan) throws IOException {
    Path inputPath = tmpDir.getPath("in");
    writeKeyValuesToHFile(inputPath, kvs);

    Pipeline pipeline = new MRPipeline(HFileSourceIT.class, conf);
    PCollection<Result> results = HFileUtils.scanHFiles(pipeline, inputPath, scan);
    return ImmutableList.copyOf(results.materialize());
  }
View Full Code Here
277
278
279
280
281
282
283
284
285
  private List<KeyValue> doTestReadHFiles(List<KeyValue> kvs, Scan scan) throws IOException {
    Path inputPath = tmpDir.getPath("in");
    writeKeyValuesToHFile(inputPath, kvs);

    Pipeline pipeline = new MRPipeline(HFileSourceIT.class, conf);
    PCollection<KeyValue> results = pipeline.read(FromHBase.hfile(inputPath));
    return ImmutableList.copyOf(results.materialize());
  }
View Full Code Here
123
124
125
126
127
128
129
    hbaseTestUtil.startMiniHBaseCluster(1, 1);
  }

  @Test
  public void testWordCount() throws Exception {
    run(new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration()));
  }
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
    pipeline.done();

    //verify HBaseTarget supports deletes.
    Scan clearScan = new Scan();
    clearScan.addFamily(COUNTS_COLFAM);
    pipeline = new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration());
    HBaseSourceTarget clearSource = new HBaseSourceTarget(outputTableName, clearScan);
    PTable<ImmutableBytesWritable, Result> counts = pipeline.read(clearSource);
    pipeline.write(clearCounts(counts), new HBaseTarget(outputTableName));
    pipeline.done();
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
  @Rule
  public TemporaryPath tmpDir = TemporaryPaths.create();

  @Test
  public void testNoBreakpoint() throws Exception {
    run(new MRPipeline(BreakpointIT.class, tmpDir.getDefaultConfiguration()),
        tmpDir.copyResourceFileName("shakes.txt"),
        tmpDir.getFileName("out1"),
        tmpDir.getFileName("out2"),
        false);
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
        false);
  }

  @Test
  public void testBreakpoint() throws Exception {
    run(new MRPipeline(BreakpointIT.class, tmpDir.getDefaultConfiguration()),
        tmpDir.copyResourceFileName("shakes.txt"),
        tmpDir.getFileName("out1"),
        tmpDir.getFileName("out2"),
        true);
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  public transient TemporaryPath tmpDir = TemporaryPaths.create();

  @Test
  public void materializedColShouldBeWritten() throws Exception {
    File textFile = tmpDir.copyResourceFile("shakes.txt");
    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> genericCollection = pipeline.readTextFile(textFile.getAbsolutePath());
    pipeline.run();
    PCollection<String> filter = genericCollection.filter("Filtering data", FilterFns.<String>ACCEPT_ALL());
    filter.materialize();
    pipeline.run();
    File file = tmpDir.getFile("output.txt");
    Target outFile = To.textFile(file.getAbsolutePath());
    PCollection<String> write = filter.write(outFile);
    write.materialize();
    pipeline.run();
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
 
 
  @Test
  public void testPGroupedTableToMultipleOutputs() throws IOException{
    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
    PGroupedTable<String, String> groupedLineTable = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt")).by(IdentityFn.<String>getInstance(), Writables.strings()).groupByKey();
   
    PTable<String, String> ungroupedTableA = groupedLineTable.ungroup();
    PTable<String, String> ungroupedTableB = groupedLineTable.ungroup();
   
    File outputDirA = tmpDir.getFile("output_a");
    File outputDirB = tmpDir.getFile("output_b");
   
    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
    PipelineResult result = pipeline.done();
    for(StageResult stageResult : result.getStageResults()){
      assertTrue(stageResult.getStageName().length() > 1);
      assertTrue(stageResult.getStageId().length() > 1);
    }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.mr.MRPipeline

  • com.cloudera.cdk.data.crunch.TestCrunchDatasets
  • org.apache.bigtop.bigpetstore.etl.CrunchETL
  • org.apache.crunch.Breakpoint2IT
  • org.apache.crunch.BreakpointIT
  • org.apache.crunch.CancelJobsIT
  • org.apache.crunch.CheckpointIT
  • org.apache.crunch.CleanTextIT
  • org.apache.crunch.CollectionPObjectIT
  • org.apache.crunch.CollectionsIT
  • org.apache.crunch.CollectionsLengthIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.