Examples of org.apache.crunch.PipelineResult

org.apache.crunch.PipelineResult
Container for the results of a call to {@code run} or {@code done} on thePipeline interface that includes details and statistics about the component stages of the data pipeline.

        targetDeps.remove(output);
      }
    }
    if (status.get() != Status.FAILED || status.get() != Status.KILLED) {
      status.set(Status.SUCCEEDED);
      result = new PipelineResult(ImmutableList.of(new PipelineResult.StageResult("Spark", null)),
          Status.SUCCEEDED);
      set(result);
    } else {
      set(PipelineResult.EMPTY);
    }

View Full Code Here

  private static class MemExecution extends AbstractFuture<PipelineResult> implements PipelineExecution {


    private PipelineResult res;


    public MemExecution() {
      this.res = new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)),
          PipelineExecution.Status.SUCCEEDED);
    }

View Full Code Here

    PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, Writables.strings()));
    PCollection<String> words = split(shakespeare, "\\s+");
    PTable<String, Long> wordCounts = words.count();
    pipeline.write(convertToKeyValues(wordCounts), ToHBase.hfile(outputPath));


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    KeyValue kv = readFromHFiles(fs, outputPath, "and");
    assertEquals(427L, Bytes.toLong(kv.getValue()));
  }

View Full Code Here

    HFileUtils.writePutsToHFilesForIncrementalLoad(
        wordCountPuts,
        testTable,
        outputPath);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    new LoadIncrementalHFiles(HBASE_TEST_UTILITY.getConfiguration())
        .doBulkLoad(outputPath, testTable);


    Map<String, Long> EXPECTED = ImmutableMap.<String, Long>builder()

View Full Code Here

    HFileUtils.writePutsToHFilesForIncrementalLoad(
        convertToPuts(longWordCounts),
        table2,
        outputPath2);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    loader.doBulkLoad(outputPath1, table1);
    loader.doBulkLoad(outputPath2, table2);


    assertEquals(396L, getWordCountFromTable(table1, "of"));

View Full Code Here

    HFileUtils.writePutsToHFilesForIncrementalLoad(
        wordCountPuts,
        testTable,
        outputPath);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    int hfilesCount = 0;
    Configuration conf = HBASE_TEST_UTILITY.getConfiguration();
    FileSystem fs = outputPath.getFileSystem(conf);
    for (FileStatus e : fs.listStatus(new Path(outputPath, Bytes.toString(TEST_FAMILY)))) {

View Full Code Here

    this.tempDirectory = createTempDirectory(conf);
  }


  @Override
  public PipelineResult done() {
    PipelineResult res = PipelineResult.EMPTY;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;

View Full Code Here

    return runtime;
  }


  @Override
  public PipelineResult done() {
    PipelineResult res = super.done();
    if (sparkContext != null) {
      sparkContext.stop();
      sparkContext = null;
    }
    return res;

View Full Code Here

        } else if (!failures.isEmpty()) {
          status.set(Status.FAILED);
        } else {
          status.set(Status.SUCCEEDED);
        }
        result = new PipelineResult(stages, status.get());
        set(result);
      }
    } catch (InterruptedException e) {
      throw new AssertionError(e); // Nobody should interrupt us.
    } catch (IOException e) {

View Full Code Here

      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
      assertEquals(kvs.get(i).toString(), lines.get(i));

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.crunch.PipelineResult

$.WordCount

org.apache.crunch.examples.AverageBytesByIP

org.apache.crunch.examples.TotalBytesByIP

org.apache.crunch.examples.TotalWordCount

org.apache.crunch.examples.WordCount

org.apache.crunch.impl.dist.DistributedPipeline

org.apache.crunch.impl.mem.MemPipeline

org.apache.crunch.impl.mem.MemPipeline$MemExecution

org.apache.crunch.impl.mr.exec.MRExecutor

org.apache.crunch.impl.mr.MRPipeline

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.