Examples of org.apache.crunch.PipelineResult$StageResult

org.apache.crunch.PipelineResult
Container for the results of a call to {@code run} or {@code done} on thePipeline interface that includes details and statistics about the component stages of the data pipeline.

  private void testStreamTextInputFile() throws Exception {
    String inputPath = tmpDir.copyResourceFileName("test-documents/hello1.txt");
    String[] expected = new String[] {"hello foo", "hello world"};
    String[] args = getInitialArgs(LOAD_SOLR_LINE);
    args = ObjectArrays.concat(args, inputPath);    
    PipelineResult pipelineResult = runIntoSolr(args, expected);
    Assert.assertTrue(pipelineResult.getStageResults().get(0).getCounterValue("morphline", "morphline.app.numRecords") > 0);
  }

View Full Code Here

    String inputPath = tmpDir.copyResourceFileName("test-documents/hello1.txt");
    String[] expected = new String[] {};
    String[] args = getInitialArgs(FAIL_FILE);
    args = ObjectArrays.concat(args, inputPath);    
    numExpectedFailedRecords = 1;
    PipelineResult pipelineResult = runIntoSolr(args, expected);
    Assert.assertTrue(pipelineResult.getStageResults().get(0).getCounterValue("morphline", "morphline.app.numRecords") > 0);
  }

View Full Code Here

    String[] expected = new String[] {};
    String[] args = getInitialArgs(THROW_EXCEPTION_FILE);
    args = ObjectArrays.concat(args, inputPath);    
    numExpectedExceptionRecords = 1;
    if (pipelineType != PipelineType.memory) {
      PipelineResult pipelineResult = runIntoSolr(args, expected);
      Assert.assertTrue(pipelineResult.getStageResults().get(0).getCounterValue("morphline", "morphline.app.numRecords") > 0);
    } else {
      try {
        runIntoSolr(args, expected);
        Assert.fail();
      } catch (MorphlineRuntimeException e) {

View Full Code Here

    Assert.assertEquals(1, res);
    Assert.assertNull(tool.pipelineResult);
  }
  
  private PipelineResult runIntoSolr(String[] args, String[] expected) throws Exception {
    PipelineResult pipelineResult = runPipeline(args);    
    if (!isDryRun) {
      List<Map<String, Object>> records;
      records = new ArrayList();
      commit();        
      QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort("text", SolrQuery.ORDER.asc));

View Full Code Here

    PCollection<Record> stagedLogs = read(source);


    getPipeline().write(stagedLogs,
        CrunchDatasets.asTarget(persistent), Target.WriteMode.APPEND);


    PipelineResult result = run();


    if (result.succeeded()) {
      // remove the source data partition from staging
      ready.deleteAll();
      return 0;
    } else {
      return 1;

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.crunch.PipelineResult$StageResult

$.WordCount

org.apache.crunch.examples.AverageBytesByIP

org.apache.crunch.examples.TotalBytesByIP

org.apache.crunch.examples.TotalWordCount

org.apache.crunch.examples.WordCount

org.apache.crunch.impl.dist.DistributedPipeline

org.apache.crunch.impl.mem.MemPipeline

org.apache.crunch.impl.mem.MemPipeline$MemExecution

org.apache.crunch.impl.mr.exec.MRExecutor

org.apache.crunch.impl.mr.MRPipeline

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.