Package org.apache.crunch.PipelineResult

Examples of org.apache.crunch.PipelineResult.StageResult


   
    PTable<IntWritable, Text> out = Mapreduce.map(two, TestMapper.class, IntWritable.class, Text.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
View Full Code Here


   
    PTable<Text, LongWritable> out = Mapreduce.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(19, sr.getCounters().findCounter("words", "where").getValue());
  }
View Full Code Here

   
    PTable<Text, LongWritable> out = Mapred.map(two, TestMapper.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
View Full Code Here

   
    PTable<Text, LongWritable> out = Mapred.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(108, sr.getCounters().findCounter("thou", "count").getValue());
  }
View Full Code Here

  public void testCountersEnabled() throws IOException {
    PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, tmpDir.getDefaultConfiguration()),
        WritableTypeFamily.getInstance());
   
    assertEquals(1, result.getStageResults().size());
    StageResult stageResult = result.getStageResults().get(0);

    String counterGroup = CrunchOutputs.class.getName();
    assertEquals(3, stageResult.getCounterNames().get(counterGroup).size());
    assertEquals(1l, stageResult.getCounterValue(counterGroup, "out1"));
    assertEquals(1l, stageResult.getCounterValue(counterGroup, "out2"));
    assertEquals(0l, stageResult.getCounterValue(counterGroup, "out3"));
  }
View Full Code Here

   
    PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, configuration),
        WritableTypeFamily.getInstance());
   
    assertEquals(1, result.getStageResults().size());
    StageResult stageResult = result.getStageResults().get(0);
   
    assertFalse(stageResult.getCounterNames().containsKey(CrunchOutputs.CRUNCH_OUTPUTS));
  }
View Full Code Here

      pipeline.write(collection, CrunchDatasets.asTarget(to), Target.WriteMode.APPEND);

      PipelineResult result = pipeline.done();

      StageResult sr = Iterables.getFirst(result.getStageResults(), null);
      if (sr != null && MAP_INPUT_RECORDS != null) {
        this.count = sr.getCounterValue(MAP_INPUT_RECORDS);
      }

      return result;

    } else {
View Full Code Here

    }
    int res = ToolRunner.run(config, tool, args);
    Assert.assertEquals(0, res);
    Assert.assertTrue(tool.pipelineResult.succeeded());     
    Assert.assertEquals(1, tool.pipelineResult.getStageResults().size());
    StageResult stageResult = tool.pipelineResult.getStageResults().get(0);
    Assert.assertEquals(numExpectedFailedRecords, stageResult.getCounterValue("morphline", "morphline.app.numFailedRecords"));
    Assert.assertEquals(numExpectedExceptionRecords, stageResult.getCounterValue("morphline", "morphline.app.numExceptionRecords"));
    return tool.pipelineResult;
  }
View Full Code Here

TOP

Related Classes of org.apache.crunch.PipelineResult.StageResult

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.