Package org.apache.crunch.PipelineResult

Examples of org.apache.crunch.PipelineResult.StageResult

92
93
94
95
96
97
98
99
   
    PTable<IntWritable, Text> out = Mapreduce.map(two, TestMapper.class, IntWritable.class, Text.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
View Full Code Here
112
113
114
115
116
117
118
119
   
    PTable<Text, LongWritable> out = Mapreduce.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(19, sr.getCounters().findCounter("words", "where").getValue());
  }
View Full Code Here
106
107
108
109
110
111
112
113
   
    PTable<Text, LongWritable> out = Mapred.map(two, TestMapper.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
View Full Code Here
126
127
128
129
130
131
132
133
   
    PTable<Text, LongWritable> out = Mapred.reduce(two.groupByKey(), TestReducer.class, Text.class, LongWritable.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(108, sr.getCounters().findCounter("thou", "count").getValue());
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
  public void testCountersEnabled() throws IOException {
    PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, tmpDir.getDefaultConfiguration()),
        WritableTypeFamily.getInstance());
   
    assertEquals(1, result.getStageResults().size());
    StageResult stageResult = result.getStageResults().get(0);

    String counterGroup = CrunchOutputs.class.getName();
    assertEquals(3, stageResult.getCounterNames().get(counterGroup).size());
    assertEquals(1l, stageResult.getCounterValue(counterGroup, "out1"));
    assertEquals(1l, stageResult.getCounterValue(counterGroup, "out2"));
    assertEquals(0l, stageResult.getCounterValue(counterGroup, "out3"));
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
   
    PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, configuration),
        WritableTypeFamily.getInstance());
   
    assertEquals(1, result.getStageResults().size());
    StageResult stageResult = result.getStageResults().get(0);
   
    assertFalse(stageResult.getCounterNames().containsKey(CrunchOutputs.CRUNCH_OUTPUTS));
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
      pipeline.write(collection, CrunchDatasets.asTarget(to), Target.WriteMode.APPEND);

      PipelineResult result = pipeline.done();

      StageResult sr = Iterables.getFirst(result.getStageResults(), null);
      if (sr != null && MAP_INPUT_RECORDS != null) {
        this.count = sr.getCounterValue(MAP_INPUT_RECORDS);
      }

      return result;

    } else {
View Full Code Here
386
387
388
389
390
391
392
393
394
395
    }
    int res = ToolRunner.run(config, tool, args);
    Assert.assertEquals(0, res);
    Assert.assertTrue(tool.pipelineResult.succeeded());     
    Assert.assertEquals(1, tool.pipelineResult.getStageResults().size());
    StageResult stageResult = tool.pipelineResult.getStageResults().get(0);
    Assert.assertEquals(numExpectedFailedRecords, stageResult.getCounterValue("morphline", "morphline.app.numFailedRecords"));
    Assert.assertEquals(numExpectedExceptionRecords, stageResult.getCounterValue("morphline", "morphline.app.numExceptionRecords"));
    return tool.pipelineResult;
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.PipelineResult.StageResult

  • org.apache.crunch.lib.MapredIT
  • org.apache.crunch.lib.MapreduceIT
  • org.apache.crunch.MultipleOutputIT
  • org.apache.solr.crunch.MemoryCrunchIndexerToolTest
  • org.kitesdk.tools.TransformTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.