Package org.apache.crunch

Examples of org.apache.crunch.PipelineResult$StageResult

339
340
341
342
343
344
345
346
347
  private static class MemExecution extends AbstractFuture<PipelineResult> implements PipelineExecution {

    private PipelineResult res;

    public MemExecution() {
      this.res = new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)),
          PipelineExecution.Status.SUCCEEDED);
    }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PAPER")));
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PLUNGER")));
    expectedJoinResult.add(Pair.of(333, Pair.of("[Someone Else,Toilet brush]", "TOILET BRUSH")));
    Iterable<Pair<Integer, Pair<String, String>>> iter = joined.materialize();
   
    PipelineResult res = pipeline.run();
    if (!inMemory) {
      assertEquals(materialize ? 2 : 1, res.getStageResults().size());
    }
    
    List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(iter);
    Collections.sort(joinedResultList);
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PLUNGER")));
    expectedJoinResult.add(Pair.of(333, Pair.of("[Someone Else,Toilet brush]", "TOILET BRUSH")));
    expectedJoinResult.add(Pair.of(444, Pair.<String,String>of("[Has No Orders,null]", null)));
    Iterable<Pair<Integer, Pair<String, String>>> iter = joined.materialize();
   
    PipelineResult res = pipeline.run();
    if (!inMemory) {
      assertEquals(materialize ? 2 : 1, res.getStageResults().size());
    }
    
    List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(iter);
    Collections.sort(joinedResultList);
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
    }
  }

  @Override
  public PipelineResult run() {
    PipelineResult res = null;
    try {
      res = plan().execute();
    } catch (CrunchRuntimeException e) {
      LOG.error(e);
      return PipelineResult.EMPTY;
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
    return res;
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = null;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;
View Full Code Here
69
70
71
72
73
74
75
        stages.add(new PipelineResult.StageResult(job.getJobName(), job.getJob().getCounters()));
      } catch (Exception e) {
        LOG.error("Exception thrown fetching job counters for stage: " + job.getJobName(), e);
      }
    }
    return new PipelineResult(stages);
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
        Writables.tableOf(Writables.strings(), Writables.doubles()));

    // write the result to a text file
    pipeline.writeTextFile(avgs, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
        .parallelDo(extractIPResponseSize, Writables.tableOf(Writables.strings(), Writables.longs())).groupByKey()
        .combineValues(agg);

    pipeline.writeTextFile(ipAddrResponseSize, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
    PTable<String, Long> counts = words.count();

    // Instruct the pipeline to write the resulting counts to a text file.
    pipeline.writeTextFile(counts, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
        // Instruct the pipeline to write the resulting counts to a text file.
        pipeline.writeTextFile(counts, outputPath);

        // Execute the pipeline as a MapReduce.
        PipelineResult result = pipeline.done();

        return result.succeeded() ? 0 : 1;
    }
View Full Code Here
TOP

Related Classes of org.apache.crunch.PipelineResult$StageResult

  • $.WordCount
  • org.apache.crunch.examples.AverageBytesByIP
  • org.apache.crunch.examples.TotalBytesByIP
  • org.apache.crunch.examples.TotalWordCount
  • org.apache.crunch.examples.WordCount
  • org.apache.crunch.impl.dist.DistributedPipeline
  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.impl.mem.MemPipeline$MemExecution
  • org.apache.crunch.impl.mr.exec.MRExecutor
  • org.apache.crunch.impl.mr.MRPipeline
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.