Package org.apache.crunch

Examples of org.apache.crunch.PipelineResult$StageResult

105
106
107
108
109
110
111
112
113
114
115
    this.tempDirectory = createTempDirectory(conf);
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = PipelineResult.DONE;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
        } else if (!failures.isEmpty()) {
          status.set(Status.FAILED);
        } else {
          status.set(Status.SUCCEEDED);
        }
        result = new PipelineResult(stages, status.get());
        set(result);
      }
    } catch (InterruptedException e) {
      throw new AssertionError(e); // Nobody should interrupt us.
    } catch (Exception e) {
View Full Code Here
363
364
365
366
367
368
369
370
371
  private static class MemExecution extends AbstractFuture<PipelineResult> implements PipelineExecution {

    private PipelineResult res;

    public MemExecution() {
      this.res = new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)),
          PipelineExecution.Status.SUCCEEDED);
    }
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
        return Status.SUCCEEDED;
      }

      @Override
      public PipelineResult getResult() {
        return new PipelineResult(ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)));
      }

      @Override
      public void kill() {
      }
View Full Code Here
253
254
255
256
257
258
259
  }
 
  @Override
  public PipelineResult run() {
    activeTargets.clear();
    return new PipelineResult(ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)));
  }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
    return res;
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = null;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
          }
        }
      }

      synchronized (this) {
        result = new PipelineResult(stages);
        if (killSignal.getCount() == 0) {
          status.set(Status.KILLED);
        } else {
          status.set(result.succeeded() ? Status.SUCCEEDED : Status.FAILED);
        }
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
      }
      runCallables(unfinished);
    }
    if (status.get() != Status.FAILED || status.get() != Status.KILLED) {
      status.set(Status.SUCCEEDED);
      set(new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("Spark", getCounters(),
              start, System.currentTimeMillis())),
          Status.SUCCEEDED));
    } else {
      set(PipelineResult.EMPTY);
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
    return runtime;
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = super.done();
    if (sparkContext != null) {
      sparkContext.stop();
      sparkContext = null;
    }
    return res;
View Full Code Here
78
79
80
81
82
83
84
85
86
        Writables.tableOf(Writables.strings(), Writables.doubles()));

    // write the result to a text file
    pipeline.writeTextFile(avgs, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.PipelineResult$StageResult

  • $.WordCount
  • org.apache.crunch.examples.AverageBytesByIP
  • org.apache.crunch.examples.TotalBytesByIP
  • org.apache.crunch.examples.TotalWordCount
  • org.apache.crunch.examples.WordCount
  • org.apache.crunch.impl.dist.DistributedPipeline
  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.impl.mem.MemPipeline$MemExecution
  • org.apache.crunch.impl.mr.exec.MRExecutor
  • org.apache.crunch.impl.mr.MRPipeline
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.