Package org.apache.crunch

Examples of org.apache.crunch.PipelineResult$StageResult

133
134
135
136
137
138
139
140
141
142
143
144
145
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PAPER")));
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PLUNGER")));
    expectedJoinResult.add(Pair.of(333, Pair.of("[Someone Else,Toilet brush]", "TOILET BRUSH")));
    Iterable<Pair<Integer, Pair<String, String>>> iter = joined.materialize();
   
    PipelineResult res = pipeline.run();
    if (!inMemory) {
      assertEquals(2, res.getStageResults().size());
    }
    
    List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(iter);
    Collections.sort(joinedResultList);
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
    }
  }

  @Override
  public PipelineResult run() {
    PipelineResult res = null;
    try {
      res = plan().execute();
    } catch (CrunchRuntimeException e) {
      LOG.error(e);
      return PipelineResult.EMPTY;
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
    return res;
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = null;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
        collection = CrunchDatasets.partition(collection, to, numWriters);
      }

      pipeline.write(collection, CrunchDatasets.asTarget(to), Target.WriteMode.APPEND);

      PipelineResult result = pipeline.done();

      StageResult sr = Iterables.getFirst(result.getStageResults(), null);
      if (sr != null && MAP_INPUT_RECORDS != null) {
        this.count = sr.getCounterValue(MAP_INPUT_RECORDS);
      }

      return result;
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
    if (numWriters >= 0) {
      task.setNumWriters(numWriters);
    }

    PipelineResult result = task.run();

    if (result.succeeded()) {
      console.info("Added {} records to \"{}\"",
          task.getCount(), datasets.get(1));
      return 0;
    } else {
      return 1;
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
    if (numWriters >= 0) {
      task.setNumWriters(numWriters);
    }

    PipelineResult result = task.run();

    if (result.succeeded()) {
      console.info("Added {} records to \"{}\"",
          task.getCount(), datasets.get(1));
      return 0;
    } else {
      return 1;
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
202
      if (numWriters >= 0) {
        task.setNumWriters(numWriters);
      }

      PipelineResult result = task.run();

      if (result.succeeded()) {
        long count = task.getCount();
        if (count > 0) {
          console.info("Added {} records to \"{}\"", count, dataset);
        }
        return 0;
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
        } else if (!failures.isEmpty()) {
          status.set(Status.FAILED);
        } else {
          status.set(Status.SUCCEEDED);
        }
        result = new PipelineResult(stages, status.get());
        set(result);
      }
    } catch (InterruptedException e) {
      throw new AssertionError(e); // Nobody should interrupt us.
    } catch (Exception e) {
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
  }

  @Override
  public PipelineResult run() {
    MSCRPlanner planner = new MSCRPlanner(this, outputTargets);
    PipelineResult res = null;
    try {
      res = planner.plan(jarClass, conf).execute();
    } catch (IOException e) {
      LOG.error(e);
      return PipelineResult.EMPTY;
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
    return res;
  }

  @Override
  public PipelineResult done() {
    PipelineResult res = null;
    if (!outputTargets.isEmpty()) {
      res = run();
    }
    cleanup();
    return res;
View Full Code Here
TOP

Related Classes of org.apache.crunch.PipelineResult$StageResult

  • $.WordCount
  • org.apache.crunch.examples.AverageBytesByIP
  • org.apache.crunch.examples.TotalBytesByIP
  • org.apache.crunch.examples.TotalWordCount
  • org.apache.crunch.examples.WordCount
  • org.apache.crunch.impl.dist.DistributedPipeline
  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.impl.mem.MemPipeline$MemExecution
  • org.apache.crunch.impl.mr.exec.MRExecutor
  • org.apache.crunch.impl.mr.MRPipeline
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.