Package org.apache.crunch.materialize

Examples of org.apache.crunch.materialize.MaterializableIterable

268
269
270
271
272
273
274
275
276
277
278
279
280
          }
        }
      }
      for (PCollectionImpl<?> output : pcolToRdd.keySet()) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
138
139
        stages.add(new PipelineResult.StageResult(job.getJobName(), job.getMapredJobID().toString(), job.getCounters()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
      // Remove completed outputs and mark materialized output locations
      // for subsequent job processing.
      for (PCollectionImpl<?> output : currentStage) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
      }

      if (!hasFailures) {
        for (PCollectionImpl<?> c : outputTargets.keySet()) {
          if (toMaterialize.containsKey(c)) {
            MaterializableIterable iter = toMaterialize.get(c);
            if (iter.isSourceTarget()) {
              iter.materialize();
              c.materializeAt((SourceTarget) iter.getSource());
            }
          } else {
            boolean materialized = false;
            for (Target t : outputTargets.get(c)) {
              if (!materialized) {
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
354
355
        unfinished.removeAll(targets);
      }
      if (status.get() == Status.RUNNING) {
        for (PCollectionImpl<?> output : pcolToRdd.keySet()) {
          if (toMaterialize.containsKey(output)) {
            MaterializableIterable mi = toMaterialize.get(output);
            if (mi.isSourceTarget()) {
              output.materializeAt((SourceTarget) mi.getSource());
            }
          }
          targetDeps.remove(output);
        }
      }
View Full Code Here
272
273
274
275
276
277
278
279
280
281
282
283
284
          }
        }
      }
      for (PCollectionImpl<?> output : pcolToRdd.keySet()) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        stages.add(new PipelineResult.StageResult(job.getJobName(), job.getJob().getCounters()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
      // Remove completed outputs and mark materialized output locations
      // for subsequent job processing.
      for (PCollectionImpl<?> output : currentStage) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
            job.getStartTimeMsec(), job.getJobStartTimeMsec(), job.getJobEndTimeMsec(), job.getEndTimeMsec()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        stages.add(new PipelineResult.StageResult(job.getJobName(), job.getJob().getCounters()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
TOP

Related Classes of org.apache.crunch.materialize.MaterializableIterable

  • org.apache.crunch.impl.dist.DistributedPipeline
  • org.apache.crunch.impl.mr.exec.MRExecutor
  • org.apache.crunch.impl.mr.MRPipeline
  • org.apache.crunch.impl.mr.plan.MSCRPlanner
  • org.apache.crunch.impl.spark.SparkRuntime
  • org.apache.crunch.LongPipelinePlannerIT
  • org.apache.crunch.impl.mr.run.CrunchRuntimeException
  • org.apache.crunch.CrunchRuntimeException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.