Package org.apache.crunch.materialize

Examples of org.apache.crunch.materialize.MaterializableIterable

64
65
66
67
68
69
70
71
72
73
74
75
76
          emitter.emit(s);
        }
      }
    }, strings());

    MaterializableIterable matIt = (MaterializableIterable)iso.materialize();
    ParallelDoOptions.Builder builder = ParallelDoOptions.builder().sourceTargets((SourceTarget)matIt.getSource());
    final String collectionPath = matIt.getPath().toString();

    PTable<Integer, String> splitMap = keyedLower.parallelDo("split-map",
        new MapFn<Pair<Integer, String>, Pair<Integer, String>>() {
      @Override
      public Pair<Integer, String> map(Pair<Integer, String> input) {
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
351
352
        unfinished.removeAll(targets);
      }
      if (status.get() == Status.RUNNING) {
        for (PCollectionImpl<?> output : pcolToRdd.keySet()) {
          if (toMaterialize.containsKey(output)) {
            MaterializableIterable mi = toMaterialize.get(output);
            if (mi.isSourceTarget()) {
              output.materializeAt((SourceTarget) mi.getSource());
            }
          }
          targetDeps.remove(output);
        }
      }
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
160
  private Set<Target> getDependencies(PipelineCallable<?> callable) {
    Set<Target> deps = Sets.newHashSet(callable.getAllTargets().values());
    for (PCollection pc : callable.getAllPCollections().values()) {
      PCollectionImpl pcImpl = (PCollectionImpl) pc;
      deps.addAll(pcImpl.getTargetDependencies());
      MaterializableIterable iter = (MaterializableIterable) pc.materialize();
      Source pcSrc = iter.getSource();
      if (pcSrc instanceof Target) {
        deps.add((Target) pcSrc);
      }
    }
    return deps;
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
      }

      if (!hasFailures) {
        for (PCollectionImpl<?> c : outputTargets.keySet()) {
          if (toMaterialize.containsKey(c)) {
            MaterializableIterable iter = toMaterialize.get(c);
            if (iter.isSourceTarget()) {
              iter.materialize();
              c.materializeAt((SourceTarget) iter.getSource());
            }
          } else {
            boolean materialized = false;
            for (Target t : outputTargets.get(c)) {
              if (!materialized) {
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
190
      // Remove completed outputs and mark materialized output locations
      // for subsequent job processing.
      for (PCollectionImpl<?> output : currentStage) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
      // Remove completed outputs and mark materialized output locations
      // for subsequent job processing.
      for (PCollectionImpl<?> output : currentStage) {
        if (toMaterialize.containsKey(output)) {
          MaterializableIterable mi = toMaterialize.get(output);
          if (mi.isSourceTarget()) {
            output.materializeAt((SourceTarget) mi.getSource());
          }
        }
        targetDeps.remove(output);
      }
    }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
        stages.add(new PipelineResult.StageResult(job.getJobName(), job.getMapredJobID().toString(), job.getCounters()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
      LOG.error(e);
      return PipelineResult.EMPTY;
    }
    for (PCollectionImpl<?> c : outputTargets.keySet()) {
      if (outputTargetsToMaterialize.containsKey(c)) {
        MaterializableIterable iter = outputTargetsToMaterialize.get(c);
        iter.materialize();
        c.materializeAt(iter.getSourceTarget());
        outputTargetsToMaterialize.remove(c);
      } else {
        boolean materialized = false;
        for (Target t : outputTargets.get(c)) {
          if (!materialized && t instanceof Source) {
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
168
      LOG.error(e);
      return PipelineResult.EMPTY;
    }
    for (PCollectionImpl<?> c : outputTargets.keySet()) {
      if (outputTargetsToMaterialize.containsKey(c)) {
        MaterializableIterable iter = outputTargetsToMaterialize.get(c);
        if (iter.isSourceTarget()) {
          iter.materialize();
          c.materializeAt((SourceTarget) iter.getSource());
        }
        outputTargetsToMaterialize.remove(c);
      } else {
        boolean materialized = false;
        for (Target t : outputTargets.get(c)) {
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
            job.getStartTimeMsec(), job.getJobStartTimeMsec(), job.getJobEndTimeMsec(), job.getEndTimeMsec()));
      }

      for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (toMaterialize.containsKey(c)) {
          MaterializableIterable iter = toMaterialize.get(c);
          if (iter.isSourceTarget()) {
            iter.materialize();
            c.materializeAt((SourceTarget) iter.getSource());
          }
        } else {
          boolean materialized = false;
          for (Target t : outputTargets.get(c)) {
            if (!materialized) {
View Full Code Here
TOP

Related Classes of org.apache.crunch.materialize.MaterializableIterable

  • org.apache.crunch.impl.dist.DistributedPipeline
  • org.apache.crunch.impl.mr.exec.MRExecutor
  • org.apache.crunch.impl.mr.MRPipeline
  • org.apache.crunch.impl.mr.plan.MSCRPlanner
  • org.apache.crunch.impl.spark.SparkRuntime
  • org.apache.crunch.LongPipelinePlannerIT
  • org.apache.crunch.impl.mr.run.CrunchRuntimeException
  • org.apache.crunch.CrunchRuntimeException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.