Package org.apache.crunch.impl.mr.run

Examples of org.apache.crunch.impl.mr.run.CrunchRuntimeException

193
194
195
196
197
198
199
200
201
202
  protected T createNewInstance(Class<T> targetClass) {
    try {
      return targetClass.newInstance();
    } catch (InstantiationException e) {
      throw new CrunchRuntimeException(e);
    } catch (IllegalAccessException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
80
81
82
83
84
85
86
87
    job.setOutputFormatClass(TableOutputFormat.class);
    conf.set(TableOutputFormat.OUTPUT_TABLE, table);
    try {
      TableMapReduceUtil.addDependencyJars(job);
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
183
184
185
186
187
188
189
190
191
192
      datumWriter.write(source, binaryEncoder);
      binaryEncoder.flush();
      binaryDecoder = DecoderFactory.get().binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
      datumReader.read(target, binaryDecoder);
    } catch (Exception e) {
      throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
    }

    return target;
  }
View Full Code Here
193
194
195
196
197
198
199
200
201
202
  protected T createNewInstance(Class<T> targetClass) {
    try {
      return targetClass.newInstance();
    } catch (InstantiationException e) {
      throw new CrunchRuntimeException(e);
    } catch (IllegalAccessException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
   * @return A table keyed on the join key, containing pairs of joined values
   */
  public static <K, U, V> PTable<K, Pair<U, V>> join(PTable<K, U> left, PTable<K, V> right) {

    if (!(right.getPipeline() instanceof MRPipeline)) {
      throw new CrunchRuntimeException("Map-side join is only supported within a MapReduce context");
    }

    MRPipeline pipeline = (MRPipeline) right.getPipeline();
    pipeline.materialize(right);

    // TODO Move necessary logic to MRPipeline so that we can theoretically
    // optimize his by running the setup of multiple map-side joins concurrently
    pipeline.run();

    ReadableSourceTarget<Pair<K, V>> readableSourceTarget = pipeline.getMaterializeSourceTarget(right);
    if (!(readableSourceTarget instanceof SourcePathTargetImpl)) {
      throw new CrunchRuntimeException("Right-side contents can't be read from a path");
    }

    // Suppress warnings because we've just checked this cast via instanceof
    @SuppressWarnings("unchecked")
    SourcePathTargetImpl<Pair<K, V>> sourcePathTarget = (SourcePathTargetImpl<Pair<K, V>>) readableSourceTarget;
View Full Code Here
39
40
41
42
43
44
45
46
47
48
  @Override
  public void emit(T emitted) {
    try {
      this.outputs.write(outputName, converter.outputKey(emitted), converter.outputValue(emitted));
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    } catch (InterruptedException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  protected void setValueType(String valueType) {
    if (valueClass == null) {
      try {
        valueClass = Class.forName(valueType).asSubclass(Writable.class);
      } catch (ClassNotFoundException e) {
        throw new CrunchRuntimeException(e);
      }
    } else if (!valueType.equals(valueClass.getName())) {
      throw new IllegalStateException("Incoming " + valueType + " is not " + valueClass);
    }
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
            return localPath.makeQualified(FileSystem.getLocal(getConfiguration()));

          }
        }
      } catch (IOException e) {
        throw new CrunchRuntimeException(e);
      }

      throw new CrunchRuntimeException("Can't find local cache file for '" + inputPath + "'");
    }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
          .getDefaultFileSource(getCacheFilePath());
      Iterable<Pair<K, V>> iterable = null;
      try {
        iterable = sourceTarget.read(getConfiguration());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error reading right-side of map side join: ", e);
      }

      joinMap = ArrayListMultimap.create();
      for (Pair<K, V> joinPair : iterable) {
        joinMap.put(joinPair.first(), joinPair.second());
View Full Code Here
38
39
40
41
42
43
44
45
46
47
    try {
      K key = converter.outputKey(emitted);
      V value = converter.outputValue(emitted);
      this.context.write(key, value);
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    } catch (InterruptedException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.mr.run.CrunchRuntimeException

  • org.apache.crunch.impl.mr.emit.MultipleOutputEmitter
  • org.apache.crunch.impl.mr.emit.OutputEmitter
  • org.apache.crunch.impl.mr.MRPipeline
  • org.apache.crunch.io.hbase.HBaseTarget
  • org.apache.crunch.lib.join.MapsideJoin
  • org.apache.crunch.lib.join.MapsideJoin$MapsideJoinDoFn
  • org.apache.crunch.materialize.MaterializableIterable
  • org.apache.crunch.types.avro.AvroDeepCopier
  • org.apache.crunch.types.Protos
  • org.apache.crunch.types.PTypes$JacksonInputMapFn
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.