Package org.apache.crunch

Examples of org.apache.crunch.CrunchRuntimeException

83
84
85
86
87
88
89
90
        // Hacking this for Hadoop1 and Hadoop2
        getConfiguration().set("mapreduce.job.cache.local.files", sparkFiles);
        getConfiguration().set("mapred.cache.localFiles", sparkFiles);
      }
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
        if (v1 != v2 && (v1 != null && !v1.equals(v2))) {
          try {
            w1[index].readFields(new DataInputStream(new ByteArrayInputStream(v1.getBytes())));
            w2[index].readFields(new DataInputStream(new ByteArrayInputStream(v2.getBytes())));
          } catch (IOException e) {
            throw new CrunchRuntimeException(e);
          }
          if (w1[index] instanceof WritableComparable && w2[index] instanceof WritableComparable) {
            int cmp = ((WritableComparable) w1[index]).compareTo((WritableComparable) w2[index]);
            if (cmp != 0) {
              return order * cmp;
 
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
        try {
          Class cls = Class.forName(className);
          w1[i] = WritableFactories.newInstance(cls);
          w2[i] = WritableFactories.newInstance(cls);
        } catch (Exception e) {
          throw new CrunchRuntimeException(e);
        }
        Order order = Order.valueOf(split[1]);
        columnOrders[i] = ColumnOrder.by(i + 1, order);
      }
    }
View Full Code Here
189
190
191
192
193
194
195
196
      binaryEncoder.flush();
      binaryDecoder = DecoderFactory.get()
          .binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
      return datumReader.read(target, binaryDecoder);
    } catch (Exception e) {
      throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
    }
  }
View Full Code Here
197
198
199
200
201
202
203
204
205
206
  protected T createNewInstance(Class<T> targetClass) {
    try {
      return targetClass.newInstance();
    } catch (InstantiationException e) {
      throw new CrunchRuntimeException(e);
    } catch (IllegalAccessException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    boolean exists = target.handleExisting(writeMode, ((PCollectionImpl) pcollection).getLastModifiedAt(),
        getConfiguration());
    if (exists && writeMode == Target.WriteMode.CHECKPOINT) {
      SourceTarget<?> st = target.asSourceTarget(pcollection.getPType());
      if (st == null) {
        throw new CrunchRuntimeException("Target " + target + " does not support checkpointing");
      } else {
        ((PCollectionImpl) pcollection).materializeAt(st);
      }
      return;
    } else if (writeMode != Target.WriteMode.APPEND && targetInCurrentRun(target)) {
      throw new CrunchRuntimeException("Target " + target + " is already written in current run." +
          " Use WriteMode.APPEND in order to write additional data to it.");
    }
    addOutput((PCollectionImpl<?>) pcollection, target);
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
        if (writableClasses != null) {
          Writable w = WritableFactories.newInstance(writableClasses.get(i));
          try {
            w.readFields(new DataInputStream(new ByteArrayInputStream(values[i].getBytes())));
          } catch (IOException e) {
            throw new CrunchRuntimeException(e);
          }
          buf.append(w.toString());
        } else {
          buf.append(values[i].toString());
        }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
      try {
        Class<?> keyClass = Class.forName(fields.get(2));
        Class<?> valueClass = Class.forName(fields.get(3));
        out.put(name, new OutputConfig(bundle, keyClass, valueClass));
      } catch (ClassNotFoundException e) {
        throw new CrunchRuntimeException(e);
      }
    }
    return out;
  }
View Full Code Here
282
283
284
285
286
287
288
289
290
  private static <W extends Writable> W create(Class<W> clazz, BytesWritable bytes) {
    W instance = (W) WritableFactories.newInstance(clazz);
    try {
      instance.readFields(new DataInputStream(new ByteArrayInputStream(bytes.getBytes())));
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    }
    return instance;
  }
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
  @Override
  public void write(PCollection<?> collection, Target target, Target.WriteMode writeMode) {
    target.handleExisting(writeMode, -1, getConfiguration());
    if (writeMode != Target.WriteMode.APPEND && activeTargets.contains(target)) {
      throw new CrunchRuntimeException("Target " + target
          + " is already written in the current run."
          + " Use WriteMode.APPEND in order to write additional data to it.");
    }
    activeTargets.add(target);
    if (target instanceof PathTarget) {
View Full Code Here
TOP

Related Classes of org.apache.crunch.CrunchRuntimeException

  • com.twitter.elephantbird.crunch.CrunchElephantBirdExample$PersonInAddressBookFn
  • org.apache.crunch.fn.SDoubleFlatMapFunction
  • org.apache.crunch.fn.SDoubleFunction
  • org.apache.crunch.fn.SFlatMapFunction
  • org.apache.crunch.fn.SFlatMapFunction2
  • org.apache.crunch.fn.SFunction
  • org.apache.crunch.fn.SFunction2
  • org.apache.crunch.fn.SPairFlatMapFunction
  • org.apache.crunch.fn.SPairFunction
  • org.apache.crunch.impl.dist.collect.PCollectionImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.