Package org.apache.crunch

Examples of org.apache.crunch.CrunchRuntimeException


      W instance = (W) WritableFactories.newInstance(clazz);
      BytesWritable bytes = (BytesWritable) writable;
      try {
        instance.readFields(new DataInputStream(new ByteArrayInputStream(bytes.getBytes())));
      } catch (IOException e) {
        throw new CrunchRuntimeException(e);
      }
      return instance;
    }
  }
View Full Code Here


    @Override
    public void configure(Configuration conf) {
      try {
        serializeWritableComparableCodes(conf);
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error serializing writable comparable codes", e);
      }

      for (MapFn fn : fns) {
        fn.configure(conf);
      }
View Full Code Here

    boolean exists = target.handleExisting(writeMode, ((PCollectionImpl) pcollection).getLastModifiedAt(),
        getConfiguration());
    if (exists && writeMode == Target.WriteMode.CHECKPOINT) {
      SourceTarget<?> st = target.asSourceTarget(pcollection.getPType());
      if (st == null) {
        throw new CrunchRuntimeException("Target " + target + " does not support checkpointing");
      } else {
        ((PCollectionImpl) pcollection).materializeAt(st);
      }
      return;
    } else if (writeMode != Target.WriteMode.APPEND && targetInCurrentRun(target)) {
      throw new CrunchRuntimeException("Target " + target + " is already written in current run." +
          " Use WriteMode.APPEND in order to write additional data to it.");
    }
    addOutput((PCollectionImpl<?>) pcollection, target);
  }
View Full Code Here

    @Override
    public void configure(Configuration conf) {
      try {
        serializeWritableComparableCodes(conf);
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error serializing writable comparable codes", e);
      }
      for (MapFn fn : fns) {
        fn.configure(conf);
      }
    }
View Full Code Here

  public void materialize() {
    try {
      materialized = source.read(pipeline.getConfiguration());
    } catch (IOException e) {
      LOG.error("Could not materialize: " + source, e);
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here

      binaryEncoder.flush();
      binaryDecoder = DecoderFactory.get()
          .binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
      return datumReader.read(target, binaryDecoder);
    } catch (Exception e) {
      throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
    }
  }
View Full Code Here

  protected T createNewInstance(Class<T> targetClass) {
    try {
      return targetClass.newInstance();
    } catch (InstantiationException e) {
      throw new CrunchRuntimeException(e);
    } catch (IllegalAccessException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here

  @Override
  public void write(PCollection<?> collection, Target target, Target.WriteMode writeMode) {
    target.handleExisting(writeMode, -1, getConfiguration());
    if (writeMode != Target.WriteMode.APPEND && activeTargets.contains(target)) {
      throw new CrunchRuntimeException("Target " + target
          + " is already written in the current run."
          + " Use WriteMode.APPEND in order to write additional data to it.");
    }
    activeTargets.add(target);
    if (target instanceof PathTarget) {
View Full Code Here

    FileSystem fs = null;
    try {
      fs = FileSystem.get(conf);
    } catch (IOException e) {
      LOG.error("Could not retrieve FileSystem object to check for existing path", e);
      throw new CrunchRuntimeException(e);
    }
   
    boolean exists = false;
    try {
      exists = fs.exists(path);
    } catch (IOException e) {
      LOG.error("Exception checking existence of path: " + path, e);
      throw new CrunchRuntimeException(e);
    }
   
    if (exists) {
      switch (strategy) {
      case DEFAULT:
        LOG.error("Path " + path + " already exists!");
        throw new CrunchRuntimeException("Path already exists: " + path);
      case OVERWRITE:
        LOG.info("Removing data at existing path: " + path);
        try {
          fs.delete(path, true);
        } catch (IOException e) {
          LOG.error("Exception thrown removing data at path: " + path, e);
        }
        break;
      case APPEND:
        LOG.info("Adding output files to existing path: " + path);
        break;
      default:
        throw new CrunchRuntimeException("Unknown WriteMode:  " + strategy);
      }
    } else {
      LOG.info("Will write output files to new path: " + path);
    }
  }
View Full Code Here

  @Override
  public void write(PCollection<?> collection, Target target,
      Target.WriteMode writeMode) {
    target.handleExisting(writeMode, getConfiguration());
    if (writeMode != WriteMode.APPEND && activeTargets.contains(target)) {
      throw new CrunchRuntimeException("Target " + target + " is already written in the current run." +
          " Use WriteMode.APPEND in order to write additional data to it.");
    }
    activeTargets.add(target);
    if (target instanceof PathTarget) {
      Path path = ((PathTarget) target).getPath();
View Full Code Here

TOP

Related Classes of org.apache.crunch.CrunchRuntimeException

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.