Examples of com.google.appengine.tools.mapreduce.CorruptDataException

Package com.google.appengine.tools.mapreduce

Examples of com.google.appengine.tools.mapreduce.CorruptDataException

com.google.appengine.tools.mapreduce.CorruptDataException
The exception generated if any of the data appears to be corrupt. This should cause the MapReduce to fail.

  }


  @SuppressWarnings({"unchecked", "resource"})
  private static <T> T deserializeFromStream(InputStream in, final boolean ignoreHeader) {
    ObjectInputStream oin = null;
    CorruptDataException e = null;
    try {
      oin = new ConciseObjectInputStream(in, ignoreHeader);
      Object value = oin.readObject();
      if (value instanceof Flag) {
        CompressionType compression = ((Flag) value).getCompressionType();
        oin = compression.wrap(oin);
        value = oin.readObject();
      }
      return (T) value;
    } catch (IOException | ClassNotFoundException e1) {
      e = new CorruptDataException("Deserialization error: " + e1.getMessage(), e1);
      throw e;
    } finally {
      if (oin != null) {
        try {
          oin.close();

View Full Code Here

        Map<Key, Entity> shards = DATASTORE.get(keys);
        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        for (Key key : keys) {
          Entity shard = shards.get(key);
          if (shard == null) {
            throw new CorruptDataException("Missing data shard " + key);
          }
          byte[] shardBytes = ((Blob) shard.getProperty("content")).getBytes();
          bout.write(shardBytes, 0, shardBytes.length);
        }
        bytes = bout.toByteArray();

View Full Code Here

    int offset = 0;
    int length = buf.length;
    while (offset < buf.length) {
      int read = in.read(buf, offset, length);
      if (read < 0) {
        throw new CorruptDataException("Could not fill buffer up to requested size: " + buf.length
            + " was only able to read " + offset + " bytes.");
      }
      offset += read;
      length -= read;
    }

View Full Code Here

      while (record.type().equals(RecordType.MIDDLE)) {
        result.add(record.data());
        record = readPhysicalRecord(false);
      }
      if (!record.type().equals(RecordType.LAST)) {
        throw new CorruptDataException("Unterminated first block. Found: " + record.type.value());
      }
      result.add(record.data());
      return copyAll(result);
    }
    throw new CorruptDataException("Unexpected RecordType: " + record.type.value());
  }

View Full Code Here


  private void validateBufferIsZeros(ByteBuffer buffer) {
    for (int i = buffer.position(); i < buffer.limit(); i++) {
      byte b = buffer.get(i);
      if (b != 0) {
        throw new CorruptDataException("Found a non-zero byte: " + b
            + " before the end of the block " + i
            + " bytes after encountering a RecordType of NONE");
      }
    }
  }

View Full Code Here

    readToTmp(HEADER_LENGTH, expectEnd);


    int checksum = tmpBuffer.getInt();
    int length = tmpBuffer.getShort();
    if (length > bytesToBlockEnd || length < 0) {
      throw new CorruptDataException("Length is too large:" + length);
    }
    RecordType type = RecordType.get(tmpBuffer.get());
    if (type == RecordType.NONE && length == 0) {
      length = bytesToBlockEnd - HEADER_LENGTH;
    }
    readToTmp(length, false);


    if (!isValidCrc(checksum, tmpBuffer, type.value())) {
      throw new CorruptDataException("Checksum doesn't validate.");
    }
    return createRecordFromTmp(type);
  }

View Full Code Here

    int read = read(tmpBuffer);
    if (read == -1 && expectEnd) {
      throw new NoSuchElementException();
    }
    if (read != length) {
      throw new CorruptDataException("Premature end of file was expecting at least: "
          + length + " but found only: " + read);
    }
    offset += read;
    tmpBuffer.flip();
  }

View Full Code Here

  public KeyValue<K, Iterable<V>> fromBytes(ByteBuffer input) {
    FileServicePb.KeyValues proto;
    try {
      proto = FileServicePb.KeyValues.parseFrom(ByteString.copyFrom(input));
    } catch (InvalidProtocolBufferException e) {
      throw new CorruptDataException(e);
    }
    K key = keyMarshaller.fromBytes(proto.getKey().asReadOnlyByteBuffer());
    List<ByteString> values = proto.getValueList();
    return KeyValue.<K, Iterable<V>>of(key, new ByteStringTranslatingIterator(values));
  }

View Full Code Here

  public KeyValue<K, V> fromBytes(ByteBuffer input) {
    FileServicePb.KeyValues proto;
    try {
      proto = FileServicePb.KeyValues.parseFrom(ByteString.copyFrom(input));
    } catch (InvalidProtocolBufferException e) {
      throw new CorruptDataException(e);
    }
    K key = keyMarshaller.fromBytes(proto.getKey().asReadOnlyByteBuffer());
    V value = valueMarshaller.fromBytes(proto.getValue(0).asReadOnlyByteBuffer());
    return new KeyValue<>(key, value);
  }

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.mapreduce.CorruptDataException

com.google.appengine.tools.mapreduce.impl.KeyValueMarshaller

com.google.appengine.tools.mapreduce.impl.KeyValuesMarshaller

com.google.appengine.tools.mapreduce.impl.util.SerializationUtil

com.google.appengine.tools.mapreduce.inputs.LevelDbInputReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.