Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.Writable


  public synchronized void finalKey(WritableComparable key)
  throws IOException {
    if (top) {
      super.finalKey(key);
    } else {
      Writable value = new ImmutableBytesWritable();
      WritableComparable found = super.getClosest(midkey, value, true);
      Writables.copyWritable(found, key);
    }
  }
View Full Code Here


  public String toString() {
    StringBuffer buf = new StringBuffer("[");
    for (int i = 0; i < values.length; ++i) {
      if (has(i)) {
        if (writableClasses != null) {
          Writable w = WritableFactories.newInstance(writableClasses.get(i));
          try {
            w.readFields(new DataInputStream(new ByteArrayInputStream(values[i].getBytes())));
          } catch (IOException e) {
            throw new CrunchRuntimeException(e);
          }
          buf.append(w.toString());
        } else {
          buf.append(values[i].toString());
        }
      }
      buf.append(",");
View Full Code Here

      if (has(i) && !o.has(i)) {
        return 1;
      } else if (!has(i) && o.has(i)) {
        return -1;
      } else {
        Writable v1 = values[i];
        Writable v2 = o.values[i];
        if (v1 != v2 && (v1 != null && !v1.equals(v2))) {
          if (v1 instanceof WritableComparable && v2 instanceof WritableComparable) {
            int cmp = ((WritableComparable) v1).compareTo((WritableComparable) v2);
            if (cmp != 0) {
              return cmp;
            }
          } else {
            int cmp = v1.hashCode() - v2.hashCode();
            if (cmp != 0) {
              return cmp;
            }
          }
        }
View Full Code Here

    @Override
    public Tuple map(TupleWritable in) {
      for (int i = 0; i < values.length; i++) {
        if (in.has(i)) {
          Writable w = create(writableClasses.get(i), in.get(i));
          values[i] = fns.get(i).map(w);
        } else {
          values[i] = null;
        }
      }
View Full Code Here

      writable.clearWritten();
      for (int i = 0; i < input.size(); i++) {
        Object value = input.get(i);
        if (value != null) {
          writable.setWritten(i);
          Writable w = (Writable) fns.get(i).map(value);
          values[i] = new BytesWritable(WritableUtils.toByteArray(w));
        }
      }
      return writable;
    }
View Full Code Here

    @Override
    public Collection<T> map(GenericArrayWritable input) {
      Collection<T> collection = Lists.newArrayList();
      for (BytesWritable raw : input.get()) {
        Writable w = create(clazz, raw);
        collection.add(mapFn.map(w));
      }
      return collection;
    }
View Full Code Here

    public GenericArrayWritable map(Collection<T> input) {
      GenericArrayWritable arrayWritable = new GenericArrayWritable();
      BytesWritable[] w = new BytesWritable[input.size()];
      int index = 0;
      for (T in : input) {
        Writable v = (Writable) mapFn.map(in);
        w[index++] = new BytesWritable(WritableUtils.toByteArray(v));
      }
      arrayWritable.set(w);
      return arrayWritable;
    }
View Full Code Here

    @Override
    public Map<String, T> map(TextMapWritable input) {
      Map<String, T> out = Maps.newHashMap();
      for (Map.Entry<Text, BytesWritable> e : input.entrySet()) {
        Writable v = create(clazz, e.getValue());
        out.put(e.getKey().toString(), mapFn.map(v));
      }
      return out;
    }
View Full Code Here

    @Override
    public TextMapWritable map(Map<String, T> input) {
      TextMapWritable tmw = new TextMapWritable();
      for (Map.Entry<String, T> e : input.entrySet()) {
        Writable w = mapFn.map(e.getValue());
        tmw.put(new Text(e.getKey()), new BytesWritable(WritableUtils.toByteArray(w)));
      }
      return tmw;
    }
View Full Code Here

          numEdits++;
          switch (opcode) {
          case OP_ADD: {
            UTF8 name = new UTF8();
            ArrayWritable aw = null;
            Writable writables[];
            // version 0 does not support per file replication
            if (logVersion >= 0)
              name.readFields(in)// read name only
            else // other versions do
              // get name and replication
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get();
              if (-4 <= logVersion && writables.length != 2 ||
                  -7 <= logVersion && logVersion < -4 && writables.length != 3||
                  logVersion < -7 && writables.length != 4) {
                  throw new IOException("Incorrect data format."  +
                                        " logVersion is " + logVersion +
                                        " but writables.length is " +
                                        writables.length + ". ");
              }
              name = (UTF8) writables[0];
              replication = Short.parseShort(
                                             ((UTF8)writables[1]).toString());
              replication = adjustReplication(replication);
              if (logVersion < -4) {
                mtime = Long.parseLong(((UTF8)writables[2]).toString());
              }
              if (logVersion < -7) {
                blockSize = Long.parseLong(((UTF8)writables[3]).toString());
              }
            }
            // get blocks
            aw = new ArrayWritable(Block.class);
            aw.readFields(in);
            writables = aw.get();
            Block blocks[] = new Block[writables.length];
            System.arraycopy(writables, 0, blocks, 0, blocks.length);

            // Older versions of HDFS does not store the block size in inode.
            // If the file has more than one block, use the size of the
            // first block as the blocksize. Otherwise use the default
            // block size.
            if (-8 <= logVersion && blockSize == 0) {
              if (blocks.length > 1) {
                blockSize = blocks[0].getNumBytes();
              } else {
                long first = ((blocks.length == 1)? blocks[0].getNumBytes(): 0);
                blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
              }
            }

            // add to the file tree
            fsDir.unprotectedAddFile(name.toString(), blocks, replication,
                                     mtime, blockSize);
            break;
          }
          case OP_SET_REPLICATION: {
            UTF8 src = new UTF8();
            UTF8 repl = new UTF8();
            src.readFields(in);
            repl.readFields(in);
            replication = adjustReplication(fromLogReplication(repl));
            fsDir.unprotectedSetReplication(src.toString(),
                                            replication,
                                            null);
            break;
          }
          case OP_RENAME: {
            UTF8 src = null;
            UTF8 dst = null;
            if (logVersion >= -4) {
              src = new UTF8();
              dst = new UTF8();
              src.readFields(in);
              dst.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get();
              if (writables.length != 3) {
                throw new IOException("Incorrect data format. "
                                      + "Mkdir operation.");
              }
              src = (UTF8) writables[0];
              dst = (UTF8) writables[1];
              timestamp = Long.parseLong(((UTF8)writables[2]).toString());
            }
            fsDir.unprotectedRenameTo(src.toString(), dst.toString(), timestamp);
            break;
          }
          case OP_DELETE: {
            UTF8 src = null;
            if (logVersion >= -4) {
              src = new UTF8();
              src.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get();
              if (writables.length != 2) {
                throw new IOException("Incorrect data format. "
                                      + "delete operation.");
              }
              src = (UTF8) writables[0];
              timestamp = Long.parseLong(((UTF8)writables[1]).toString());
            }
            fsDir.unprotectedDelete(src.toString(), timestamp);
            break;
          }
          case OP_MKDIR: {
            UTF8 src = null;
            if (logVersion >= -4) {
              src = new UTF8();
              src.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get();
              if (writables.length != 2) {
                throw new IOException("Incorrect data format. "
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.Writable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.