Examples of org.apache.hadoop.io.WritableComparable

org.apache.hadoop.io.WritableComparable
data private int counter; private long timestamp; public void write(DataOutput out) throws IOException { out.writeInt(counter); out.writeLong(timestamp); } public void readFields(DataInput in) throws IOException { counter = in.readInt(); timestamp = in.readLong(); } public int compareTo(MyWritableComparable w) { int thisValue = this.value; int thatValue = ((IntWritable)o).value; return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1)); } }

    @SuppressWarnings("unchecked")
    @Override
    public synchronized WritableComparable getClosest(WritableComparable key,
      Writable val)
    throws IOException {
      WritableComparable closest = null;
      if (top) {
        // If top, the lowest possible key is midkey.  Do not have to check
        // what comes back from super getClosest.  Will return exact match or
        // greater.
        closest = (key.compareTo(this.midkey) < 0)?
          this.midkey: super.getClosest(key, val);
      } else {
        // We're serving bottom of the file.
        if (key.compareTo(this.midkey) < 0) {
          // Check key is within range for bottom.
          closest = super.getClosest(key, val);
          // midkey was made against largest store file at time of split. Smaller
          // store files could have anything in them.  Check return value is
          // not beyond the midkey (getClosest returns exact match or next
          // after).
          if (closest != null && closest.compareTo(this.midkey) >= 0) {
            // Don't let this value out.
            closest = null;
          }
        }
        // Else, key is > midkey so let out closest = null.

View Full Code Here

      if (firstNextCall) {
        firstNextCall = false;
        if (this.top) {
          // Seek to midkey.  Midkey may not exist in this file.  That should be
          // fine.  Then we'll either be positioned at end or start of file.
          WritableComparable nearest = getClosest(midkey, val);
          // Now copy the mid key into the passed key.
          if (nearest != null) {
            Writables.copyWritable(nearest, key);
            return true;
          }

View Full Code Here

        try {
            objClass = Class.forName(className);
        } catch (ClassNotFoundException e) {
            throw new IOException("Could not find class " + className + ", while attempting to de-serialize it ", e);
        }
        WritableComparable writable = null;
        try {
            writable = (WritableComparable) objClass.newInstance();
        } catch (Exception e) {
            String msg = "Could create instance of class " + className
                    + ", while attempting to de-serialize it. (no default constructor ?)";
            throw new IOException(msg, e);
        }


        // read the fields of the object from DataInput
        writable.readFields(in);
        return writable;
    }

View Full Code Here

  public void run(RecordReader input, OutputCollector output,
                  Reporter reporter)
    throws IOException {
    try {
      // allocate key & value instances that are re-used for all entries
      WritableComparable key = input.createKey();
      Writable value = input.createValue();
      
      while (input.next(key, value)) {
        // map pair to output
        mapper.map(key, value, output, reporter);

View Full Code Here

        continue;
      }


      // Need to use new configuration object so that it has the proper classloaders.
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, new Configuration());
      WritableComparable key = (WritableComparable)
          reader.getKeyClass().newInstance();
      Writable value = (Writable) reader.getValueClass().newInstance();
      while (reader.next(key, value)) {
        strings.add(value.toString());
      }

View Full Code Here

      if (top) {
        super.finalKey(key); 
      } else {
        reset();
        Writable value = new ImmutableBytesWritable();
        WritableComparable k = super.getClosest(midkey, value, true);
        ByteArrayOutputStream byteout = new ByteArrayOutputStream();
        DataOutputStream out = new DataOutputStream(byteout);
        k.write(out);
        ByteArrayInputStream bytein =
          new ByteArrayInputStream(byteout.toByteArray());
        DataInputStream in = new DataInputStream(bytein);
        key.readFields(in);
      }

View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public synchronized WritableComparable getClosest(WritableComparable key,
      Writable val)
    throws IOException {
      WritableComparable closest = null;
      if (top) {
        // If top, the lowest possible key is midkey.  Do not have to check
        // what comes back from super getClosest.  Will return exact match or
        // greater.
        closest = (key.compareTo(this.midkey) < 0)?
          this.midkey: super.getClosest(key, val);
      } else {
        // We're serving bottom of the file.
        if (key.compareTo(this.midkey) < 0) {
          // Check key is within range for bottom.
          closest = super.getClosest(key, val);
          // midkey was made against largest store file at time of split. Smaller
          // store files could have anything in them.  Check return value is
          // not beyond the midkey (getClosest returns exact match or next
          // after).
          if (closest != null && closest.compareTo(this.midkey) >= 0) {
            // Don't let this value out.
            closest = null;
          }
        }
        // Else, key is > midkey so let out closest = null.

View Full Code Here

      if (firstNextCall) {
        firstNextCall = false;
        if (this.top) {
          // Seek to midkey.  Midkey may not exist in this file.  That should be
          // fine.  Then we'll either be positioned at end or start of file.
          WritableComparable nearest = getClosest(midkey, val);
          // Now copy the mid key into the passed key.
          if (nearest != null) {
            Writables.copyWritable(nearest, key);
            return true;
          }

View Full Code Here

   * using two HalfMapFiles.
   * @throws Exception
   */
  public void testBasicHalfMapFile() throws Exception {
    Path p = writeMapFile(getName());
    WritableComparable midkey = getMidkey(p);
    checkHalfMapFile(p, midkey);
  }

View Full Code Here

      new MapFile.Reader(this.fs, p.toString(), this.conf);
    HStoreKey key = new HStoreKey();
    ImmutableBytesWritable value = new ImmutableBytesWritable();
    reader.next(key, value);
    String firstKey = key.toString();
    WritableComparable midkey = reader.midKey();
    reader.finalKey(key);
    LOG.info("First key " + firstKey + ", midkey " + midkey.toString()
        + ", last key " + key.toString());
    reader.close();
    return midkey;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.WritableComparable

com.cloudera.sqoop.mapreduce.TestImportJob

com.ebay.erl.mobius.core.datajoin.DataJoinKey

com.ebay.erl.mobius.core.datajoin.DataJoinKeyPartitioner

com.ebay.erl.mobius.core.mapred.MobiusInputSampler

com.ebay.erl.mobius.core.model.ReadFieldImpl

com.ebay.erl.mobius.core.model.WriteImpl

com.tdunning.plume.local.lazy.MSCRCombiner

com.thinkaurelius.faunus.mapreduce.transform.PropertyMap$Map

edu.isi.karma.mapreduce.driver.TestJSONIdentityMapReduce

edu.umd.hooka.CreateMetadata

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.