Examples of org.apache.hadoop.io.WritableComparable

org.apache.hadoop.io.WritableComparable
data private int counter; private long timestamp; public void write(DataOutput out) throws IOException { out.writeInt(counter); out.writeLong(timestamp); } public void readFields(DataInput in) throws IOException { counter = in.readInt(); timestamp = in.readLong(); } public int compareTo(MyWritableComparable w) { int thisValue = this.value; int thatValue = ((IntWritable)o).value; return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1)); } }

        try {
            objClass = Class.forName(className);
        } catch (ClassNotFoundException e) {
            throw new IOException("Could not find class " + className + ", while attempting to de-serialize it ", e);
        }
        WritableComparable writable = null;
        try {
            writable = (WritableComparable) objClass.newInstance();
        } catch (Exception e) {
            String msg = "Could create instance of class " + className
                    + ", while attempting to de-serialize it. (no default constructor ?)";
            throw new IOException(msg, e);
        }


        // read the fields of the object from DataInput
        writable.readFields(in);
        return writable;
    }

View Full Code Here

   * @throws Exception
   */
  @SuppressWarnings("unchecked")
  public void testNextingAtOffsetinHalfMapFile() throws Exception {
    Path p = writeMapFile(getName());
    WritableComparable midkey = getMidkey(p);
    // Now test reading from the top.
    HalfMapFileReader top = new HalfMapFileReader(this.fs, p.toString(),
      this.conf, Reference.Range.top, midkey, null);
    ImmutableBytesWritable val = new ImmutableBytesWritable();
    final String startRowPrefix = "zy";
    HStoreKey key = new HStoreKey(startRowPrefix);
    WritableComparable closest = top.getClosest(key, val);
    assertTrue(closest.toString().startsWith(startRowPrefix));
    int count = 0;
    for (boolean first = true; top.next(key, val); count++) {
      if (first) {
        assertTrue(key.toString().compareTo(startRowPrefix) > 0);
      }

View Full Code Here

   * using two HalfMapFiles.
   * @throws Exception
   */
  public void testBasicHalfMapFile() throws Exception {
    Path p = writeMapFile(getName());
    WritableComparable midkey = getMidkey(p);
    checkHalfMapFile(p, midkey);
  }

View Full Code Here

      new MapFile.Reader(this.fs, p.toString(), this.conf);
    HStoreKey key = new HStoreKey();
    ImmutableBytesWritable value = new ImmutableBytesWritable();
    reader.next(key, value);
    String firstKey = key.toString();
    WritableComparable midkey = reader.midKey();
    reader.finalKey(key);
    LOG.info("First key " + firstKey + ", midkey " + midkey.toString()
        + ", last key " + key.toString());
    reader.close();
    return midkey;
  }

View Full Code Here

      LOG.info("Last in top: " + key.toString());


      // Next test using a midkey that does not exist in the file.
      // First, do a key that is < than first key. Ensure splits behave
      // properly.
      WritableComparable badkey = new HStoreKey("   ");
      bottom = new HalfMapFileReader(this.fs, p.toString(),
          this.conf, Reference.Range.bottom, badkey, null);
      // When badkey is < than the bottom, should return no values.
      assertFalse(bottom.next(key, value));
      // Now read from the top.

View Full Code Here

  public void run(RecordReader input, OutputCollector output,
                  Reporter reporter)
    throws IOException {
    try {
      // allocate key & value instances that are re-used for all entries
      WritableComparable key = input.createKey();
      Writable value = input.createValue();
      
      while (input.next(key, value)) {
        // map pair to output
        mapper.map(key, value, output, reporter);

View Full Code Here

                  Reporter reporter)
    throws IOException {
    try {
      // allocate key & value instances these objects will not be reused
      // because execution of Mapper.map is not serialized.
      WritableComparable key = input.createKey();
      Writable value = input.createValue();


      while (input.next(key, value)) {


        // Run Mapper.map execution asynchronously in a separate thread.

View Full Code Here

    MapFile.Writer writer =
      new MapFile.Writer(conf, fs, out,
          reader.getKeyClass().asSubclass(WritableComparable.class),
          reader.getValueClass());


    WritableComparable key =
      ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf);
    Writable value =
      ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf);


    while (reader.next(key, value))               // copy all entries

View Full Code Here

      this.positions = new long[1024];


      try {
        int skip = INDEX_SKIP;
        LongWritable position = new LongWritable();
        WritableComparable lastKey = null;
        long lastIndex = -1;
        ArrayList<WritableComparable> keyBuilder = new ArrayList<WritableComparable>(1024);
        while (true) {
          WritableComparable k = comparator.newKey();


          if (!index.next(k, position))
            break;


          // check order to make sure comparator is compatible

View Full Code Here

      int low = 0;
      int high = count-1;


      while (low <= high) {
        int mid = (low + high) >>> 1;
        WritableComparable midVal = keys[mid];
        int cmp = comparator.compare(midVal, key);


        if (cmp < 0)
          low = mid + 1;
        else if (cmp > 0)

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.WritableComparable

com.cloudera.sqoop.mapreduce.TestImportJob

com.ebay.erl.mobius.core.datajoin.DataJoinKey

com.ebay.erl.mobius.core.datajoin.DataJoinKeyPartitioner

com.ebay.erl.mobius.core.mapred.MobiusInputSampler

com.ebay.erl.mobius.core.model.ReadFieldImpl

com.ebay.erl.mobius.core.model.WriteImpl

com.tdunning.plume.local.lazy.MSCRCombiner

com.thinkaurelius.faunus.mapreduce.transform.PropertyMap$Map

edu.isi.karma.mapreduce.driver.TestJSONIdentityMapReduce

edu.umd.hooka.CreateMetadata

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.