Examples of org.apache.hadoop.io.WritableComparable

org.apache.hadoop.io.WritableComparable
data private int counter; private long timestamp; public void write(DataOutput out) throws IOException { out.writeInt(counter); out.writeLong(timestamp); } public void readFields(DataInput in) throws IOException { counter = in.readInt(); timestamp = in.readLong(); } public int compareTo(MyWritableComparable w) { int thisValue = this.value; int thatValue = ((IntWritable)o).value; return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1)); } }

      MapWritable meta = datum.getMetaData();
      if (meta.size() > 0) {
        MapWritable newMeta = new MapWritable();
        Iterator it = meta.keySet().iterator();
        while (it.hasNext()) {
          WritableComparable k = (WritableComparable)it.next();
          Writable v = meta.get(k);
          if (k instanceof UTF8) {
            Text t = new Text(k.toString());
            k = t;
          }
          newMeta.put(k, v);
        }
        datum.setMetaData(newMeta);

View Full Code Here

    final Hit[] hitArr = new Hit[docList.size()];
    for (int i = 0; i < hitArr.length; i++) {
      final SolrDocument solrDoc = docList.get(i);


      final Object raw = solrDoc.getFirstValue(query.getParams().getSortField());
      WritableComparable sortValue;


      if (raw instanceof Integer) {
        sortValue = new IntWritable(((Integer)raw).intValue());
      } else if (raw instanceof Float) {
        sortValue = new FloatWritable(((Float)raw).floatValue());

View Full Code Here

   * using two HalfMapFiles.
   * @throws Exception
   */
  public void testBasicHalfMapFile() throws Exception {
    Path p = writeMapFile(getName());
    WritableComparable midkey = getMidkey(p);
    checkHalfMapFile(p, midkey);
  }

View Full Code Here

      new MapFile.Reader(this.fs, p.toString(), this.conf);
    HStoreKey key = new HStoreKey();
    ImmutableBytesWritable value = new ImmutableBytesWritable();
    reader.next(key, value);
    String firstKey = key.toString();
    WritableComparable midkey = reader.midKey();
    reader.finalKey(key);
    LOG.info("First key " + firstKey + ", midkey " + midkey.toString()
        + ", last key " + key.toString());
    reader.close();
    return midkey;
  }

View Full Code Here

        ((HStoreKey) midkey).getRow().toString());


      // Next test using a midkey that does not exist in the file.
      // First, do a key that is < than first key. Ensure splits behave
      // properly.
      WritableComparable badkey = new HStoreKey(new Text("   "));
      bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
          this.conf, HStoreFile.Range.bottom, badkey);
      // When badkey is < than the bottom, should return no values.
      assertFalse(bottom.next(key, value));
      // Now read from the top.

View Full Code Here

    
    private boolean doFirstNextProcessing(WritableComparable key, Writable val)
    throws IOException {
      // Seek to midkey.  Midkey may not exist in this file.  That should be
      // fine.  Then we'll either be positioned at end or start of file.
      WritableComparable nearest = getClosest(this.midkey, val);
      // Now copy the mid key into the passed key.
      if (nearest != null) {
        Writables.copyWritable(nearest, key);
        return true;
      }

View Full Code Here

        if (splitable) {
          splitable = !curHSF.isReference();
        }
      }
      MapFile.Reader r = this.readers.get(mapIndex);
      WritableComparable midkey = r.midKey();
      if (midkey != null) {
        midKey.set(((HStoreKey)midkey).getRow());
      }
    } catch(IOException e) {
      LOG.warn("Failed getting store size", e);

View Full Code Here

  private void mergeSortedFiles(FileStatus[] status, Path destinationFilePath,
      Class convertedKeyClass, Class rawKeyClass, Class rawValueClass)
      throws IOException {
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf,
        destinationFilePath, rawKeyClass, rawValueClass, CompressionType.NONE);
    WritableComparable convertedKey;
    MapWritable value;


    Map<Integer, SequenceFile.Reader> readers = new HashMap<Integer, SequenceFile.Reader>();
    for (int i = 0; i < status.length; i++) {
      SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
          convertedKeyClass, MapWritable.class, conf);
      sorter.setMemory(conf
          .getInt("bsp.input.runtime.partitioning.sort.mb", 50) * 1024 * 1024);
      sorter.setFactor(conf.getInt(
          "bsp.input.runtime.partitioning.sort.factor", 10));
      sorter.sort(status[i].getPath(), status[i].getPath().suffix(".sorted"));


      readers.put(i,
          new SequenceFile.Reader(fs, status[i].getPath().suffix(".sorted"),
              conf));
    }


    for (int i = 0; i < readers.size(); i++) {
      convertedKey = (WritableComparable) ReflectionUtils.newInstance(
          convertedKeyClass, conf);
      value = new MapWritable();


      readers.get(i).next(convertedKey, value);
      candidates.put(i, new KeyValuePair(convertedKey, value));
    }


    while (readers.size() > 0) {
      convertedKey = (WritableComparable) ReflectionUtils.newInstance(
          convertedKeyClass, conf);
      value = new MapWritable();


      int readerIndex = 0;
      WritableComparable firstKey = null;
      MapWritable rawRecord = null;


      for (Map.Entry<Integer, KeyValuePair<WritableComparable, MapWritable>> keys : candidates
          .entrySet()) {
        if (firstKey == null) {
          readerIndex = keys.getKey();
          firstKey = keys.getValue().getKey();
          rawRecord = (MapWritable) keys.getValue().getValue();
        } else {
          WritableComparable currentKey = keys.getValue().getKey();
          if (firstKey.compareTo(currentKey) > 0) {
            readerIndex = keys.getKey();
            firstKey = currentKey;
            rawRecord = (MapWritable) keys.getValue().getValue();
          }

View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public synchronized WritableComparable getClosest(WritableComparable key,
      Writable val)
    throws IOException {
      WritableComparable closest = null;
      if (top) {
        // If top, the lowest possible key is midkey.  Do not have to check
        // what comes back from super getClosest.  Will return exact match or
        // greater.
        closest = (key.compareTo(this.midkey) < 0)?
          this.midkey: super.getClosest(key, val);
      } else {
        // We're serving bottom of the file.
        if (key.compareTo(this.midkey) < 0) {
          // Check key is within range for bottom.
          closest = super.getClosest(key, val);
          // midkey was made against largest store file at time of split. Smaller
          // store files could have anything in them.  Check return value is
          // not beyond the midkey (getClosest returns exact match or next
          // after).
          if (closest != null && closest.compareTo(this.midkey) >= 0) {
            // Don't let this value out.
            closest = null;
          }
        }
        // Else, key is > midkey so let out closest = null.

View Full Code Here

      if (firstNextCall) {
        firstNextCall = false;
        if (this.top) {
          // Seek to midkey.  Midkey may not exist in this file.  That should be
          // fine.  Then we'll either be positioned at end or start of file.
          WritableComparable nearest = getClosest(midkey, val);
          // Now copy the mid key into the passed key.
          if (nearest != null) {
            Writables.copyWritable(nearest, key);
            return true;
          }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.WritableComparable

com.cloudera.sqoop.mapreduce.TestImportJob

com.ebay.erl.mobius.core.datajoin.DataJoinKey

com.ebay.erl.mobius.core.datajoin.DataJoinKeyPartitioner

com.ebay.erl.mobius.core.mapred.MobiusInputSampler

com.ebay.erl.mobius.core.model.ReadFieldImpl

com.ebay.erl.mobius.core.model.WriteImpl

com.tdunning.plume.local.lazy.MSCRCombiner

com.thinkaurelius.faunus.mapreduce.transform.PropertyMap$Map

edu.isi.karma.mapreduce.driver.TestJSONIdentityMapReduce

edu.umd.hooka.CreateMetadata

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.