Examples of MapWritable

org.apache.hadoop.io.MapWritable
A Writable Map.
org.apache.nutch.crawl.MapWritable
A writable map, with a similar behavior as java.util.HashMap. In addition to the size of key and value writable tuple two additional bytes are stored to identify the Writable classes. This means that a maximum of 255 different class types can be used for key and value objects. A binary-id to class mapping is defined in a static block of this class. However it is possible to use custom implementations of Writable. For these custom Writables we write the byte id - utf class name tuple into the header of each MapWritable that uses these types. @author Stefan Groschupf @deprecated Use org.apache.hadoop.io.MapWritable instead.

Examples of org.apache.hadoop.io.MapWritable

   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

        LOG.warn("Unable to connect to master. Retrying. Error was:", e);
        sleeper.sleep();
      }
    }
    this.hbaseMaster = master;
    MapWritable result = null;
    long lastMsg = 0;
    while(!stopRequested.get()) {
      try {
        this.requestCount.set(0);
        MemoryUsage memory =

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

  /**
   * @return Subset of configuration to pass initializing regionservers: e.g.
   * the filesystem to use and root directory to use.
   */
  protected MapWritable createConfigurationSubset() {
    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    // Get the real address of the HRS.
    String rsAddress = HBaseServer.getRemoteAddress();
    if (rsAddress != null) {
      mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
    }
    
    return addConfig(mw, "fs.default.name");
  }

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

      createMyEphemeralNode();
   
      // Try and register with the Master; tell it we are here.  Break if
      // server is stopped or the clusterup flag is down or hdfs went wacky.
      while (keepLooping()) {
        MapWritable w = reportForDuty();
        if (w == null) {
          LOG.warn("reportForDuty failed; sleeping and then retrying.");
          this.sleeper.sleep();
        } else {
          handleReportForDutyResponse(w);

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

        newValue.append(originalValue.charAt(i));
      }
      
      // Now set the value to be collected


      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
      
      output.collect(tKey, outval);
    }

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

    }


    // apply redirect repr url logic for each datum
    for (CrawlDatum datum : datums) {


      MapWritable metadata = datum.getMetaData();
      Text reprUrl = (Text)metadata.get(Nutch.WRITABLE_REPR_URL_KEY);
      byte status = datum.getStatus();
      boolean isCrawlDb = (CrawlDatum.hasDbStatus(datum));
      boolean segFetched = (status == CrawlDatum.STATUS_FETCH_SUCCESS);


      // only if the crawl datum is from the crawldb or is a successfully
      // fetched page from the segments
      if ((isCrawlDb || segFetched) && reprUrl != null) {


        String src = reprUrl.toString();
        String dest = url;
        URL srcUrl = null;
        URL dstUrl = null;


        // both need to be well formed urls
        try {
          srcUrl = new URL(src);
          dstUrl = new URL(url);
        }
        catch (MalformedURLException e) {
        }


        // if the src and repr urls are the same after the new logic then
        // remove the repr url from the metadata as it is no longer needed
        if (srcUrl != null && dstUrl != null) {
          String reprOut = URLUtil.chooseRepr(src, dest, true);
          if (reprOut.equals(dest)) {
            LOG.info("Removing " + reprOut + " from " + dest);
            metadata.remove(Nutch.WRITABLE_REPR_URL_KEY);
          }
        }
      }


      // collect each datum

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

      OutputCollector<Text, CrawlDatum> output,
      Reporter reporter) throws IOException {
    newKey.set(key.toString());
    if (withMetadata) {
      CrawlDatum datum = (CrawlDatum)value;
      MapWritable meta = datum.getMetaData();
      if (meta.size() > 0) {
        MapWritable newMeta = new MapWritable();
        Iterator it = meta.keySet().iterator();
        while (it.hasNext()) {
          WritableComparable k = (WritableComparable)it.next();
          Writable v = meta.get(k);
          if (k instanceof UTF8) {
            Text t = new Text(k.toString());
            k = t;
          }
          newMeta.put(k, v);
        }
        datum.setMetaData(newMeta);
      }
    }
    output.collect(newKey, value);

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

        newValue.append(originalValue.charAt(i));
      }
      
      // Now set the value to be collected


      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
      
      output.collect(tKey, outval);
    }

View Full Code Here

Examples of org.apache.hadoop.io.MapWritable

      scannerId = regionServer.openScanner(rl.getRegionInfo().getRegionName(),
          HConstants.COLUMN_FAMILY_ARRAY, new Text(),
          System.currentTimeMillis(), null);
      while (true) {
        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
        MapWritable values = regionServer.next(scannerId);
        if (values == null || values.size() == 0) {
          break;
        }
        
        for (Map.Entry<Writable, Writable> e: values.entrySet()) {
          HStoreKey k = (HStoreKey) e.getKey();
          results.put(k.getColumn(),
              ((ImmutableBytesWritable) e.getValue()).get());
        }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.