Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.MapWritable


   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +
View Full Code Here


        LOG.warn("Unable to connect to master. Retrying. Error was:", e);
        sleeper.sleep();
      }
    }
    this.hbaseMaster = master;
    MapWritable result = null;
    long lastMsg = 0;
    while(!stopRequested.get()) {
      try {
        this.requestCount.set(0);
        MemoryUsage memory =
View Full Code Here

  /**
   * @return Subset of configuration to pass initializing regionservers: e.g.
   * the filesystem to use and root directory to use.
   */
  protected MapWritable createConfigurationSubset() {
    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    // Get the real address of the HRS.
    String rsAddress = HBaseServer.getRemoteAddress();
    if (rsAddress != null) {
      mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
    }
   
    return addConfig(mw, "fs.default.name");
  }
View Full Code Here

      createMyEphemeralNode();
  
      // Try and register with the Master; tell it we are here.  Break if
      // server is stopped or the clusterup flag is down or hdfs went wacky.
      while (keepLooping()) {
        MapWritable w = reportForDuty();
        if (w == null) {
          LOG.warn("reportForDuty failed; sleeping and then retrying.");
          this.sleeper.sleep();
        } else {
          handleReportForDutyResponse(w);
View Full Code Here

   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +
View Full Code Here

        newValue.append(originalValue.charAt(i));
      }
     
      // Now set the value to be collected

      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
     
      output.collect(tKey, outval);
    }
View Full Code Here

    }

    // apply redirect repr url logic for each datum
    for (CrawlDatum datum : datums) {

      MapWritable metadata = datum.getMetaData();
      Text reprUrl = (Text)metadata.get(Nutch.WRITABLE_REPR_URL_KEY);
      byte status = datum.getStatus();
      boolean isCrawlDb = (CrawlDatum.hasDbStatus(datum));
      boolean segFetched = (status == CrawlDatum.STATUS_FETCH_SUCCESS);

      // only if the crawl datum is from the crawldb or is a successfully
      // fetched page from the segments
      if ((isCrawlDb || segFetched) && reprUrl != null) {

        String src = reprUrl.toString();
        String dest = url;
        URL srcUrl = null;
        URL dstUrl = null;

        // both need to be well formed urls
        try {
          srcUrl = new URL(src);
          dstUrl = new URL(url);
        }
        catch (MalformedURLException e) {
        }

        // if the src and repr urls are the same after the new logic then
        // remove the repr url from the metadata as it is no longer needed
        if (srcUrl != null && dstUrl != null) {
          String reprOut = URLUtil.chooseRepr(src, dest, true);
          if (reprOut.equals(dest)) {
            LOG.info("Removing " + reprOut + " from " + dest);
            metadata.remove(Nutch.WRITABLE_REPR_URL_KEY);
          }
        }
      }

      // collect each datum
View Full Code Here

      OutputCollector<Text, CrawlDatum> output,
      Reporter reporter) throws IOException {
    newKey.set(key.toString());
    if (withMetadata) {
      CrawlDatum datum = (CrawlDatum)value;
      MapWritable meta = datum.getMetaData();
      if (meta.size() > 0) {
        MapWritable newMeta = new MapWritable();
        Iterator it = meta.keySet().iterator();
        while (it.hasNext()) {
          WritableComparable k = (WritableComparable)it.next();
          Writable v = meta.get(k);
          if (k instanceof UTF8) {
            Text t = new Text(k.toString());
            k = t;
          }
          newMeta.put(k, v);
        }
        datum.setMetaData(newMeta);
      }
    }
    output.collect(newKey, value);
View Full Code Here

        newValue.append(originalValue.charAt(i));
      }
     
      // Now set the value to be collected

      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
     
      output.collect(tKey, outval);
    }
View Full Code Here

      scannerId = regionServer.openScanner(rl.getRegionInfo().getRegionName(),
          HConstants.COLUMN_FAMILY_ARRAY, new Text(),
          System.currentTimeMillis(), null);
      while (true) {
        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
        MapWritable values = regionServer.next(scannerId);
        if (values == null || values.size() == 0) {
          break;
        }
       
        for (Map.Entry<Writable, Writable> e: values.entrySet()) {
          HStoreKey k = (HStoreKey) e.getKey();
          results.put(k.getColumn(),
              ((ImmutableBytesWritable) e.getValue()).get());
        }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.MapWritable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.