Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.MapWritable

2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +
View Full Code Here
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
        LOG.warn("Unable to connect to master. Retrying. Error was:", e);
        sleeper.sleep();
      }
    }
    this.hbaseMaster = master;
    MapWritable result = null;
    long lastMsg = 0;
    while(!stopRequested.get()) {
      try {
        this.requestCount.set(0);
        MemoryUsage memory =
View Full Code Here
565
566
567
568
569
570
571
572
573
574
575
576
577
578
  /**
   * @return Subset of configuration to pass initializing regionservers: e.g.
   * the filesystem to use and root directory to use.
   */
  protected MapWritable createConfigurationSubset() {
    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    // Get the real address of the HRS.
    String rsAddress = HBaseServer.getRemoteAddress();
    if (rsAddress != null) {
      mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
    }
   
    return addConfig(mw, "fs.default.name");
  }
View Full Code Here
746
747
748
749
750
751
752
753
754
755
756
      createMyEphemeralNode();
  
      // Try and register with the Master; tell it we are here.  Break if
      // server is stopped or the clusterup flag is down or hdfs went wacky.
      while (keepLooping()) {
        MapWritable w = reportForDuty();
        if (w == null) {
          LOG.warn("reportForDuty failed; sleeping and then retrying.");
          this.sleeper.sleep();
        } else {
          handleReportForDutyResponse(w);
View Full Code Here
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
   * @return A Map of key/value configurations we got from the Master else
   * null if we failed to register.
   * @throws IOException
   */
  private MapWritable reportForDuty() throws IOException {
    MapWritable result = null;
    ServerName masterServerName = getMaster();
    if (masterServerName == null) return result;
    try {
      this.requestCount.set(0);
      LOG.info("Telling master at " + masterServerName + " that we are up " +
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
        newValue.append(originalValue.charAt(i));
      }
     
      // Now set the value to be collected

      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
     
      output.collect(tKey, outval);
    }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    }

    // apply redirect repr url logic for each datum
    for (CrawlDatum datum : datums) {

      MapWritable metadata = datum.getMetaData();
      Text reprUrl = (Text)metadata.get(Nutch.WRITABLE_REPR_URL_KEY);
      byte status = datum.getStatus();
      boolean isCrawlDb = (CrawlDatum.hasDbStatus(datum));
      boolean segFetched = (status == CrawlDatum.STATUS_FETCH_SUCCESS);

      // only if the crawl datum is from the crawldb or is a successfully
      // fetched page from the segments
      if ((isCrawlDb || segFetched) && reprUrl != null) {

        String src = reprUrl.toString();
        String dest = url;
        URL srcUrl = null;
        URL dstUrl = null;

        // both need to be well formed urls
        try {
          srcUrl = new URL(src);
          dstUrl = new URL(url);
        }
        catch (MalformedURLException e) {
        }

        // if the src and repr urls are the same after the new logic then
        // remove the repr url from the metadata as it is no longer needed
        if (srcUrl != null && dstUrl != null) {
          String reprOut = URLUtil.chooseRepr(src, dest, true);
          if (reprOut.equals(dest)) {
            LOG.info("Removing " + reprOut + " from " + dest);
            metadata.remove(Nutch.WRITABLE_REPR_URL_KEY);
          }
        }
      }

      // collect each datum
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
      OutputCollector<Text, CrawlDatum> output,
      Reporter reporter) throws IOException {
    newKey.set(key.toString());
    if (withMetadata) {
      CrawlDatum datum = (CrawlDatum)value;
      MapWritable meta = datum.getMetaData();
      if (meta.size() > 0) {
        MapWritable newMeta = new MapWritable();
        Iterator it = meta.keySet().iterator();
        while (it.hasNext()) {
          WritableComparable k = (WritableComparable)it.next();
          Writable v = meta.get(k);
          if (k instanceof UTF8) {
            Text t = new Text(k.toString());
            k = t;
          }
          newMeta.put(k, v);
        }
        datum.setMetaData(newMeta);
      }
    }
    output.collect(newKey, value);
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
        newValue.append(originalValue.charAt(i));
      }
     
      // Now set the value to be collected

      MapWritable outval = new MapWritable();
      outval.put(TEXT_OUTPUT_COLUMN, new ImmutableBytesWritable(
          newValue.toString().getBytes(HConstants.UTF8_ENCODING)));
     
      output.collect(tKey, outval);
    }
View Full Code Here
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
      scannerId = regionServer.openScanner(rl.getRegionInfo().getRegionName(),
          HConstants.COLUMN_FAMILY_ARRAY, new Text(),
          System.currentTimeMillis(), null);
      while (true) {
        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
        MapWritable values = regionServer.next(scannerId);
        if (values == null || values.size() == 0) {
          break;
        }
       
        for (Map.Entry<Writable, Writable> e: values.entrySet()) {
          HStoreKey k = (HStoreKey) e.getKey();
          results.put(k.getColumn(),
              ((ImmutableBytesWritable) e.getValue()).get());
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.MapWritable

  • backtype.hadoop.ElasticScheme
  • bbejeck.mapred.coocurrance.StripesReducerTest
  • ch.uzh.ifi.ddis.hadoop.CsvInputFormat$CsvRecordReader
  • com.digitalpebble.behemoth.BehemothDocument
  • com.digitalpebble.behemoth.DocumentFilter
  • com.digitalpebble.behemoth.gate.GATEProcessor
  • com.digitalpebble.behemoth.io.warc.WARCConverterJob
  • com.digitalpebble.behemoth.solr.SOLRWriter
  • com.digitalpebble.behemoth.tika.TikaProcessor
  • com.linkedin.camus.coders.CamusWrapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.