Examples of org.apache.hadoop.io.MapWritable

org.apache.hadoop.io.MapWritable
A Writable Map.

    // Do initial RPC setup.  The final argument indicates that the RPC should retry indefinitely.
    this.hbaseMaster = (HMasterRegionInterface)HbaseRPC.waitForProxy(
      HMasterRegionInterface.class, HMasterRegionInterface.versionID,
      new HServerAddress(conf.get(MASTER_ADDRESS)).getInetSocketAddress(),
      this.conf, -1);
    MapWritable result = null;
    long lastMsg = 0;
    while(!stopRequested.get()) {
      try {
        this.requestCount.set(0);
        this.serverInfo.setLoad(new HServerLoad(0, onlineRegions.size()));

View Full Code Here

   */
  public void run() {
    regionServerThread = Thread.currentThread();
    boolean quiesceRequested = false;
    try {
      MapWritable w = null;
      while (!stopRequested.get()) {
        w = reportForDuty();
        if (w != null) {
          init(w);
          break;

View Full Code Here

    while (!stopRequested.get() && !getMaster()) {
      sleeper.sleep();
      LOG.warn("Unable to get master for initialization");
    }


    MapWritable result = null;
    long lastMsg = 0;
    while(!stopRequested.get()) {
      try {
        this.requestCount.set(0);
        MemoryUsage memory =

View Full Code Here

  /**
   * @return Subset of configuration to pass initializing regionservers: e.g.
   * the filesystem to use and root directory to use.
   */
  protected MapWritable createConfigurationSubset() {
    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    // Get the real address of the HRS.
    String rsAddress = HBaseServer.getRemoteAddress();
    if (rsAddress != null) {
      mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
    }
    
    return addConfig(mw, "fs.default.name");
  }

View Full Code Here

  /**
   * @return Subset of configuration to pass initializing regionservers: e.g.
   * the filesystem to use and root directory to use.
   */
  protected MapWritable createConfigurationSubset() {
    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
    return addConfig(mw, "fs.default.name");
  }

View Full Code Here

  /**
   * @return True if successfully invoked {@link #reportForDuty()}
   * @throws IOException
   */
  private boolean tryReportForDuty() throws IOException {
    MapWritable w = reportForDuty();
    if (w != null) {
      handleReportForDutyResponse(w);
      return true;
    }
    sleeper.sleep();

View Full Code Here

    while (!stopped && (masterAddress = getMaster()) == null) {
      sleeper.sleep();
      LOG.warn("Unable to get master for initialization");
    }


    MapWritable result = null;
    long lastMsg = 0;
    
    //If some exception occured while reporting to Master, we should 
    //recheck the master address to avoid the address has been updated
    boolean recheckMasterAddr = false;

View Full Code Here

    serverInfo.setServerAddress(new HServerAddress(address));


    // Register with server manager
    this.serverManager.regionServerStartup(serverInfo, serverCurrentTime);
    // Send back some config info
    MapWritable mw = createConfigurationSubset();
     mw.put(new Text("hbase.regionserver.address"),
         serverInfo.getServerAddress());
    return mw;
  }

View Full Code Here

    if (fileType == FileType.AVRO_DATA_FILE) {
      LOG.debug("Configuring for Avro export");
      ConnManager connManager = context.getConnManager();
      Map<String, Integer> columnTypeInts =
        connManager.getColumnTypes(tableName, options.getSqlQuery());
      MapWritable columnTypes = new MapWritable();
      for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
        Text columnName = new Text(e.getKey());
        Text columnText = new Text(
            connManager.toJavaType(tableName, e.getKey(), e.getValue()));
        columnTypes.put(columnName, columnText);
      }
      DefaultStringifier.store(job.getConfiguration(), columnTypes,
          AvroExportMapper.AVRO_COLUMN_TYPES_MAP);
    }

View Full Code Here

  private DoubleMatrix normalizeMatrix(
      BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer,
      DoubleMatrix featureMatrix, IntWritable msgFeatureMatrix, boolean broadcast) 
          throws IOException, SyncException, InterruptedException {
    // send to master peer
    MapWritable msg = new MapWritable();
    MatrixWritable mtx = new MatrixWritable(featureMatrix);
    msg.put(msgFeatureMatrix, mtx);
    String master = peer.getPeerName(peer.getNumPeers()/2);
    peer.send(master, msg);
    peer.sync();


    // normalize
    DoubleMatrix res = null;
    if (peer.getPeerName().equals(master)) {
      res = new DenseDoubleMatrix(featureMatrix.getRowCount(),
                                  featureMatrix.getColumnCount(), 0);
      int incomingMsgCount = 0;
      while ( (msg = peer.getCurrentMessage()) != null) {
        MatrixWritable tmp = (MatrixWritable) msg.get(msgFeatureMatrix);
        res.add(tmp.getMatrix());
        incomingMsgCount++;
      }
      res.divide(incomingMsgCount);
    }


    if (broadcast) {
      if (peer.getPeerName().equals(master)) {
        // broadcast to all
        msg = new MapWritable();
        msg.put(msgFeatureMatrix, new MatrixWritable(res));
        // send to all
        for (String peerName : peer.getAllPeerNames()) {
          peer.send(peerName, msg);
        }
      }
      peer.sync();
      // receive normalized value from master
      msg = peer.getCurrentMessage();
      featureMatrix = ((MatrixWritable)msg.get(msgFeatureMatrix)).getMatrix();
    }
    return res;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.MapWritable

backtype.hadoop.ElasticScheme

bbejeck.mapred.coocurrance.StripesReducerTest

ch.uzh.ifi.ddis.hadoop.CsvInputFormat$CsvRecordReader

com.digitalpebble.behemoth.BehemothDocument

com.digitalpebble.behemoth.DocumentFilter

com.digitalpebble.behemoth.gate.GATEProcessor

com.digitalpebble.behemoth.io.warc.WARCConverterJob

com.digitalpebble.behemoth.solr.SOLRWriter

com.digitalpebble.behemoth.tika.TikaProcessor

com.linkedin.camus.coders.CamusWrapper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.