* Copyright 2010 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* HRegionServer makes a set of HRegions available to clients. It checks in with

* the HMaster. There are many HRegionServers in a single HBase deployment.

*/

    Runnable, RegionServerServices, Server {

  public static final Log LOG = LogFactory.getLog(HRegionServer.class);

  // Set when a report to the master comes back with a message asking us to

  // shutdown. Also set by call to stop when debugging or running unit tests

  // of HRegionServer in isolation.

  protected volatile boolean stopped = false;

  // A state before we go into stopped state.  At this stage we're closing user

  // space regions.

  private boolean stopping = false;

  // Go down hard. Used if file system becomes unavailable and also in

  // debugging and unit tests.

  protected volatile boolean abortRequested;

  private volatile boolean killed = false;

  // If false, the file system has become unavailable

  protected volatile boolean fsOk;

  protected HServerInfo serverInfo;

  protected final Configuration conf;

  private final HConnection connection;

  protected final AtomicBoolean haveRootRegion = new AtomicBoolean(false);

  private FileSystem fs;

  private Path rootDir;

  private final Random rand = new Random();

  /**

   * Map of regions currently being served by this region server. Key is the

   * encoded region name.  All access should be synchronized.

   */

  protected final Map<String, HRegion> onlineRegions =

    new HashMap<String, HRegion>();

  protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

  private final LinkedBlockingQueue<HMsg> outboundMsgs = new LinkedBlockingQueue<HMsg>();

  final int numRetries;

  protected final int threadWakeFrequency;

  private final int msgInterval;

  protected final int numRegionsToReport;

  private final long maxScannerResultSize;

  // Remote HMaster

  private HMasterRegionInterface hbaseMaster;

  // Server to handle client requests. Default access so can be accessed by

  // unit tests.

  HBaseServer server;

  // Leases

  private Leases leases;

  // Request counter

  private volatile AtomicInteger requestCount = new AtomicInteger();

  // Info server. Default access so can be used by unit tests. REGIONSERVER

  // is name of the webapp and the attribute name used stuffing this instance

  // into web context.

  InfoServer infoServer;

  /** region server process name */

  public static final String REGIONSERVER = "regionserver";

  /*

   * Space is reserved in HRS constructor and then released when aborting to

   * recover from an OOME. See HBASE-706. TODO: Make this percentage of the heap

   * or a minimum.

   */

  private final LinkedList<byte[]> reservedSpace = new LinkedList<byte[]>();

  private RegionServerMetrics metrics;

  // Compactions

  CompactSplitThread compactSplitThread;

  // Cache flushing

  MemStoreFlusher cacheFlusher;

  /*

   * Check for major compactions.

   */

  Chore majorCompactionChecker;

  // HLog and HLog roller. log is protected rather than private to avoid

  // eclipse warning when accessed by inner classes

  protected volatile HLog hlog;

  LogRoller hlogRoller;

  // flag set after we're done setting up server threads (used for testing)

  protected volatile boolean isOnline;

  final Map<String, InternalScanner> scanners = new ConcurrentHashMap<String, InternalScanner>();

  // zookeeper connection and watcher

  private ZooKeeperWatcher zooKeeper;

  // master address manager and watcher

  private MasterAddressTracker masterAddressManager;

  // catalog tracker

  private CatalogTracker catalogTracker;

  // Cluster Status Tracker

  private ClusterStatusTracker clusterStatusTracker;

  // A sleeper that sleeps for msgInterval.

  private final Sleeper sleeper;

  private final int rpcTimeout;

  // The main region server thread.

  @SuppressWarnings("unused")

  private Thread regionServerThread;

  // Instance of the hbase executor service.

  private ExecutorService service;

  // Replication services. If no replication, this handler will be null.

  private Replication replicationHandler;

  /**

   * Starts a HRegionServer at the default location

   *

   * @param conf

   * @throws IOException

   * @throws InterruptedException

   */

  public HRegionServer(Configuration conf) throws IOException, InterruptedException {

    this.fsOk = true;

    this.conf = conf;

    this.connection = HConnectionManager.getConnection(conf);

    this.isOnline = false;

    // check to see if the codec list is available:

    String [] codecs = conf.getStrings("hbase.regionserver.codecs",

        (String[])null);

    if (codecs != null) {

      for (String codec : codecs) {

        if (!CompressionTest.testCompression(codec)) {

          throw new IOException("Compression codec " + codec +

              " not supported, aborting RS construction");

        }

      }

    }

    // Config'ed params

    this.numRetries = conf.getInt("hbase.client.retries.number", 10);

    this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,

        10 * 1000);

    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);

    sleeper = new Sleeper(this.msgInterval, this);

    this.maxScannerResultSize = conf.getLong(

        HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,

        HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);

    this.numRegionsToReport = conf.getInt(

        "hbase.regionserver.numregionstoreport", 10);

    this.rpcTimeout = conf.getInt(

        HConstants.HBASE_RPC_TIMEOUT_KEY,

        HConstants.DEFAULT_HBASE_RPC_TIMEOUT);

    this.abortRequested = false;

    this.stopped = false;

    // Server to handle client requests

    String machineName = DNS.getDefaultHost(conf.get(

        "hbase.regionserver.dns.interface", "default"), conf.get(

        "hbase.regionserver.dns.nameserver", "default"));

    String addressStr = machineName + ":" +

      conf.get(HConstants.REGIONSERVER_PORT,

        Integer.toString(HConstants.DEFAULT_REGIONSERVER_PORT));

    HServerAddress address = new HServerAddress(addressStr);

    this.server = HBaseRPC.getServer(this,

        new Class<?>[]{HRegionInterface.class, HBaseRPCErrorHandler.class,

        OnlineRegions.class},

        address.getBindAddress(),

      address.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),

        conf.getInt("hbase.regionserver.metahandler.count", 10),

        false, conf, QOS_THRESHOLD);

    this.server.setErrorHandler(this);

    this.server.setQosFunction(new QosFunction());

    // HServerInfo can be amended by master.  See below in reportForDuty.

    this.serverInfo = new HServerInfo(new HServerAddress(new InetSocketAddress(

        address.getBindAddress(), this.server.getListenerAddress().getPort())),

        System.currentTimeMillis(), this.conf.getInt(

            "hbase.regionserver.info.port", 60030), machineName);

    if (this.serverInfo.getServerAddress() == null) {

      throw new NullPointerException("Server address cannot be null; "

          + "hbase-958 debugging");

    }

  }

  private static final int NORMAL_QOS = 0;

  private static final int QOS_THRESHOLD = 10;  // the line between low and high qos

  private static final int HIGH_QOS = 100;

  @Retention(RetentionPolicy.RUNTIME)

  private @interface QosPriority {

    int priority() default 0;

  }

  class QosFunction implements Function<Writable,Integer> {

    private final Map<String, Integer> annotatedQos;

    public QosFunction() {

      Map<String, Integer> qosMap = new HashMap<String, Integer>();

      for (Method m : HRegionServer.class.getMethods()) {

        QosPriority p = m.getAnnotation(QosPriority.class);

        if (p != null) {

          qosMap.put(m.getName(), p.priority());

        }

      }

     

      annotatedQos = qosMap;

    }

    public boolean isMetaRegion(byte[] regionName) {

      HRegion region;

      try {

        region = getRegion(regionName);

      } catch (NotServingRegionException ignored) {

        return false;

      }

      return region.getRegionInfo().isMetaRegion();

    }

    @Override

    public Integer apply(Writable from) {

      if (!(from instanceof HBaseRPC.Invocation)) return NORMAL_QOS;

      HBaseRPC.Invocation inv = (HBaseRPC.Invocation) from;

      String methodName = inv.getMethodName();

     

      Integer priorityByAnnotation = annotatedQos.get(methodName);

      if (priorityByAnnotation != null) {

        return priorityByAnnotation;

      }

      // scanner methods...

      if (methodName.equals("next") || methodName.equals("close")) {

        // translate!

        Long scannerId;

        try {

          scannerId = (Long) inv.getParameters()[0];

        } catch (ClassCastException ignored) {

          // LOG.debug("Low priority: " + from);

          return NORMAL_QOS; // doh.

        }

        String scannerIdString = Long.toString(scannerId);

        InternalScanner scanner = scanners.get(scannerIdString);

        if (scanner instanceof HRegion.RegionScanner) {

          HRegion.RegionScanner rs = (HRegion.RegionScanner) scanner;

          HRegionInfo regionName = rs.getRegionName();

          if (regionName.isMetaRegion()) {

            // LOG.debug("High priority scanner request: " + scannerId);

            return HIGH_QOS;

          }

        }

      } else if (inv.getParameterClasses().length == 0) {

       // Just let it through.  This is getOnlineRegions, etc.

      } else if (inv.getParameterClasses()[0] == byte[].class) {

        // first arg is byte array, so assume this is a regionname:

        if (isMetaRegion((byte[]) inv.getParameters()[0])) {

          // LOG.debug("High priority with method: " + methodName +

          // " and region: "

          // + Bytes.toString((byte[]) inv.getParameters()[0]));

          return HIGH_QOS;

        }

      } else if (inv.getParameterClasses()[0] == MultiAction.class) {

        MultiAction ma = (MultiAction) inv.getParameters()[0];

        Set<byte[]> regions = ma.getRegions();

        // ok this sucks, but if any single of the actions touches a meta, the

        // whole

        // thing gets pingged high priority. This is a dangerous hack because

        // people

        // can get their multi action tagged high QOS by tossing a Get(.META.)

        // AND this

        // regionserver hosts META/-ROOT-

        for (byte[] region : regions) {

          if (isMetaRegion(region)) {

            // LOG.debug("High priority multi with region: " +

            // Bytes.toString(region));

            return HIGH_QOS; // short circuit for the win.

          }

        }

      }

      // LOG.debug("Low priority: " + from.toString());

      return NORMAL_QOS;

    }

  }

  /**

   * Creates all of the state that needs to be reconstructed in case we are

   * doing a restart. This is shared between the constructor and restart(). Both

   * call it.

   *

   * @throws IOException

   * @throws InterruptedException

   */

  private void initialize() throws IOException, InterruptedException {

    try {

      initializeZooKeeper();

      initializeThreads();

      int nbBlocks = conf.getInt("hbase.regionserver.nbreservationblocks", 4);

      for (int i = 0; i < nbBlocks; i++) {

        reservedSpace.add(new byte[HConstants.DEFAULT_SIZE_RESERVATION_BLOCK]);

      }

    } catch (Throwable t) {

      // Call stop if error or process will stick around for ever since server

      // puts up non-daemon threads.

      LOG.error("Stopping HRS because failed initialize", t);

      this.server.stop();

    }

  }

  /**

   * Bring up connection to zk ensemble and then wait until a master for this

   * cluster and then after that, wait until cluster 'up' flag has been set.

   * This is the order in which master does things.

   * Finally put up a catalog tracker.

   * @throws IOException

   * @throws InterruptedException

   */

  private void initializeZooKeeper() throws IOException, InterruptedException {

    // Open connection to zookeeper and set primary watcher

    zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +

      serverInfo.getServerAddress().getPort(), this);

    // Create the master address manager, register with zk, and start it.  Then

    // block until a master is available.  No point in starting up if no master

    // running.

    this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);

    this.masterAddressManager.start();

    blockAndCheckIfStopped(this.masterAddressManager);

    // Wait on cluster being up.  Master will set this flag up in zookeeper

    // when ready.

    this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);

    this.clusterStatusTracker.start();

    blockAndCheckIfStopped(this.clusterStatusTracker);

    // Create the catalog tracker and start it;

    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection,

      this, this.conf.getInt("hbase.regionserver.catalog.timeout", Integer.MAX_VALUE));

    catalogTracker.start();

  }

  /**

   * Utilty method to wait indefinitely on a znode availability while checking

   * if the region server is shut down

   * @param tracker znode tracker to use

   * @throws IOException any IO exception, plus if the RS is stopped

   * @throws InterruptedException

   */

  private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)

      throws IOException, InterruptedException {

    while (tracker.blockUntilAvailable(this.msgInterval) == null) {

      if (this.stopped) {

        throw new IOException("Received the shutdown message while waiting.");

      }

    }

  }

  /**

   * @return False if cluster shutdown in progress

   */

  private boolean isClusterUp() {

    return this.clusterStatusTracker.isClusterUp();

  }

  private void initializeThreads() throws IOException {

    // Cache flushing thread.

    this.cacheFlusher = new MemStoreFlusher(conf, this);

    // Compaction thread

    this.compactSplitThread = new CompactSplitThread(this);

    // Background thread to check for major compactions; needed if region

    // has not gotten updates in a while. Make it run at a lesser frequency.

    int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY

        + ".multiplier", 1000);

    this.majorCompactionChecker = new MajorCompactionChecker(this,

        this.threadWakeFrequency * multiplier, this);

    this.leases = new Leases((int) conf.getLong(

        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,

        HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),

        this.threadWakeFrequency);

  }

  /**

   * The HRegionServer sticks in this loop until closed. It repeatedly checks in

   * with the HMaster, sending heartbeats & reports, and receiving HRegion

   * load/unload instructions.

   */

  public void run() {

    try {

      // Initialize threads and wait for a master

      initialize();

    } catch (Exception e) {

      abort("Fatal exception during initialization", e);

    }

    this.regionServerThread = Thread.currentThread();

    try {

      while (!this.stopped) {

        if (tryReportForDuty()) break;

      }

      long lastMsg = 0;

      List<HMsg> outboundMessages = new ArrayList<HMsg>();

      // The main run loop.

      for (int tries = 0; !this.stopped && isHealthy();) {

        if (!isClusterUp()) {

          if (isOnlineRegionsEmpty()) {

            stop("Exiting; cluster shutdown set and not carrying any regions");

          } else if (!this.stopping) {

            this.stopping = true;

            closeUserRegions(this.abortRequested);

          } else if (this.stopping && LOG.isDebugEnabled()) {

            LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());

          }

        }

        long now = System.currentTimeMillis();

        // Drop into the send loop if msgInterval has elapsed or if something

        // to send. If we fail talking to the master, then we'll sleep below

        // on poll of the outboundMsgs blockingqueue.

        if ((now - lastMsg) >= msgInterval || !outboundMessages.isEmpty()) {

          try {

            doMetrics();

            tryRegionServerReport(outboundMessages);

            lastMsg = System.currentTimeMillis();

            // Reset tries count if we had a successful transaction.

            tries = 0;

            if (this.stopped) continue;

          } catch (Exception e) { // FindBugs REC_CATCH_EXCEPTION

            // Two special exceptions could be printed out here,

            // PleaseHoldException and YouAreDeadException

            if (e instanceof IOException) {

              e = RemoteExceptionHandler.checkIOException((IOException) e);

            }

            if (e instanceof YouAreDeadException) {

              // This will be caught and handled as a fatal error below

              throw e;

            }

            tries++;

            if (tries > 0 && (tries % this.numRetries) == 0) {

              // Check filesystem every so often.

              checkFileSystem();

            }

            if (this.stopped) {

              continue;

            }

            LOG.warn("Attempt=" + tries, e);

            // No point retrying immediately; this is probably connection to

            // master issue. Doing below will cause us to sleep.

            lastMsg = System.currentTimeMillis();

          }

        }

        now = System.currentTimeMillis();

        HMsg msg = this.outboundMsgs.poll((msgInterval - (now - lastMsg)), TimeUnit.MILLISECONDS);

        if (msg != null) outboundMessages.add(msg);

      } // for

    } catch (Throwable t) {

      if (!checkOOME(t)) {

        abort("Unhandled exception: " + t.getMessage(), t);

      }

    }

    this.leases.closeAfterLeasesExpire();

    this.server.stop();

    if (this.infoServer != null) {

      LOG.info("Stopping infoServer");

      try {

        this.infoServer.stop();

      } catch (Exception e) {

        e.printStackTrace();

      }

    }

    // Send cache a shutdown.

    LruBlockCache c = (LruBlockCache) StoreFile.getBlockCache(this.conf);

    if (c != null) {

      c.shutdown();

    }

    // Send interrupts to wake up threads if sleeping so they notice shutdown.

    // TODO: Should we check they are alive? If OOME could have exited already

    if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();

    if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();

    if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();

    if (this.majorCompactionChecker != null) this.majorCompactionChecker.interrupt();

    if (this.killed) {

      // Just skip out w/o closing regions.

    } else if (abortRequested) {

      if (this.fsOk) {

        closeAllRegions(abortRequested); // Don't leave any open file handles

        closeWAL(false);

      }

      LOG.info("aborting server at: " + this.serverInfo.getServerName());

    } else {

      closeAllRegions(abortRequested);

      closeWAL(true);

      closeAllScanners();

      LOG.info("stopping server at: " + this.serverInfo.getServerName());

    }

    // Interrupt catalog tracker here in case any regions being opened out in

    // handlers are stuck waiting on meta or root.

    if (this.catalogTracker != null) this.catalogTracker.stop();

    if (this.fsOk) waitOnAllRegionsToClose();

    // Make sure the proxy is down.

    if (this.hbaseMaster != null) {

      HBaseRPC.stopProxy(this.hbaseMaster);

      this.hbaseMaster = null;

    }

    this.leases.close();

    HConnectionManager.deleteConnection(conf, true);

    this.zooKeeper.close();

    if (!killed) {

      join();

    }

    LOG.info(Thread.currentThread().getName() + " exiting");

  }

  String getOnlineRegionsAsPrintableString() {

    StringBuilder sb = new StringBuilder();

    synchronized (this.onlineRegions) {

      for (HRegion r: this.onlineRegions.values()) {

        if (sb.length() > 0) sb.append(", ");

        sb.append(r.getRegionInfo().getEncodedName());

      }

    }

    return sb.toString();

  }

  /**

   * Wait on regions close.

   */

  private void waitOnAllRegionsToClose() {

    // Wait till all regions are closed before going out.

    int lastCount = -1;

    while (!isOnlineRegionsEmpty()) {