*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* HMaster is the "master server" for HBase. An HBase cluster has one active

* master.  If many masters are started, all compete.  Whichever wins goes on to

* run the cluster.  All others park themselves in their constructor until

* master or cluster shutdown or until the active master loses its lease in

* zookeeper.  Thereafter, all running master jostle to take over master role.

*

* <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In

* this case it will tell all regionservers to go down and then wait on them

* all reporting in that they are down.  This master will then shut itself down.

*

* <p>You can also shutdown just this master.  Call {@link #stopMaster()}.

*

* @see Watcher

*/

@SuppressWarnings("deprecation")

  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());

  // MASTER is name of the webapp and the attribute name used stuffing this

  //instance into web context.

  public static final String MASTER = "master";

  // Manager and zk listener for master election

  private ActiveMasterManager activeMasterManager;

  // Region server tracker

  RegionServerTracker regionServerTracker;

  // Draining region server tracker

  private DrainingServerTracker drainingServerTracker;

  // Tracker for load balancer state

  LoadBalancerTracker loadBalancerTracker;

  /** Namespace stuff */

  private TableNamespaceManager tableNamespaceManager;

  // Metrics for the HMaster

  final MetricsMaster metricsMaster;

  // file system manager for the master FS operations

  private MasterFileSystem fileSystemManager;

  // server manager to deal with region server info

  volatile ServerManager serverManager;

  // manager of assignment nodes in zookeeper

  AssignmentManager assignmentManager;

  // buffer for "fatal error" notices from region servers

  // in the cluster. This is only used for assisting

  // operations/debugging.

  MemoryBoundedLogMessageBuffer rsFatals;

  // flag set after we become the active master (used for testing)

  private volatile boolean isActiveMaster = false;

  // flag set after we complete initialization once active,

  // it is not private since it's used in unit tests

  volatile boolean initialized = false;

  // flag set after master services are started,

  // initialization may have not completed yet.

  volatile boolean serviceStarted = false;

  // flag set after we complete assignMeta.

  private volatile boolean serverShutdownHandlerEnabled = false;

  LoadBalancer balancer;

  private BalancerChore balancerChore;

  private ClusterStatusChore clusterStatusChore;

  private ClusterStatusPublisher clusterStatusPublisherChore = null;

  CatalogJanitor catalogJanitorChore;

  private LogCleaner logCleaner;

  private HFileCleaner hfileCleaner;

  MasterCoprocessorHost cpHost;

  // Time stamps for when a hmaster became active

  private long masterActiveTime;

  //should we check the compression codec type at master side, default true, HBASE-6370

  private final boolean masterCheckCompression;

  Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();

  // monitor for snapshot of hbase tables

  SnapshotManager snapshotManager;

  // monitor for distributed procedures

  MasterProcedureManagerHost mpmHost;

  /** flag used in test cases in order to simulate RS failures during master initialization */

  private volatile boolean initializationBeforeMetaAssignment = false;

  /** jetty server for master to redirect requests to regionserver infoServer */

  private org.mortbay.jetty.Server masterJettyServer;

    private static final long serialVersionUID = 2894774810058302472L;

    private static int regionServerInfoPort;

    @Override

    public void doGet(HttpServletRequest request,

        HttpServletResponse response) throws ServletException, IOException {

      String redirectUrl = request.getScheme() + "://"

        + request.getServerName() + ":" + regionServerInfoPort

        + request.getRequestURI();

      response.sendRedirect(redirectUrl);

    }

  }

  /**

   * Initializes the HMaster. The steps are as follows:

   * <p>

   * <ol>

   * <li>Initialize the local HRegionServer

   * <li>Start the ActiveMasterManager.

   * </ol>

   * <p>

   * Remaining steps of initialization occur in

   * {@link #finishActiveMasterInitialization(MonitoredTask)} after

   * the master becomes the active one.

   *

   * @throws InterruptedException

   * @throws KeeperException

   * @throws IOException

   */

  public HMaster(final Configuration conf, CoordinatedStateManager csm)

      throws IOException, KeeperException, InterruptedException {

    super(conf, csm);

    this.rsFatals = new MemoryBoundedLogMessageBuffer(

      conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));

    LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +

        ", hbase.cluster.distributed=" + this.conf.getBoolean("hbase.cluster.distributed", false));

    Replication.decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this

    // config param for task trackers, but we can piggyback off of it.

    if (this.conf.get("mapreduce.task.attempt.id") == null) {

      this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());

    }

    //should we check the compression codec type at master side, default true, HBASE-6370

    this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);

    this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));

    // Do we publish the status?

    boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,

        HConstants.STATUS_PUBLISHED_DEFAULT);

    Class<? extends ClusterStatusPublisher.Publisher> publisherClass =

        conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,

            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,

            ClusterStatusPublisher.Publisher.class);

    if (shouldPublish) {

      if (publisherClass == null) {

        LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +

            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +

            " is not set - not publishing status");

      } else {

        clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);

        Threads.setDaemonThreadRunning(clusterStatusPublisherChore.getThread());

      }

    }

    startActiveMasterManager();

    putUpJettyServer();

  }

  private void putUpJettyServer() throws IOException {

    if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {

      return;

    }

    int infoPort = conf.getInt("hbase.master.info.port.orig",

      HConstants.DEFAULT_MASTER_INFOPORT);

    // -1 is for disabling info server, so no redirecting

    if (infoPort < 0 || infoServer == null) {

      return;

    }

    RedirectServlet.regionServerInfoPort = infoServer.getPort();

    masterJettyServer = new org.mortbay.jetty.Server();

    Connector connector = new SelectChannelConnector();

    connector.setHost(conf.get("hbase.master.info.bindAddress", "0.0.0.0"));

    connector.setPort(infoPort);

    masterJettyServer.addConnector(connector);

    masterJettyServer.setStopAtShutdown(true);

    Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);

    context.addServlet(RedirectServlet.class, "/*");

    try {

      masterJettyServer.start();

    } catch (Exception e) {

      throw new IOException("Failed to start redirecting jetty server", e);

    }

  }

  /**

   * For compatibility, if failed with regionserver credentials, try the master one

   */

  protected void login(UserProvider user, String host) throws IOException {

    try {

      super.login(user, host);

    } catch (IOException ie) {

      user.login("hbase.master.keytab.file",

        "hbase.master.kerberos.principal", host);

    }

  }

  @VisibleForTesting

  public MasterRpcServices getMasterRpcServices() {

    return (MasterRpcServices)rpcServices;

  }

  public boolean balanceSwitch(final boolean b) throws IOException {

    return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);

  }

  protected String getProcessName() {

    return MASTER;

  }

  protected boolean canCreateBaseZNode() {

    return true;

  }

  protected boolean canUpdateTableDescriptor() {

    return true;

  }

  protected RSRpcServices createRpcServices() throws IOException {

    return new MasterRpcServices(this);

  }

  protected void configureInfoServer() {

    infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);

    infoServer.setAttribute(MASTER, this);

    super.configureInfoServer();

  }

  protected Class<? extends HttpServlet> getDumpServlet() {

    return MasterDumpServlet.class;

  }

  /**

   * Emit the HMaster metrics, such as region in transition metrics.

   * Surrounding in a try block just to be sure metrics doesn't abort HMaster.

   */

  protected void doMetrics() {

    try {

      if (assignmentManager != null) {

        assignmentManager.updateRegionsInTransitionMetrics();

      }

    } catch (Throwable e) {

      LOG.error("Couldn't update metrics: " + e.getMessage());

    }

  }

  MetricsMaster getMasterMetrics() {

    return metricsMaster;

  }

  /**

   * Initialize all ZK based system trackers.

   * @throws IOException

   * @throws InterruptedException

   * @throws KeeperException

   * @throws CoordinatedStateException

   */

  void initializeZKBasedSystemTrackers() throws IOException,

      InterruptedException, KeeperException, CoordinatedStateException {

    this.balancer = LoadBalancerFactory.getLoadBalancer(conf);

    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);

    this.loadBalancerTracker.start();

    this.assignmentManager = new AssignmentManager(this, serverManager,

      this.balancer, this.service, this.metricsMaster,

      this.tableLockManager);

    zooKeeper.registerListenerFirst(assignmentManager);

    this.regionServerTracker = new RegionServerTracker(zooKeeper, this,

        this.serverManager);

    this.regionServerTracker.start();

    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,

      this.serverManager);

    this.drainingServerTracker.start();

    // Set the cluster as up.  If new RSs, they'll be waiting on this before

    // going ahead with their startup.

    boolean wasUp = this.clusterStatusTracker.isClusterUp();

    if (!wasUp) this.clusterStatusTracker.setClusterUp();

    LOG.info("Server active/primary master=" + this.serverName +

        ", sessionid=0x" +

        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +

        ", setting cluster-up flag (Was=" + wasUp + ")");

    // create/initialize the snapshot manager and other procedure managers

    this.snapshotManager = new SnapshotManager();

    this.mpmHost = new MasterProcedureManagerHost();

    this.mpmHost.register(this.snapshotManager);

    this.mpmHost.register(new MasterFlushTableProcedureManager());

    this.mpmHost.loadProcedures(conf);

    this.mpmHost.initialize(this, this.metricsMaster);

  }

  /**

   * Finish initialization of HMaster after becoming the primary master.

   *

   * <ol>

   * <li>Initialize master components - file system manager, server manager,

   *     assignment manager, region server tracker, etc</li>

   * <li>Start necessary service threads - balancer, catalog janior,

   *     executor services, etc</li>

   * <li>Set cluster as UP in ZooKeeper</li>

   * <li>Wait for RegionServers to check-in</li>

   * <li>Split logs and perform data recovery, if necessary</li>

   * <li>Ensure assignment of meta/namespace regions<li>

   * <li>Handle either fresh cluster start or master failover</li>

   * </ol>

   *

   * @throws IOException

   * @throws InterruptedException

   * @throws KeeperException

   * @throws CoordinatedStateException

   */

  private void finishActiveMasterInitialization(MonitoredTask status)

      throws IOException, InterruptedException, KeeperException, CoordinatedStateException {

    isActiveMaster = true;

    /*

     * We are active master now... go initialize components we need to run.

     * Note, there may be dross in zk from previous runs; it'll get addressed

     * below after we determine if cluster startup or failover.

     */

    status.setStatus("Initializing Master file system");

    this.masterActiveTime = System.currentTimeMillis();

    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.

    this.fileSystemManager = new MasterFileSystem(this, this);

    // publish cluster ID

    status.setStatus("Publishing Cluster ID in ZooKeeper");

    ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());

    this.serverManager = createServerManager(this, this);

    synchronized (this) {

      if (shortCircuitConnection == null) {

        shortCircuitConnection = createShortCircuitConnection();

        metaTableLocator = new MetaTableLocator();

      }

    }

    // Invalidate all write locks held previously

    this.tableLockManager.reapWriteLocks();

    status.setStatus("Initializing ZK system trackers");

    initializeZKBasedSystemTrackers();

    // initialize master side coprocessors before we start handling requests

    status.setStatus("Initializing master coprocessors");

    this.cpHost = new MasterCoprocessorHost(this, this.conf);

    // start up all service threads.

    status.setStatus("Initializing master service threads");

    startServiceThreads();

    // Wake up this server to check in

    sleeper.skipSleepCycle();

    // Wait for region servers to report in

    this.serverManager.waitForRegionServers(status);

    // Check zk for region servers that are up but didn't register

    for (ServerName sn: this.regionServerTracker.getOnlineServers()) {

      // The isServerOnline check is opportunistic, correctness is handled inside

      if (!this.serverManager.isServerOnline(sn)

          && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {

        LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);

      }

    }

    // get a list for previously failed RS which need log splitting work

    // we recover hbase:meta region servers inside master initialization and

    // handle other failed servers in SSH in order to start up master node ASAP

    Set<ServerName> previouslyFailedServers = this.fileSystemManager

        .getFailedServersFromLogFolders();

    // remove stale recovering regions from previous run

    this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);

    // log splitting for hbase:meta server

    ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());

    if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {

      splitMetaLogBeforeAssignment(oldMetaServerLocation);

      // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it

      // may also host user regions

    }

    Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();

    // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers

    // instead of previouslyFailedMetaRSs alone to address the following two situations:

    // 1) the chained failure situation(recovery failed multiple times in a row).

    // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the

    // same server still has non-meta wals to be replayed so that

    // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region

    // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,

    // there is no op for the server.

    previouslyFailedMetaRSs.addAll(previouslyFailedServers);

    this.initializationBeforeMetaAssignment = true;

    // Wait for regionserver to finish initialization.

    synchronized (online) {

      while (!isStopped() && !isOnline()) {

        online.wait(100);

      }

    }

    //initialize load balancer

    this.balancer.setClusterStatus(getClusterStatus());

    this.balancer.setMasterServices(this);

    this.balancer.initialize();

    // Check if master is shutting down because of some issue

    // in initializing the regionserver or the balancer.

    if(isStopped()) return;

    // Make sure meta assigned before proceeding.

    status.setStatus("Assigning Meta Region");

    assignMeta(status, previouslyFailedMetaRSs);

    // check if master is shutting down because above assignMeta could return even hbase:meta isn't

    // assigned when master is shutting down

    if(isStopped()) return;

    status.setStatus("Submitting log splitting work for previously failed region servers");

    // Master has recovered hbase:meta region server and we put

    // other failed region servers in a queue to be handled later by SSH

    for (ServerName tmpServer : previouslyFailedServers) {

      this.serverManager.processDeadServer(tmpServer, true);

    }

    // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization

    // in meta. This must happen before we assign all user regions or else the assignment will

    // fail.

    if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {

      MetaMigrationConvertingToPB.updateMetaIfNecessary(this);

    }

    // Fix up assignment manager status

    status.setStatus("Starting assignment manager");

    this.assignmentManager.joinCluster();

    //set cluster status again after user regions are assigned

    this.balancer.setClusterStatus(getClusterStatus());

    // Start balancer and meta catalog janitor after meta and regions have

    // been assigned.

    status.setStatus("Starting balancer and catalog janitor");

    this.clusterStatusChore = new ClusterStatusChore(this, balancer);

    Threads.setDaemonThreadRunning(clusterStatusChore.getThread());

    this.balancerChore = new BalancerChore(this);

    Threads.setDaemonThreadRunning(balancerChore.getThread());

    this.catalogJanitorChore = new CatalogJanitor(this, this);

    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());

    status.setStatus("Starting namespace manager");

    initNamespace();

    if (this.cpHost != null) {

      try {

        this.cpHost.preMasterInitialization();

      } catch (IOException e) {

        LOG.error("Coprocessor preMasterInitialization() hook failed", e);

      }

    }

    status.markComplete("Initialization successful");

    LOG.info("Master has completed initialization");

    initialized = true;

    // clear the dead servers with same host name and port of online server because we are not

    // removing dead server with same hostname and port of rs which is trying to check in before

    // master initialization. See HBASE-5916.

    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();

    if (this.cpHost != null) {

      // don't let cp initialization errors kill the master

      try {

        this.cpHost.postStartMaster();

      } catch (IOException ioe) {

        LOG.error("Coprocessor postStartMaster() hook failed", ioe);

      }

    }

  }

  /**

   * Create a {@link ServerManager} instance.

   * @param master

   * @param services

   * @return An instance of {@link ServerManager}

   * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException

   * @throws IOException

   */

  ServerManager createServerManager(final Server master,

      final MasterServices services)

  throws IOException {

    // We put this out here in a method so can do a Mockito.spy and stub it out

    // w/ a mocked up ServerManager.

    return new ServerManager(master, services);

  }

  /**

   * Check <code>hbase:meta</code> is assigned. If not, assign it.

   * @param status MonitoredTask

   * @param previouslyFailedMetaRSs

   * @throws InterruptedException

   * @throws IOException

   * @throws KeeperException

   */

  void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs)

      throws InterruptedException, IOException, KeeperException {

    // Work on meta region

    int assigned = 0;

    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);

    status.setStatus("Assigning hbase:meta region");

    RegionStates regionStates = assignmentManager.getRegionStates();

    regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO);

    boolean rit = this.assignmentManager

      .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);

    boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation(

      this.getShortCircuitConnection(), this.getZooKeeper(), timeout);

    ServerName currentMetaServer = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());

    if (!metaRegionLocation) {

      // Meta location is not verified. It should be in transition, or offline.

      // We will wait for it to be assigned in enableSSHandWaitForMeta below.

      assigned++;

      if (!rit) {

        // Assign meta since not already in transition

        if (currentMetaServer != null) {

          // If the meta server is not known to be dead or online,

          // just split the meta log, and don't expire it since this

          // could be a full cluster restart. Otherwise, we will think

          // this is a failover and lose previous region locations.

          // If it is really a failover case, AM will find out in rebuilding

          // user regions. Otherwise, we are good since all logs are split

          // or known to be replayed before user regions are assigned.

          if (serverManager.isServerOnline(currentMetaServer)) {

            LOG.info("Forcing expire of " + currentMetaServer);

            serverManager.expireServer(currentMetaServer);

          }

          splitMetaLogBeforeAssignment(currentMetaServer);

          previouslyFailedMetaRSs.add(currentMetaServer);

        }

        assignmentManager.assignMeta();

      }

    } else {

      // Region already assigned. We didn't assign it. Add to in-memory state.

      regionStates.updateRegionState(

        HRegionInfo.FIRST_META_REGIONINFO, State.OPEN, currentMetaServer);

      this.assignmentManager.regionOnline(

        HRegionInfo.FIRST_META_REGIONINFO, currentMetaServer);

    }

    enableMeta(TableName.META_TABLE_NAME);

    if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())

        && (!previouslyFailedMetaRSs.isEmpty())) {

      // replay WAL edits mode need new hbase:meta RS is assigned firstly

      status.setStatus("replaying log for Meta Region");

      this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);

    }

    // Make sure a hbase:meta location is set. We need to enable SSH here since

    // if the meta region server is died at this time, we need it to be re-assigned

    // by SSH so that system tables can be assigned.

    // No need to wait for meta is assigned = 0 when meta is just verified.

    enableServerShutdownHandler(assigned != 0);

    LOG.info("hbase:meta assigned=" + assigned + ", rit=" + rit +

      ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper()));

    status.setStatus("META assigned.");

  }

  void initNamespace() throws IOException {

    //create namespace manager

    tableNamespaceManager = new TableNamespaceManager(this);

    tableNamespaceManager.start();

  }

  boolean isCatalogJanitorEnabled() {

    return catalogJanitorChore != null ?

      catalogJanitorChore.getEnabled() : false;

  }

  private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {

    if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {

      // In log replay mode, we mark hbase:meta region as recovering in ZK

      Set<HRegionInfo> regions = new HashSet<HRegionInfo>();

      regions.add(HRegionInfo.FIRST_META_REGIONINFO);

      this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);

    } else {

      // In recovered.edits mode: create recovered edits file for hbase:meta server

      this.fileSystemManager.splitMetaLog(currentMetaServer);

    }

  }

  private void enableServerShutdownHandler(

      final boolean waitForMeta) throws IOException, InterruptedException {

    // If ServerShutdownHandler is disabled, we enable it and expire those dead

    // but not expired servers. This is required so that if meta is assigning to

    // a server which dies after assignMeta starts assignment,

    // SSH can re-assign it. Otherwise, we will be

    // stuck here waiting forever if waitForMeta is specified.

    if (!serverShutdownHandlerEnabled) {

      serverShutdownHandlerEnabled = true;

      this.serverManager.processQueuedDeadServers();

    }

    if (waitForMeta) {

      metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());

      // Above check waits for general meta availability but this does not

      // guarantee that the transition has completed

      this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);

    }

  }

  private void enableMeta(TableName metaTableName) {

    if (!this.assignmentManager.getTableStateManager().isTableState(metaTableName,

        ZooKeeperProtos.Table.State.ENABLED)) {

      this.assignmentManager.setEnabledTable(metaTableName);

    }

  }

  /**

   * This function returns a set of region server names under hbase:meta recovering region ZK node

   * @return Set of meta server names which were recorded in ZK

   * @throws KeeperException

   */

  private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {

    Set<ServerName> result = new HashSet<ServerName>();

    String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,

      HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());

    List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);

    if (regionFailedServers == null) return result;

    for(String failedServer : regionFailedServers) {

      ServerName server = ServerName.parseServerName(failedServer);

      result.add(server);

    }

    return result;

  }

  @Override

  public TableDescriptors getTableDescriptors() {

    return this.tableDescriptors;

  }

  @Override

  public ServerManager getServerManager() {

    return this.serverManager;

  }

  @Override

  public MasterFileSystem getMasterFileSystem() {

    return this.fileSystemManager;

  }

  /*

   * Start up all services. If any of these threads gets an unhandled exception

   * then they just die with a logged message.  This should be fine because

   * in general, we do not expect the master to get such unhandled exceptions

   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if

   *  need to install an unexpected exception handler.

   */

  private void startServiceThreads() throws IOException{

   // Start the executor service pools

   this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,

      conf.getInt("hbase.master.executor.openregion.threads", 5));

   this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,

      conf.getInt("hbase.master.executor.closeregion.threads", 5));

   this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,

      conf.getInt("hbase.master.executor.serverops.threads", 5));

   this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,

      conf.getInt("hbase.master.executor.serverops.threads", 5));

   this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,

      conf.getInt("hbase.master.executor.logreplayops.threads", 10));

   // We depend on there being only one instance of this executor running

   // at a time.  To do concurrency, would need fencing of enable/disable of

   // tables.

   // Any time changing this maxThreads to > 1, pls see the comment at

   // AccessController#postCreateTableHandler

   this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);

   // Start log cleaner thread

   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);

   this.logCleaner =

      new LogCleaner(cleanerInterval,

         this, conf, getMasterFileSystem().getFileSystem(),

         getMasterFileSystem().getOldLogDir());

         Threads.setDaemonThreadRunning(logCleaner.getThread(), getName() + ".oldLogCleaner");

   //start the hfile archive cleaner thread

    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);

    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()

        .getFileSystem(), archiveDir);

    Threads.setDaemonThreadRunning(hfileCleaner.getThread(),

      getName() + ".archivedHFileCleaner");

    serviceStarted = true;

    if (LOG.isTraceEnabled()) {

      LOG.trace("Started service threads");

    }

  }

  protected void stopServiceThreads() {

    if (masterJettyServer != null) {

      LOG.info("Stopping master jetty server");

      try {

        masterJettyServer.stop();

      } catch (Exception e) {

        LOG.error("Failed to stop master jetty server", e);

      }

    }

    super.stopServiceThreads();

    stopChores();

    // Wait for all the remaining region servers to report in IFF we were

    // running a cluster shutdown AND we were NOT aborting.

    if (!isAborted() && this.serverManager != null &&

        this.serverManager.isClusterShutdown()) {

      this.serverManager.letRegionServersShutdown();

    }

    if (LOG.isDebugEnabled()) {

      LOG.debug("Stopping service threads");

    }

    // Clean up and close up shop

    if (this.logCleaner!= null) this.logCleaner.interrupt();

    if (this.hfileCleaner != null) this.hfileCleaner.interrupt();

    if (this.activeMasterManager != null) this.activeMasterManager.stop();

    if (this.serverManager != null) this.serverManager.stop();

    if (this.assignmentManager != null) this.assignmentManager.stop();

    if (this.fileSystemManager != null) this.fileSystemManager.stop();

    if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");

  }

  private void stopChores() {

    if (this.balancerChore != null) {

      this.balancerChore.interrupt();

    }

    if (this.clusterStatusChore != null) {

      this.clusterStatusChore.interrupt();

    }

    if (this.catalogJanitorChore != null) {

      this.catalogJanitorChore.interrupt();

    }

    if (this.clusterStatusPublisherChore != null){

      clusterStatusPublisherChore.interrupt();

    }

  }

  /**

   * @return Get remote side's InetAddress

   * @throws UnknownHostException

   */

  InetAddress getRemoteInetAddress(final int port,

      final long serverStartCode) throws UnknownHostException {

    // Do it out here in its own little method so can fake an address when

    // mocking up in tests.

    InetAddress ia = RpcServer.getRemoteIp();

    // The call could be from the local regionserver,

    // in which case, there is no remote address.

    if (ia == null && serverStartCode == startcode) {

      InetSocketAddress isa = rpcServices.getSocketAddress();

      if (isa != null && isa.getPort() == port) {

        ia = isa.getAddress();

      }

    }

    return ia;

  }

  /**

   * @return Maximum time we should run balancer for

   */

  private int getBalancerCutoffTime() {

    int balancerCutoffTime =

      getConfiguration().getInt("hbase.balancer.max.balancing", -1);

    if (balancerCutoffTime == -1) {

      // No time period set so create one

      int balancerPeriod =

        getConfiguration().getInt("hbase.balancer.period", 300000);

      balancerCutoffTime = balancerPeriod;

      // If nonsense period, set it to balancerPeriod

      if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;

    }

    return balancerCutoffTime;

  }

  public boolean balance() throws IOException {

    // if master not initialized, don't run balancer.

    if (!this.initialized) {

      LOG.debug("Master has not been initialized, don't run balancer.");

      return false;

    }

    // Do this call outside of synchronized block.

    int maximumBalanceTime = getBalancerCutoffTime();

    synchronized (this.balancer) {

      // If balance not true, don't run balancer.

      if (!this.loadBalancerTracker.isBalancerOn()) return false;

      // Only allow one balance run at at time.

      if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {

        Map<String, RegionState> regionsInTransition =

          this.assignmentManager.getRegionStates().getRegionsInTransition();

        LOG.debug("Not running balancer because " + regionsInTransition.size() +

          " region(s) in transition: " + org.apache.commons.lang.StringUtils.

            abbreviate(regionsInTransition.toString(), 256));

        return false;

      }

      if (this.serverManager.areDeadServersInProgress()) {

        LOG.debug("Not running balancer because processing dead regionserver(s): " +

          this.serverManager.getDeadServers());

        return false;

      }

      if (this.cpHost != null) {

        try {

          if (this.cpHost.preBalance()) {

            LOG.debug("Coprocessor bypassing balancer request");

            return false;

          }

        } catch (IOException ioe) {

          LOG.error("Error invoking master coprocessor preBalance()", ioe);

          return false;

        }

      }

      Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =

        this.assignmentManager.getRegionStates().getAssignmentsByTable();

      List<RegionPlan> plans = new ArrayList<RegionPlan>();

      //Give the balancer the current cluster state.

      this.balancer.setClusterStatus(getClusterStatus());

      for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {

        List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);

        if (partialPlans != null) plans.addAll(partialPlans);

      }

      long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;

      int rpCount = 0;  // number of RegionPlans balanced so far

      long totalRegPlanExecTime = 0;

      if (plans != null && !plans.isEmpty()) {

        for (RegionPlan plan: plans) {

          LOG.info("balance " + plan);

          long balStartTime = System.currentTimeMillis();

          //TODO: bulk assign

          this.assignmentManager.balance(plan);

          totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;

          rpCount++;

          if (rpCount < plans.size() &&

              // if performing next balance exceeds cutoff time, exit the loop

              (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {

            //TODO: After balance, there should not be a cutoff time (keeping it as a security net for now)

            LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +

              maximumBalanceTime);

            break;

          }

        }

      }

      if (this.cpHost != null) {

        try {

          this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);

        } catch (IOException ioe) {

          // balancing already succeeded so don't change the result

          LOG.error("Error invoking master coprocessor postBalance()", ioe);

        }

      }

    }

    // If LoadBalancer did not generate any plans, it means the cluster is already balanced.

    // Return true indicating a success.

    return true;

  }

  /**

   * @return Client info for use as prefix on an audit log string; who did an action

   */

  String getClientIdAuditPrefix() {

    return "Client=" + RequestContext.getRequestUserName() + "/" +

      RequestContext.get().getRemoteAddress();

  }

  /**

   * Switch for the background CatalogJanitor thread.

   * Used for testing.  The thread will continue to run.  It will just be a noop

   * if disabled.

   * @param b If false, the catalog janitor won't do anything.

   */

  public void setCatalogJanitorEnabled(final boolean b) {

    this.catalogJanitorChore.setEnabled(b);

  }

  @Override

  public void dispatchMergingRegions(final HRegionInfo region_a,

      final HRegionInfo region_b, final boolean forcible) throws IOException {

    checkInitialized();

    this.service.submit(new DispatchMergingRegionHandler(this,

        this.catalogJanitorChore, region_a, region_b, forcible));

  }

  void move(final byte[] encodedRegionName,

      final byte[] destServerName) throws HBaseIOException {

    RegionState regionState = assignmentManager.getRegionStates().

      getRegionState(Bytes.toString(encodedRegionName));

    if (regionState == null) {

      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));

    }

    HRegionInfo hri = regionState.getRegion();

    ServerName dest;

    if (destServerName == null || destServerName.length == 0) {

      LOG.info("Passed destination servername is null/empty so " +

        "choosing a server at random");

      final List<ServerName> destServers = this.serverManager.createDestinationServersList(

        regionState.getServerName());

      dest = balancer.randomAssignment(hri, destServers);

    } else {

      dest = ServerName.valueOf(Bytes.toString(destServerName));

      if (dest.equals(regionState.getServerName())) {

        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()

          + " because region already assigned to the same server " + dest + ".");

        return;

      }

    }

    // Now we can do the move

    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);

    try {

      checkInitialized();

      if (this.cpHost != null) {

        if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {

          return;

        }

      }

      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");

      this.assignmentManager.balance(rp);

      if (this.cpHost != null) {

        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());

      }

    } catch (IOException ioe) {

      if (ioe instanceof HBaseIOException) {

        throw (HBaseIOException)ioe;

      }

      throw new HBaseIOException(ioe);

    }

  }

  @Override

  public void createTable(HTableDescriptor hTableDescriptor,

      byte [][] splitKeys) throws IOException {

    if (isStopped()) {

      throw new MasterNotRunningException();

    }

    String namespace = hTableDescriptor.getTableName().getNamespaceAsString();

    getNamespaceDescriptor(namespace); // ensure namespace exists

    HRegionInfo[] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);

    checkInitialized();

    sanityCheckTableDescriptor(hTableDescriptor);

    if (cpHost != null) {

      cpHost.preCreateTable(hTableDescriptor, newRegions);

    }

    LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);

    this.service.submit(new CreateTableHandler(this,

      this.fileSystemManager, hTableDescriptor, conf,

      newRegions, this).prepare());

    if (cpHost != null) {

      cpHost.postCreateTable(hTableDescriptor, newRegions);

    }

  }

  /**

   * Checks whether the table conforms to some sane limits, and configured

   * values (compression, etc) work. Throws an exception if something is wrong.

   * @throws IOException

   */

  private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {

    final String CONF_KEY = "hbase.table.sanity.checks";

    if (!conf.getBoolean(CONF_KEY, true)) {

      return;

    }

    String tableVal = htd.getConfigurationValue(CONF_KEY);

    if (tableVal != null && !Boolean.valueOf(tableVal)) {

      return;

    }

    // check max file size

    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit