* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

    FServerStatusProtocol, FMasterMonitorProtocol, FMasterServices, Server {

  private static final Log LOG = LogFactory.getLog(FMaster.class.getName());

  /**

   * MASTER is name of the webapp and the attribute name used stuffing this

   * instance into web context.

   **/

  public static final String MASTER = "master";

  private ClusterId clusterId;

  /** The configuration for the Master **/

  private final Configuration conf;

  /** server for the web ui **/

  private InfoServer infoServer;

  /** Our zk client. **/

  private ZooKeeperWatcher zooKeeper;

  /** Manager and zk listener for master election. **/

  private ActiveMasterManager activeMasterManager;

  /** FServer tracker **/

  private FServerTracker fserverTracker;

  /** Draining fserver tracker **/

  private DrainingServerTracker drainingServerTracker;

  /** Tracker for load balancer state **/

  private LoadBalancerTracker loadBalancerTracker;

  /** RPC server for the FMaster **/

  private final RpcServer rpcServer;

  /** Table Lock Manager **/

  private TableLockManager tableLockManager;

  /**

   * Set after we've called WaspServer#openServer and ready to receive RPCs. Set

   * back to false after we stop rpcServer. Used by tests.

   **/

  private volatile boolean rpcServerOpen = false;

  /**

   * This servers address.

   */

  private final InetSocketAddress isa;

  // Metrics for the FMaster

  private final MetricsMaster metricsMaster;

  // TODO Metrics for the FMaster

  // private final MasterMetrics metrics;

  // server manager to deal with fserver info

  private FServerManager serverManager;

  // manager of assignment nodes in zookeeper

  AssignmentManager assignmentManager;

  // Cluster status zk tracker and local setter

  private ClusterStatusTracker clusterStatusTracker;

  // This flag is for stopping this Master instance. Its set when we are

  // stopping or aborting

  private volatile boolean stopped = false;

  // Set on abort -- usually failure of our zk session.

  private volatile boolean abort = false;

  // flag set after we become the active master (used for testing)

  private volatile boolean isActiveMaster = false;

  // flag set after we complete initialization once active (used for testing)

  private volatile boolean initialized = false;

  // Instance of the wasp executor service.

  ExecutorService executorService;

  private LoadBalancer balancer;

  private Thread balancerChore;

  private Thread clusterStatusChore;

  private StorageCleanChore storageCleaner;

  private CatalogJanitor catalogJanitorChore;

  private final ServerName serverName;

  // Time stamps for when a hmaster was started and when it became active

  private long masterStartTime;

  private long masterActiveTime;

  /** time interval for emitting metrics values */

  private final int msgInterval;

  /**

   * MX Bean for MasterInfo

   */

  private ObjectName mxBean = null;

  /**

   * Initializes the FMaster. The steps are as follows:

   * <p>

   * <ol>

   * <li>Initialize FMaster RPC and address

   * <li>Connect to ZooKeeper.

   * </ol>

   * <p>

   * Remaining steps of initialization occur in {@link #run()} so that they run

   * in their own thread rather than within the context of the constructor.

   *

   * @throws InterruptedException

   */

  public FMaster(final Configuration conf) throws IOException, KeeperException,

      InterruptedException {

    this.conf = new Configuration(conf);

    // Set how many times to retry talking to another server over HConnection.

    FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG);

    // Server to handle client requests.

    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(

        conf.get("wasp.master.dns.interface", "default"),

        conf.get("wasp.master.dns.nameserver", "default")));

    int port = conf.getInt(FConstants.MASTER_PORT,

        FConstants.DEFAULT_MASTER_PORT);

    // Creation of a ISA will force a resolve.

    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);

    if (initialIsa.getAddress() == null) {

      throw new IllegalArgumentException("Failed resolve of " + initialIsa);

    }

    this.rpcServer = WaspRPC.getServer(FMaster.class, this, new Class<?>[]{

        FMasterMonitorProtocol.class, FMasterAdminProtocol.class,

        FServerStatusProtocol.class, FMetaServerProtocol.class},

        initialIsa.getHostName(), // BindAddress is IP we got for this server.

        initialIsa.getPort(), conf);

    // Set our address.

    this.isa = this.rpcServer.getListenerAddress();

    this.serverName = new ServerName(this.isa.getHostName(),

        this.isa.getPort(), System.currentTimeMillis());

    // set the thread name now we have an address

    setName(MASTER + "-" + this.serverName.toString());

    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(),

        this, true);

    // metrics interval: using the same property as fserver.

    this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000);

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));

  }

  /**

   * Stall startup if we are designated a backup master; i.e. we want someone

   * else to become the master before proceeding.

   *

   * @param c

   * @param amm

   * @throws InterruptedException

   */

  private static void stallIfBackupMaster(final Configuration c,

      final ActiveMasterManager amm) throws InterruptedException {

    // If we're a backup master, stall until a primary to writes his address

    if (!c.getBoolean(FConstants.MASTER_TYPE_BACKUP,

        FConstants.DEFAULT_MASTER_TYPE_BACKUP)) {

      return;

    }

    LOG.debug("FMaster started in backup mode.  "

        + "Stalling until master znode is written.");

    // This will only be a minute or so while the cluster starts up,

    // so don't worry about setting watches on the parent znode

    while (!amm.isActiveMaster()) {

      LOG.debug("Waiting for master address ZNode to be written "

          + "(Also watching cluster state node)");

      Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));

    }

  }

  MetricsMaster getMetrics() {

    return metricsMaster;

  }

  /**

   * Main processing loop for the FMaster.

   * <ol>

   * <li>Block until becoming active master

   * <li>Finish initialization via finishInitialization(MonitoredTask)

   * <li>Enter loop until we are stopped

   * <li>Stop services and perform cleanup once stopped

   * </ol>

   */

  @Override

  public void run() {

    MonitoredTask startupStatus = TaskMonitor.get().createStatus(

        "Master startup");

    startupStatus.setDescription("Master startup");

    masterStartTime = System.currentTimeMillis();

    try {

      /*

       * Block on becoming the active master.

       *

       * We race with other masters to write our address into ZooKeeper. If we

       * succeed, we are the primary/active master and finish initialization.

       *

       * If we do not succeed, there is another active master and we should now

       * wait until it dies to try and become the next active master. If we do

       * not succeed on our first attempt, this is no longer a cluster startup.

       */

      becomeActiveMaster(startupStatus);

      // We are either the active master or we were asked to shutdown

      if (!this.stopped) {

        finishInitialization(startupStatus, false);

        loop();

      }

    } catch (Throwable t) {

      abort("Unhandled exception. Starting shutdown.", t);

    } finally {

      startupStatus.cleanup();

      stopChores();

      // Wait for all the remaining fservers to report in IFF we were

      // running a cluster shutdown AND we were NOT aborting.

      if (!this.abort && this.serverManager != null

          && this.serverManager.isClusterShutdown()) {

        this.serverManager.letFServersShutdown();

      }

      stopServiceThreads();

      // Stop services started for both backup and active masters

      if (this.activeMasterManager != null)

        this.activeMasterManager.stop();

      if (this.serverManager != null)

        this.serverManager.stop();

      if (this.assignmentManager != null)

        this.assignmentManager.stop();

      this.zooKeeper.close();

    }

    LOG.info("FMaster main thread exiting");

  }

  /**

   * Try becoming active master.

   *

   * @param startupStatus

   * @return True if we could successfully become the active master.

   * @throws InterruptedException

   */

  private boolean becomeActiveMaster(MonitoredTask startupStatus)

      throws InterruptedException {

    this.activeMasterManager = new ActiveMasterManager(zooKeeper,

        this.serverName, this);

    this.zooKeeper.registerListener(activeMasterManager);

    stallIfBackupMaster(this.conf, this.activeMasterManager);

    // The ClusterStatusTracker is setup before the other

    // ZKBasedSystemTrackers because it's needed by the activeMasterManager

    // to check if the cluster should be shutdown.

    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);

    this.clusterStatusTracker.start();

    return this.activeMasterManager.blockUntilBecomingActiveMaster(

        startupStatus, this.clusterStatusTracker);

  }

  /**

   * Initialize all ZK based system trackers.

   *

   * @throws java.io.IOException

   * @throws InterruptedException

   */

  private void initializeZKBasedSystemTrackers() throws IOException,

      InterruptedException, KeeperException {

    this.balancer = LoadBalancerFactory.getLoadBalancer(conf);

    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);

    this.loadBalancerTracker.start();

    this.assignmentManager = new AssignmentManager(this, serverManager,

        this.balancer, this.executorService, this.metricsMaster);

    zooKeeper.registerListenerFirst(assignmentManager);

    this.fserverTracker = new FServerTracker(zooKeeper, this,

        this.serverManager);

    this.fserverTracker.start();

    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,

        this.serverManager);

    this.drainingServerTracker.start();

    this.tableLockManager = new TableLockManager();

    // Set the cluster as up. If new FSs, they'll be waiting on this before

    // going ahead with their startup.

    boolean wasUp = this.clusterStatusTracker.isClusterUp();

    if (!wasUp)

      this.clusterStatusTracker.setClusterUp();

    LOG.info("Server active/primary master; "

        + this.serverName

        + ", sessionid=0x"

        + Long.toHexString(this.zooKeeper.getRecoverableZooKeeper()

            .getSessionId()) + ", cluster-up flag was=" + wasUp);

  }

  // Check if we should stop every 100ms

  private Sleeper stopSleeper = new Sleeper(100, this);

  private void loop() {

    long lastMsgTs = 0l;

    long now = 0l;

    while (!this.stopped) {

      now = System.currentTimeMillis();

      if ((now - lastMsgTs) >= this.msgInterval) {

        doMetrics();

        lastMsgTs = System.currentTimeMillis();

      }

      stopSleeper.sleep();

    }

  }

  /**

   * Emit the FMaster metrics, such as entityGroup in transition metrics.

   * Surrounding in a try block just to be sure metrics doesn't abort FMaster.

   */

  private void doMetrics() {

    try {

      this.assignmentManager.updateEntityGroupsInTransitionMetrics();

    } catch (Throwable e) {

      LOG.error("Couldn't update metrics: " + e.getMessage());

    }

  }

  /**

   * Finish initialization of FMaster after becoming the primary master.

   *

   * <ol>

   * <li>Initialize master components - server manager, assignment manager,

   * fserver tracker, etc</li>

   * <li>Start necessary service threads - rpc server, info server, executor

   * services, etc</li>

   * <li>Set cluster as UP in ZooKeeper</li>

   * <li>Wait for FServers to check-in</li>

   * <li>

   * <li>Handle either fresh cluster start or master failover</li>

   * </ol>

   *

   * @param masterRecovery

   *

   * @throws java.io.IOException

   * @throws InterruptedException

   * @throws org.apache.zookeeper.KeeperException

   */

  private void finishInitialization(MonitoredTask status, boolean masterRecovery)

      throws IOException, InterruptedException, KeeperException {

    isActiveMaster = true;

    // init cluster ID from zk

    status.setStatus("Publishing Cluster ID in ZooKeeper");

    this.clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);

    if (this.clusterId == null) {

      this.clusterId = new ClusterId();

      ZKClusterId.setClusterId(this.zooKeeper, this.clusterId);

    }

    /*

     * We are active master now... go initialize components we need to run.

     * Note, there may be dross in zk from previous runs; it'll get addressed

     * below after we determine if cluster startup or failover.

     */

    this.masterActiveTime = System.currentTimeMillis();

    if (!masterRecovery) {

      this.executorService = new ExecutorService(getServerName().toString());

      this.serverManager = createServerManager(this, this);

    }

    // check the FMeta if we should create and init it

    FMetaUtil.checkAndInit(conf);

    status.setStatus("Initializing ZK system trackers");

    initializeZKBasedSystemTrackers();

    if (!masterRecovery) {

      // start up all service threads.

      status.setStatus("Initializing master service threads");

      startServiceThreads();

    }

    // Wait for fservers to report in.

    this.serverManager.waitForFServers(status);

    // Check zk for fservers that are up but didn't register

    for (ServerName sn : this.fserverTracker.getOnlineServers()) {

      if (!this.serverManager.isServerOnline(sn)) {

        // Not registered; add it.

        LOG.info("Registering server found up in zk but who has not yet "

            + "reported in: " + sn);

        this.serverManager.recordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD);

      }

    }

    if (!masterRecovery) {

      this.assignmentManager.startTimeOutMonitor();

    }

    this.balancer.setMasterServices(this);

    // Fix up assignment manager status

    status.setStatus("Starting assignment manager");

    this.assignmentManager.joinCluster();

    this.balancer.setClusterStatus(getClusterStatus());

    // Fixing up missing daughters if any

    status.setStatus("Fixing up missing daughters");

    fixupDaughters(status);

    if (!masterRecovery) {

      // Start balancer and meta catalog janitor after meta and entityGroups

      // have

      // been assigned.

      status.setStatus("Starting balancer and catalog janitor");

      this.clusterStatusChore = getAndStartClusterStatusChore(this);

      this.balancerChore = getAndStartBalancerChore(this);

      this.catalogJanitorChore = new CatalogJanitor(this, this);

      startCatalogJanitorChore();

    }

    status.markComplete("Initialization successful");

    LOG.info("Master has completed initialization");

    initialized = true;

    // clear the dead servers with same host name and port of online server

    // because we are not

    // removing dead server with same hostname and port of rs which is trying to

    // check in before

    // master initialization.

    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();

  }

  /**

   * Useful for testing purpose also where we have master restart scenarios.

   */

  protected void startCatalogJanitorChore() {

    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());

  }

  /**

   * Create a {@link FServerManager} instance.

   *

   * @param master

   * @param services

   * @return An instance of {@link FServerManager}

   * @throws com.alibaba.wasp.ZooKeeperConnectionException

   * @throws java.io.IOException

   */

  FServerManager createServerManager(final Server master,

      final FMasterServices services) throws IOException {

    // We put this out here in a method so can do a Mockito.spy and stub it out

    // w/ a mocked up ServerManager.

    return new FServerManager(master, services);

  }

  void fixupDaughters(final MonitoredTask status) throws IOException {

    Map<EntityGroupInfo, Result> offlineSplitParents = FMetaReader

        .getOfflineSplitParents(this.conf);

    // Now work on our list of found parents. See if any we can clean up.

    int fixups = 0;

    for (Map.Entry<EntityGroupInfo, Result> e : offlineSplitParents.entrySet()) {

      ServerName sn = EntityGroupInfo.getServerName(e.getValue());

      if (!serverManager.isServerDead(sn)) { // Otherwise, let SSH take care of

                                             // it

        fixups += ServerShutdownHandler.fixupDaughters(e.getValue(),

            assignmentManager, this.conf);

      }

    }

    if (fixups != 0) {

      LOG.info("Scanned the catalog and fixed up " + fixups

          + " missing daughter entityGroup(s)");

    }

  }

  public long getProtocolVersion(String protocol, long clientVersion) {

    if (FMasterMonitorProtocol.class.getName().equals(protocol)) {

      return FMasterMonitorProtocol.VERSION;

    } else if (FMasterAdminProtocol.class.getName().equals(protocol)) {

      return FMasterAdminProtocol.VERSION;

    } else if (FServerStatusProtocol.class.getName().equals(protocol)) {

      return FServerStatusProtocol.VERSION;

    } else if (FMetaServerProtocol.class.getName().equals(protocol)) {

      return FMetaServerProtocol.VERSION;

    }

    // unknown protocol

    LOG.warn("Version requested for unimplemented protocol: " + protocol);

    return -1;

  }

  /** @return InfoServer object. Maybe null. */

  public InfoServer getInfoServer() {

    return this.infoServer;

  }

  @Override

  public Configuration getConfiguration() {

    return this.conf;

  }

  @Override

  public FServerManager getFServerManager() {

    return this.serverManager;

  }

  @Override

  public ExecutorService getExecutorService() {

    return this.executorService;

  }

  @Override

  public TableLockManager getTableLockManager() {

    return this.tableLockManager;

  }

  /**

   * Get the ZK wrapper object - needed by master_jsp.java

   *

   * @return the zookeeper wrapper

   */

  public ZooKeeperWatcher getZooKeeperWatcher() {

    return this.zooKeeper;

  }

  /*

   * Start up all services. If any of these threads gets an unhandled exception

   * then they just die with a logged message. This should be fine because in

   * general, we do not expect the master to get such unhandled exceptions as

   * OOMEs; it should be lightly loaded. See what FServer does if need to

   * install an unexpected exception handler.

   */

  void startServiceThreads() throws IOException {

    // Start the executor service pools

    this.executorService.startExecutorService(

        ExecutorType.MASTER_OPEN_ENTITYGROUP,

        conf.getInt("wasp.master.executor.openentitygroup.threads", 5));

    this.executorService.startExecutorService(

        ExecutorType.MASTER_CLOSE_ENTITYGROUP,

        conf.getInt("wasp.master.executor.closeentitygroup.threads", 5));

    this.executorService.startExecutorService(

        ExecutorType.MASTER_SERVER_OPERATIONS,

        conf.getInt("wasp.master.executor.serverops.threads", 3));

    // We depend on there being only one instance of this executor running

    // at a time. To do concurrency, would need fencing of enable/disable of

    // tables.

    this.executorService.startExecutorService(

        ExecutorType.MASTER_TABLE_OPERATIONS, 1);

    // Start storage table cleaner thread

   String n = Thread.currentThread().getName();

   int cleanerInterval = conf.getInt("wasp.master.cleaner.interval", 60 * 60 * 1000);

   this.storageCleaner =

      new StorageCleanChore(cleanerInterval,

         this, conf, AbstractMetaService.getService(conf));

         Threads.setDaemonThreadRunning(storageCleaner.getThread(), n + ".storageCleaner");

    // Put up info server.

    int port = this.conf.getInt(FConstants.MASTER_INFO_PORT, FConstants.DEFAULT_MASTER_INFOPORT);

    if (port >= 0) {

      String a = this.conf.get("wasp.master.info.bindAddress", "0.0.0.0");

      this.infoServer = new InfoServer(MASTER, a, port, false, this.conf);

      this.infoServer.addServlet("status", "/master-status",

          FMasterStatusServlet.class);

      this.infoServer.addServlet("dump", "/dump", FMasterDumpServlet.class);

      this.infoServer.setAttribute(MASTER, this);

      this.infoServer.start();

    }

    // Start allowing requests to happen.

    this.rpcServer.openServer();

    this.rpcServerOpen = true;

    if (LOG.isDebugEnabled()) {

      LOG.debug("Started service threads");

    }

  }

  /**

   * Use this when trying to figure when its ok to send in rpcs. Used by tests.

   *

   * @return True if we have successfully run {@link WaspServer#openServer()}

   */

  boolean isRpcServerOpen() {

    return this.rpcServerOpen;

  }

  private void stopServiceThreads() {

    if (LOG.isDebugEnabled()) {

      LOG.debug("Stopping service threads");

    }

    if (this.rpcServer != null)

      this.rpcServer.stop();

    this.rpcServerOpen = false;

    // Clean up and close up shop

    if (this.storageCleaner!= null) this.storageCleaner.interrupt();

    if (this.infoServer != null) {

      LOG.info("Stopping infoServer");

      try {

        this.infoServer.stop();

      } catch (Exception ex) {

        ex.printStackTrace();

      }

    }

    if (this.executorService != null)

      this.executorService.shutdown();

  }

  private static Thread getAndStartClusterStatusChore(FMaster master) {

    if (master == null || master.balancer == null) {

      return null;

    }

    Chore chore = new ClusterStatusChore(master, master.balancer);

    return Threads.setDaemonThreadRunning(chore.getThread());

  }

  private static Thread getAndStartBalancerChore(final FMaster master) {

    // Start up the load balancer chore

    Chore chore = new BalancerChore(master);

    return Threads.setDaemonThreadRunning(chore.getThread());

  }

  private void stopChores() {

    if (this.balancerChore != null) {

      this.balancerChore.interrupt();

    }

    if (this.clusterStatusChore != null) {

      this.clusterStatusChore.interrupt();

    }

    if (this.catalogJanitorChore != null) {

      this.catalogJanitorChore.interrupt();

    }

  }

  @Override

  public FServerStartupResponse fServerStartup(RpcController controller,

      FServerStartupRequest request) throws ServiceException {

    // Register with server manager

    try {

      InetAddress ia = getRemoteInetAddress(request.getPort(),

          request.getServerStartCode());

      ServerName sn = this.serverManager.fserverStartup(ia, request.getPort(),

          request.getServerStartCode(), request.getServerCurrentTime());

      // Send back some config info

      FServerStartupResponse.Builder resp = createConfigurationSubset();

      StringStringPair.Builder entry = StringStringPair.newBuilder()

          .setName(FConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)

          .setValue(sn.getHostname());

      resp.addMapEntries(entry.build());

      return resp.build();

    } catch (IOException ioe) {

      throw new ServiceException(ioe);

    }

  }

  /**

   * @return Get remote side's InetAddress

   * @throws java.net.UnknownHostException

   */

  InetAddress getRemoteInetAddress(final int port, final long serverStartCode)

      throws UnknownHostException {

    // Do it out here in its own little method so can fake an address when

    // mocking up in tests.

    return NettyServer.getRemoteAddress().getAddress();

  }

  /**

   * @return Subset of configuration to pass initializing FServers. e.g. hbase

   *         address

   */

  protected FServerStartupResponse.Builder createConfigurationSubset() {

    FServerStartupResponse.Builder resp = addConfig(

        FServerStartupResponse.newBuilder(), FConstants.ZOOKEEPER_QUORUM);

    return addConfig(resp, FConstants.ZOOKEEPER_ZNODE_PARENT);

  }

  private FServerStartupResponse.Builder addConfig(

      final FServerStartupResponse.Builder resp, final String key) {

    StringStringPair.Builder entry = StringStringPair.newBuilder().setName(key)

        .setValue(this.conf.get(key));

    resp.addMapEntries(entry.build());

    return resp;

  }

  @Override

  public FServerReportResponse fServerReport(RpcController controller,

      FServerReportRequest request) throws ServiceException {

    try {

      ServerLoadProtos sl = request.getLoad();

      this.serverManager.fserverReport(

          ProtobufUtil.toServerName(request.getServer()), new ServerLoad(sl));

      // Up our metrics.

      if (sl != null && this.metricsMaster != null) {

        this.metricsMaster.incrementRequests(sl.getTotalNumberOfRequests());

      }

    } catch (IOException ioe) {

      throw new ServiceException(ioe);

    }

    return FServerReportResponse.newBuilder().build();

  }

  @Override

  public ReportRSFatalErrorResponse reportRSFatalError(

      RpcController controller, ReportRSFatalErrorRequest request)

      throws ServiceException {

    String errorText = request.getErrorMessage();

    ServerName sn = ProtobufUtil.toServerName(request.getServer());

    String msg = "FServer " + sn.toString() + " reported a fatal error:\n"

        + errorText;

    LOG.error(msg);

    return ReportRSFatalErrorResponse.newBuilder().build();

  }

  /**

   * @return Maximum time we should run balancer for

   */

  private int getBalancerCutoffTime() {

    int balancerCutoffTime = getConfiguration().getInt(

        "wasp.balancer.max.balancing", -1);

    if (balancerCutoffTime == -1) {

      // No time period set so create one -- do half of balancer period.

      int balancerPeriod = getConfiguration().getInt("wasp.balancer.period",

          300000);

      balancerCutoffTime = balancerPeriod / 2;

      // If nonsense period, set it to balancerPeriod

      if (balancerCutoffTime <= 0)

        balancerCutoffTime = balancerPeriod;

    }

    return balancerCutoffTime;

  }

  public boolean balance() throws IOException {

    // if master not initialized, don't run balancer.

    if (!this.initialized) {

      LOG.debug("Master has not been initialized, don't run balancer.");

      return false;

    }

    // If balance not true, don't run balancer.

    if (!this.loadBalancerTracker.isBalancerOn())

      return false;

    // Do this call outside of synchronized block.

    int maximumBalanceTime = getBalancerCutoffTime();

    long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;

    boolean balancerRan;

    synchronized (this.balancer) {

      // Only allow one balance run at at time.

      if (this.assignmentManager.getEntityGroupStates()

          .isEntityGroupsInTransition()) {

        Map<String, EntityGroupState> entityGroupsInTransition = this.assignmentManager

            .getEntityGroupStates().getEntityGroupsInTransition();

        LOG.debug("Not running balancer because "

            + entityGroupsInTransition.size()

            + " entityGroup(s) in transition: "

            + org.apache.commons.lang.StringUtils.abbreviate(

                entityGroupsInTransition.toString(), 256));

        return false;

      }

      if (this.serverManager.areDeadServersInProgress()) {

        LOG.debug("Not running balancer because processing dead fserver(s): "

            + this.serverManager.getDeadServers());

        return false;

      }

      Map<String, Map<ServerName, List<EntityGroupInfo>>> assignmentsByTable = this.assignmentManager

          .getEntityGroupStates().getAssignmentsByTable();

      List<EntityGroupPlan> plans = new ArrayList<EntityGroupPlan>();

      // Give the balancer the current cluster state.

      this.balancer.setClusterStatus(getClusterStatus());

      for (Map<ServerName, List<EntityGroupInfo>> assignments : assignmentsByTable

          .values()) {

        List<EntityGroupPlan> partialPlans = this.balancer

            .balanceCluster(assignments);

        if (partialPlans != null)

          plans.addAll(partialPlans);

      }

      int rpCount = 0; // number of EntityGroupPlans balanced so far

      long totalRegPlanExecTime = 0;

      balancerRan = plans != null;

      if (plans != null && !plans.isEmpty()) {

        for (EntityGroupPlan plan : plans) {

          LOG.info("balance " + plan);

          long balStartTime = System.currentTimeMillis();

          this.assignmentManager.balance(plan);

          totalRegPlanExecTime += System.currentTimeMillis() - balStartTime;

          rpCount++;

          if (rpCount < plans.size() &&

          // if performing next balance exceeds cutoff time, exit the loop

              (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {

            LOG.debug("No more balancing till next balance run; maximumBalanceTime="

                + maximumBalanceTime);

            break;

          }

        }

      }

    }

    return balancerRan;

  }

  @Override

  public BalanceResponse balance(RpcController c, BalanceRequest request)

      throws ServiceException {

    try {

      return BalanceResponse.newBuilder().setBalancerRan(balance()).build();

    } catch (IOException e) {

      throw new ServiceException(e);

    }

  }

  enum BalanceSwitchMode {

    SYNC,

      ASYNC

  }

  /**

   * Assigns balancer switch according to BalanceSwitchMode

   *

   * @param b

   *          new balancer switch

   * @param mode

   *          BalanceSwitchMode

   * @return old balancer switch

   */

  public boolean switchBalancer(final boolean b, BalanceSwitchMode mode)

      throws IOException {

    boolean oldValue = this.loadBalancerTracker.isBalancerOn();

    boolean newValue = b;

    try {

      try {

        if (mode == BalanceSwitchMode.SYNC) {

          synchronized (this.balancer) {

            this.loadBalancerTracker.setBalancerOn(newValue);

          }

        } else {

          this.loadBalancerTracker.setBalancerOn(newValue);

        }

      } catch (KeeperException ke) {

        throw new IOException(ke);

      }

      LOG.info("BalanceSwitch=" + newValue);

    } catch (IOException ioe) {

      LOG.warn("Error flipping balance switch", ioe);

    }

    return oldValue;