* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* FServer makes a set of EntityGroups available to clients. It checks in with

* the FMaster. There are many EntityGroups in a single wasp deployment.

*/

    FServerServices {

  public static final Log LOG = LogFactory.getLog(FServer.class);

  // Set when a report to the master comes back with a message asking us to

  // shutdown. Also set by call to stop when debugging or running action tests

  // of FServer in isolation.

  protected volatile boolean stopped = false;

  // A state before we go into stopped state. At this stage we're closing user

  // space entityGroups.

  private boolean stopping = false;

  // Go down hard. Used if file system becomes unavailable and also in

  // debugging and action tests.

  protected volatile boolean abortRequested;

  // A sleeper that sleeps for msgInterval.

  private final Sleeper sleeper;

  private MetricsFServer metricsFServer;

  // Server to handle client requests. Default access so can be accessed by

  // action tests.

  RpcServer rpcServer;

  // Instance of the wasp executor service.

  private ExecutorService service;

  private java.util.concurrent.ExecutorService pool;

  private BaseDriver driver;

  // Cluster Status Tracker

  private ClusterStatusTracker clusterStatusTracker;

  private TableSchemaCacheReader tableSchemaReader;

  // Info server. Default access so can be used by unit tests. FSERVER

  // is name of the webapp and the attribute name used stuffing this instance

  // into web context.

  InfoServer infoServer;

  public static final String FSERVER = "fserver";

  /** fserver configuration name */

  public static final String FSERVER_CONF = "fserver_conf";

  private FServerStatusProtocol waspMaster;

  // Port we put up the webui on.

  protected int webuiport = -1;

  /**

   * Space is reserved in FS constructor and then released when aborting to

   * recover from an OOME.

   */

  private final LinkedList<byte[]> reservedSpace = new LinkedList<byte[]>();

  protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

  protected final Configuration conf;

  private final int rpcTimeout;

  private final int msgInterval;

  /**

   * The lease timeout period for client scanners (milliseconds).

   */

  private final int scannerLeaseTimeoutPeriod;

  private final Random rand = new Random();

  /**

   * MX Bean for FServerInfo

   */

  private ObjectName mxBean = null;

  /**

   * This servers start code.

   */

  private final long startcode;

  // flag set after we're done setting up server threads

  protected volatile boolean isOnline;

  // zookeeper connection and watcher

  private ZooKeeperWatcher zooKeeper;

  /**

   * The server name the Master sees us as. Its made from the hostname the

   * master passes us, port, and server startcode. Gets set after registration

   * against Master. The hostname can differ from the hostname in {@link #isa}

   * but usually doesn't if both servers resolve .

   */

  private ServerName serverNameFromMasterPOV;

  private final InetSocketAddress isa;

  protected final int numEntityGroupsToReport;

  private final StorageActionManager actionManager;

  /**

   * EntityGroupName vs current action in progress true - if open entityGroup

   * action in progress false - if close entityGroup action in progress

   */

  private final ConcurrentSkipListMap<byte[], Boolean> entityGroupsInTransitionInFS = new ConcurrentSkipListMap<byte[], Boolean>(

      Bytes.BYTES_COMPARATOR);

  // master address manager and watcher

  private MasterAddressTracker masterAddressManager;

  protected final ConcurrentHashMap<String, EntityGroupScanner> scanners = new ConcurrentHashMap<String, EntityGroupScanner>();

  /*

   * Strings to be used in forming the exception message for

   * EntityGroupsAlreadyInTransitionException.

   */

  private static final String OPEN = "OPEN";

  private static final String CLOSE = "CLOSE";

  private AtomicInteger requestCount = new AtomicInteger();

  // Split

  public SplitThread splitThread;

  private Leases leases;

  /**

   * Map of entityGroups currently being served by this FServer. Key is the

   * encoded entityGroup name. All access should be synchronized.

   */

  protected final Map<String, EntityGroup> onlineEntityGroups = new ConcurrentHashMap<String, EntityGroup>();

  private GlobalEntityGroup globalEntityGroup;

  private volatile boolean killed = false;

  private MessageBroker broker;

  private final boolean brokerOpen = false;

  /**

   * Starts a FServer at the default location

   *

   * @param conf

   * @throws java.io.IOException

   * @throws InterruptedException

   */

  public FServer(Configuration conf) throws IOException, InterruptedException {

    this.conf = conf;

    this.isOnline = false;

    // Set how many times to retry talking to another server over FConnection.

    FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG);

    // Config'ed params

    this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000);

    this.sleeper = new Sleeper(this.msgInterval, this);

    this.numEntityGroupsToReport = conf.getInt(

        "wasp.fserver.numentitygroupstoreport", 10);

    this.rpcTimeout = conf.getInt(FConstants.WASP_RPC_TIMEOUT_KEY,

        FConstants.DEFAULT_WASP_RPC_TIMEOUT);

    this.abortRequested = false;

    this.stopped = false;

    this.actionManager = new StorageActionManager(conf);

    // Server to handle client requests.

    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(

        conf.get("wasp.fserver.dns.interface", "default"),

        conf.get("wasp.fserver.dns.nameserver", "default")));

    int port = conf.getInt(FConstants.FSERVER_PORT,

        FConstants.DEFAULT_FSERVER_PORT);

    // Creation of a HSA will force a resolve.

    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);

    if (initialIsa.getAddress() == null) {

      throw new IllegalArgumentException("Failed resolve of " + initialIsa);

    }

    this.rpcServer = WaspRPC.getServer(FServer.class, this, new Class<?>[]{

        ClientProtocol.class, AdminProtocol.class, WaspRPCErrorHandler.class,

        OnlineEntityGroups.class}, initialIsa.getHostName(), // BindAddress is

        // IP we got for

        // this server.

        initialIsa.getPort(), conf);

    // Set our address.

    this.isa = this.rpcServer.getListenerAddress();

    this.leases = new Leases(conf.getInt(FConstants.THREAD_WAKE_FREQUENCY,

        10 * 1000));

    this.startcode = System.currentTimeMillis();

    int maxThreads = conf.getInt("wasp.transaction.threads.max", 150);

    this.pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS,

        new SynchronousQueue<Runnable>(), new DaemonThreadFactory(

            "thread factory"));

    ((ThreadPoolExecutor) this.pool).allowCoreThreadTimeOut(true);

    this.scannerLeaseTimeoutPeriod = conf.getInt(

        FConstants.WASP_CLIENT_SCANNER_TIMEOUT_PERIOD,

        FConstants.DEFAULT_WASP_CLIENT_SCANNER_TIMEOUT_PERIOD);

    this.driver = new BaseDriver(this);

    this.splitThread = new SplitThread(this);

    this.globalEntityGroup = new GlobalEntityGroup(this);

  }

  String getClusterId() {

    return this.conf.get(FConstants.CLUSTER_ID);

  }

  /**

   * All initialization needed before we go register with Master.

   *

   * @throws java.io.IOException

   * @throws InterruptedException

   */

  private void preRegistrationInitialization(){

    try {

      initializeZooKeeper();

      initializeThreads();

    } catch (Throwable t) {

      // Call stop if error or process will stick around for ever since server

      // puts up non-daemon threads.

      this.rpcServer.stop();

      abort("Initialization of RS failed.  Hence aborting RS.", t);

    }

  }

  /**

   * Bring up connection to zk ensemble and then wait until a master for this

   * cluster and then after that, wait until cluster 'up' flag has been set.

   * This is the order in which master does things. Finally put up a catalog

   * tracker.

   *

   * @throws java.io.IOException

   * @throws InterruptedException

   */

  private void initializeZooKeeper() throws IOException, InterruptedException {

    // Open connection to zookeeper and set primary watcher

    this.zooKeeper = new ZooKeeperWatcher(conf, FSERVER + ":"

        + this.isa.getPort(), this);

    // Create the master address manager, register with zk, and start it. Then

    // block until a master is available. No point in starting up if no master

    // running.

    this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);

    this.masterAddressManager.start();

    blockAndCheckIfStopped(this.masterAddressManager);

    // Wait on cluster being up. Master will set this flag up in zookeeper

    // when ready.

    this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);

    this.clusterStatusTracker.start();

    blockAndCheckIfStopped(this.clusterStatusTracker);

    // Retrieve clusterId

    // Since cluster status is now up

    // ID should have already been set by HMaster

    try {

      String clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper)

          .toString();

      if (clusterId == null) {

        this.abort("Cluster ID has not been set");

      }

      this.conf.set(FConstants.CLUSTER_ID, clusterId);

      LOG.info("ClusterId : " + clusterId);

    } catch (KeeperException e) {

      this.abort("Failed to retrieve Cluster ID", e);

    }

  }

  private void startServiceThreads() throws IOException {

    String n = Thread.currentThread().getName();

    // Start executor services

    this.service = new ExecutorService(getServerName().toString());

    this.service.startExecutorService(ExecutorService.ExecutorType.FSERVER_OPEN_ENTITYGROUP,

        conf.getInt("wasp.fserver.executor.openentitygroup.threads", 3));

    this.service.startExecutorService(ExecutorService.ExecutorType.FSERVER_CLOSE_ENTITYGROUP,

        conf.getInt("wasp.fserver.executor.closeentitygroup.threads", 3));

    this.leases.setName(n + ".leaseChecker");

    this.leases.start();

    // Put up the webui. Webui may come up on port other than configured if

    // that port is occupied. Adjust serverInfo if this is the case.

    this.webuiport = putUpWebUI();

    this.rpcServer.start();

    this.rpcServer.openServer();

    //start 2PC broker

    if(brokerOpen) {

      this.broker.initlize();

      this.broker.start();

    }

  }

  /**

   * Puts up the webui.

   *

   * @return Returns final port -- maybe different from what we started with.

   * @throws java.io.IOException

   */

  private int putUpWebUI() throws IOException {

    int port = this.conf.getInt(FConstants.FSERVER_INFO_PORT,

        FConstants.DEFAULT_FSERVER_INFOPORT);

    // -1 is for disabling info server

    if (port < 0)

      return port;

    String addr = this.conf.get("wasp.fserver.info.bindAddress", "0.0.0.0");

    // check if auto port bind enabled

    boolean auto = this.conf.getBoolean(FConstants.FSERVER_INFO_PORT_AUTO,

        false);

    while (true) {

      try {

        this.infoServer = new InfoServer("fserver", addr, port, false,

            this.conf);

        this.infoServer.addServlet("status", "/fs-status",

            FSStatusServlet.class);

        this.infoServer.addServlet("dump", "/dump", FSDumpServlet.class);

        this.infoServer.setAttribute(FSERVER, this);

        this.infoServer.setAttribute(FSERVER_CONF, conf);

        this.infoServer.start();

        break;

      } catch (BindException e) {

        if (!auto) {

          // auto bind disabled throw BindException

          throw e;

        }

        // auto bind enabled, try to use another port

        LOG.info("Failed binding http info server to port: " + port);

        port++;

      }

    }

    return port;

  }

  @Override

  public boolean isStopping() {

    return this.stopping;

  }

  @Override

  public Map<byte[], Boolean> getEntityGroupsInTransitionInFS() {

    return this.entityGroupsInTransitionInFS;

  }

  @Override

  public void addToOnlineEntityGroups(EntityGroup eg) {

    this.onlineEntityGroups.put(eg.getEntityGroupInfo().getEncodedName(), eg);

  }

  @Override

  public boolean removeFromOnlineEntityGroups(String encodedEntityGroupName) {

    EntityGroup toReturn = null;

    toReturn = this.onlineEntityGroups.remove(encodedEntityGroupName);

    return toReturn != null;

  }

  @Override

  public EntityGroup getFromOnlineEntityGroups(String encodedEntityGroupName) {

    return this.onlineEntityGroups.get(encodedEntityGroupName);

  }

  @Override

  public List<EntityGroup> getOnlineEntityGroups(byte[] tableName)

      throws IOException {

    List<EntityGroup> tableEntityGroups = new ArrayList<EntityGroup>();

    synchronized (this.onlineEntityGroups) {

      for (EntityGroup entityGroup : this.onlineEntityGroups.values()) {

        EntityGroupInfo entityGroupInfo = entityGroup.getEntityGroupInfo();

        if (Bytes.equals(entityGroupInfo.getTableName(), tableName)) {

          tableEntityGroups.add(entityGroup);

        }

      }

    }

    return tableEntityGroups;

  }

  private void initializeThreads() throws IOException {

    if(brokerOpen) {

      this.broker = new MessageBroker(this, conf);

    }

  }

  @Override

  public void run() {

//      preRegistrationInitialization();

//    } catch (Exception e) {

//      this.rpcServer.stop();

//      abort("Fatal exception during initialization", e);

//    }

    try {

      initializeZooKeeper();

    } catch (Exception t) {

      this.rpcServer.stop();

      abort("Initialization of FS failed.  Hence aborting FS.", t);

    }

    try {

      while (keepLooping()) {

        FServerStartupResponse w = reportForDuty();

        if (w == null) {

          LOG.warn("reportForDuty failed; sleeping and then retrying.");

          this.sleeper.sleep();

        } else {

          handleReportForDutyResponse(w);

          break;

        }

      }

      // We registered with the Master. Go into run mode.

      long lastMsg = 0;

      long oldRequestCount = -1;

      // The main run loop.

      while (!this.stopped && isHealthy()) {

        if (!isClusterUp()) {

          if (isOnlineEntityGroupsEmpty()) {

            stop("Exiting; cluster shutdown set and not carrying any entityGroups");

          } else if (!this.stopping) {

            this.stopping = true;

            LOG.info("Closing user entityGroups");

            closeUserEntityGroups(this.abortRequested);

          } else if (this.stopping) {

            boolean allUserEntityGroupsOffline = areAllUserEntityGroupsOffline();

            if (allUserEntityGroupsOffline) {

              // Set stopped if no requests since last time we went around the

              // loop.

              if (oldRequestCount == this.requestCount.get()) {

                stop("Stopped;");

                break;

              }

              oldRequestCount = this.requestCount.get();

            } else {

              // Make sure all entityGroups have been closed -- some

              // entityGroups may

              // have not got it because we were splitting at the time of

              // the call to closeUserEntityGroups.

              closeUserEntityGroups(this.abortRequested);

            }

            LOG.debug("Waiting on " + getOnlineEntityGroupsAsPrintableString());

          }

          driver.close();

        }

        long now = System.currentTimeMillis();

        if ((now - lastMsg) >= msgInterval) {

          doMetrics();

          tryFServerReport(lastMsg, now);

          lastMsg = System.currentTimeMillis();

        }

        if (!this.stopped)

          this.sleeper.sleep();

      }

    } catch (Throwable t) {

      abort("Unhandled exception: " + t.getMessage(), t);

    }

    // Run shutdown.

    if (mxBean != null) {

      MBeans.unregister(mxBean);

      mxBean = null;

    }

    this.leases.closeAfterLeasesExpire();

    this.rpcServer.stop();

    if (this.splitThread != null) {

      this.splitThread.interruptIfNecessary();

    }

    if (this.killed) {

      // Just skip out w/o closing entityGroups. Used when testing.

    } else if (abortRequested) {

      closeUserEntityGroups(abortRequested); // Don't leave any open file

                                             // handles

      LOG.info("aborting server " + this.serverNameFromMasterPOV);

    } else {

      closeUserEntityGroups(abortRequested);

      closeAllScanners();

      LOG.info("stopping server " + this.serverNameFromMasterPOV);

    }

    globalEntityGroup.close();

    if (!this.killed) {

      waitOnAllEntityGroupsToClose(abortRequested);

      LOG.info("stopping server " + this.serverNameFromMasterPOV

          + "; all entityGroups closed.");

    }

    // Make sure the proxy is down.

    if (this.waspMaster != null) {

      WaspRPC.stopProxy(this.waspMaster);

      this.waspMaster = null;

    }

    this.leases.close();

    if (infoServer != null) {

      try {

        infoServer.stop();

      } catch (Exception e) {

        LOG.warn("Failed stop infoServer", e);

      }

    }

    if(this.broker != null) {

      try {

        this.broker.close();

      } catch (IOException e) {

        LOG.warn("Failed close broker", e);

      }

    }

    if (!killed) {

      join();

    }

    try {

      deleteMyEphemeralNode();

    } catch (KeeperException e) {

      LOG.warn("Failed deleting my ephemeral node", e);

    }

    // We may have failed to delete the znode at the previous step, but

    // we delete the file anyway: a second attempt to delete the znode is likely

    // to fail again.

    ZNodeClearer.deleteMyEphemeralNodeOnDisk();

    this.zooKeeper.close();

    LOG.info("stopping server " + this.serverNameFromMasterPOV

        + "; zookeeper connection closed.");

    LOG.info(Thread.currentThread().getName() + " exiting");

  }

  private boolean areAllUserEntityGroupsOffline() {

    if (getNumberOfOnlineEntityGroups() > 2) {

      return false;

    }

    boolean allUserEntityGroupsOffline = true;

    for (Map.Entry<String, EntityGroup> e : this.onlineEntityGroups.entrySet()) {

      if (e != null) {

        allUserEntityGroupsOffline = false;

        break;

      }

    }

    return allUserEntityGroupsOffline;

  }

  /**

   * Verify that server is healthy

   */

  private boolean isHealthy() {

    // Verify that all threads are alive

    if (!leases.isAlive()) {

      stop("One or more threads are no longer alive -- stop");

      return false;

    }

    return true;

  }

  void tryFServerReport(long reportStartTime, long reportEndTime)

      throws IOException {

    WaspProtos.ServerLoadProtos sl = buildServerLoad(reportStartTime,

        reportEndTime);

    this.requestCount.set(0);

    this.driver.resetMetricsCount();

    try {

      this.waspMaster.fServerReport(

          null,

          RequestConverter.buildFServerReportRequest(sl,

              ProtobufUtil.toServerName(this.serverNameFromMasterPOV)));

    } catch (ServiceException se) {

      IOException ioe = ProtobufUtil.getRemoteException(se);

      if (ioe instanceof YouAreDeadException) {

        // This will be caught and handled as a fatal error in run()

        throw ioe;

      }

      // Couldn't connect to the master, get location from zk and reconnect

      // Method blocks until new master is found or we are stopped

      getMaster();

    }

  }

  private WaspProtos.ServerLoadProtos buildServerLoad(long reportStartTime,

      long reportEndTime) {

    MetricsFServerWrapper fserverWrapper = this.metricsFServer

        .getFServerWrapper();

    Collection<EntityGroup> entityGroups = getOnlineEntityGroupsLocalContext();

    WaspProtos.ServerLoadProtos.Builder serverLoad = WaspProtos.ServerLoadProtos

        .newBuilder();

    serverLoad.setNumberOfRequests((int) fserverWrapper.getRequestsPerSecond());

    serverLoad.setTotalNumberOfRequests(requestCount.get());

    serverLoad.setNumberOfConnections(rpcServer.getNumberOfConnections());

    for (EntityGroup entityGroup : entityGroups) {

      serverLoad

          .addEntityGroupsLoads(createEntityGroupLoad(entityGroup).entityGroupLoadPB);

    }

    serverLoad.setReportStartTime(reportStartTime);

    serverLoad.setReportEndTime(reportEndTime);

    serverLoad.setAvgWriteTime(driver.getAvgWriteTime());

    serverLoad.setAvgReadTime(driver.getAvgReadTime());

    return serverLoad.build();

  }

  String getOnlineEntityGroupsAsPrintableString() {

    StringBuilder sb = new StringBuilder();

    for (EntityGroup r : this.onlineEntityGroups.values()) {

      if (sb.length() > 0)

        sb.append(", ");

      sb.append(r.getEntityGroupInfo().getEncodedName());

    }

    return sb.toString();

  }

  /**

   * For tests and web ui. This method will only work if FServer is in the same

   * JVM as client; EntityGroup cannot be serialized to cross an rpc.

   *

   * @see #getOnlineEntityGroups()

   */

  public Collection<EntityGroup> getOnlineEntityGroupsLocalContext() {

    Collection<EntityGroup> entityGroups = this.onlineEntityGroups.values();

    return Collections.unmodifiableCollection(entityGroups);

  }

  /**

   * Register bean with platform management server

   */

  void registerMBean() {

    MXBeanImpl mxBeanInfo = MXBeanImpl.init(this);

    mxBean = MBeans.register("FServer", "FServer", mxBeanInfo);

    LOG.info("Registered FServer MXBean");

  }

  /**

   * Let the master know we're here Run initialization using parameters passed

   * us by the master.

   *

   * @return A Map of key/value configurations we got from the Master else null

   *         if we failed to register.

   *

   * @throws java.io.IOException

   */

  private FServerStartupResponse reportForDuty() throws IOException {

    FServerStartupResponse result = null;

    ServerName masterServerName = getMaster();

    if (masterServerName == null)

      return result;

    try {

      this.requestCount.set(0);

      LOG.info("Telling master at " + masterServerName + " that we are up "

          + "with port=" + this.isa.getPort() + ", startcode=" + this.startcode);

      long now = EnvironmentEdgeManager.currentTimeMillis();

      int port = this.isa.getPort();

      FServerStartupRequest.Builder request = FServerStartupRequest

          .newBuilder();

      request.setPort(port);

      request.setServerStartCode(this.startcode);

      request.setServerCurrentTime(now);

      result = this.waspMaster.fServerStartup(null, request.build());

    } catch (ServiceException se) {

      IOException ioe = ProtobufUtil.getRemoteException(se);

      if (ioe instanceof ClockOutOfSyncException) {

        LOG.fatal("Master rejected startup because clock is out of sync", ioe);

        // Re-throw IOE will cause ES to abort

        throw ioe;

      } else {

        LOG.warn("error telling master we are up", se);

      }

    }

    return result;

  }

  /*

   * Run init. Sets up hlog and starts up all server threads.

   *

   * @param c Extra configuration.

   */

  protected void handleReportForDutyResponse(final FServerStartupResponse c)

      throws IOException {

    try {

      for (WaspProtos.StringStringPair e : c.getMapEntriesList()) {

        String key = e.getName();

        // The hostname the master sees us as.

        if (key.equals(FConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {

          String hostnameFromMasterPOV = e.getValue();

          this.serverNameFromMasterPOV = new ServerName(hostnameFromMasterPOV,

              this.isa.getPort(), this.startcode);

          LOG.info("Master passed us hostname to use. Was="

              + this.isa.getHostName() + ", Now="

              + this.serverNameFromMasterPOV.getHostname());

          continue;

        }

        String value = e.getValue().toString();

        if (LOG.isDebugEnabled()) {

          LOG.debug("Config from master: " + key + "=" + value);

        }

        this.conf.set(key, value);

      }

      // Set our ephemeral znode up in zookeeper now we have a name.

      createMyEphemeralNode();

      // Save it in a file, this will allow to see if we crash

      ZNodeClearer.writeMyEphemeralNodeOnDisk(getMyEphemeralNodePath());

      this.metricsFServer = new MetricsFServer(new MetricsFServerWrapperImpl(

          this));

      this.tableSchemaReader = TableSchemaCacheReader.getInstance(this.conf);

      startServiceThreads();

      LOG.info("Serving as "

          + this.serverNameFromMasterPOV

          + ", RPC listening on "

          + this.isa

          + ", sessionid=0x"

          + Long.toHexString(this.zooKeeper.getRecoverableZooKeeper()

              .getSessionId()));

      isOnline = true;

    } catch (Throwable e) {

      this.isOnline = false;

      stop("Failed initialization");

      throw convertThrowableToIOE(cleanup(e, "Failed init"),

          "FServer startup failed");

    } finally {

      sleeper.skipSleepCycle();

    }

  }

  private void createMyEphemeralNode() throws KeeperException {

    ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper,

        getMyEphemeralNodePath(), FConstants.EMPTY_BYTE_ARRAY);

  }

  private void deleteMyEphemeralNode() throws KeeperException {

    ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());

  }

  private String getMyEphemeralNodePath() {

    return ZKUtil.joinZNode(this.zooKeeper.fsZNode, getServerName().toString());

  }

  /**

   * @see FServer#abort(String, Throwable)

   */

  public void abort(String reason) {

    abort(reason, null);

  }

  /**

   * Cause the server to exit without closing the entityGroups it is serving,

   * the log it is using and without notifying the master. OOME.

   *

   * @param reason

   *          the reason we are aborting

   * @param cause

   *          the exception that caused the abort, or null

   */

  @Override

  public void abort(String reason, Throwable cause) {

    String msg = "ABORTING FServer " + this + ": " + reason;

    if (cause != null) {

      LOG.fatal(msg, cause);

    } else {

      LOG.fatal(msg);

    }

    this.abortRequested = true;

    this.reservedSpace.clear();

    if (this.metricsFServer != null) {

      LOG.info("Dump of metrics: " + this.metricsFServer);

    }

    // Do our best to report our abort to the master, but this may not work

    try {

      if (cause != null) {

        msg += "\nCause:\n" + StringUtils.stringifyException(cause);

      }

    } catch (Throwable t) {

      LOG.warn("Unable to report fatal error to master", t);

    }

    stop(reason);

  }

  @Override

  public boolean isAborted() {

    return this.abortRequested;

  }

  @Override

  public void stop(String reason) {

    this.stopped = true;

    LOG.info("STOPPED: " + reason);

    // Wakes run() if it is sleeping

    sleeper.skipSleepCycle();

  }

  @Override

  public boolean isStopped() {

    return this.stopped;

  }

  /**

   * Checks to see if the storage system is still accessible. If not, sets

   * abortRequested and stopRequested

   *

   * @return false if storage system is not available

   */

  public boolean checkStorageSystem() {

    try {

      HTableInterface htable = this.getActionManager().getTable(

          conf.get(FConstants.METASTORE_TABLE,

              FConstants.DEFAULT_METASTORE_TABLE));

      htable.close();