* Copyright 2010 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A non-instantiable class that manages creation of {@link HConnection}s.

* <p>The simplest way to use this class is by using {@link #createConnection(Configuration)}.

* This creates a new {@link HConnection} that is managed by the caller.

* From this {@link HConnection} {@link HTableInterface} implementations are retrieved

* with {@link HConnection#getTable(byte[])}. Example:

* <pre>

* {@code

* HConnection connection = HConnectionManager.createConnection(config);

* HTableInterface table = connection.getTable("table1");

* // use the table as needed, for a single operation and a single thread

* table.close();

* connection.close();

* }

* </pre>

* <p>The following logic and API will be removed in the future:

* <p>This class has a static Map of {@link HConnection} instances keyed by

* {@link Configuration}; all invocations of {@link #getConnection(Configuration)}

* that pass the same {@link Configuration} instance will be returned the same

* {@link  HConnection} instance (Adding properties to a Configuration

* instance does not change its object identity).  Sharing {@link HConnection}

* instances is usually what you want; all clients of the {@link HConnection}

* instances share the HConnections' cache of Region locations rather than each

* having to discover for itself the location of meta, root, etc.  It makes

* sense for the likes of the pool of HTables class {@link HTablePool}, for

* instance (If concerned that a single {@link HConnection} is insufficient

* for sharing amongst clients in say an heavily-multithreaded environment,

* in practise its not proven to be an issue.  Besides, {@link HConnection} is

* implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a

* connection per cluster-member, exclusively).

*

* <p>But sharing connections

* makes clean up of {@link HConnection} instances a little awkward.  Currently,

* clients cleanup by calling

* {@link #deleteConnection(Configuration)}.  This will shutdown the

* zookeeper connection the HConnection was using and clean up all

* HConnection resources as well as stopping proxies to servers out on the

* cluster. Not running the cleanup will not end the world; it'll

* just stall the closeup some and spew some zookeeper connection failed

* messages into the log.  Running the cleanup on a {@link HConnection} that is

* subsequently used by another will cause breakage so be careful running

* cleanup.

* <p>To create a {@link HConnection} that is not shared by others, you can

* create a new {@link Configuration} instance, pass this new instance to

* {@link #getConnection(Configuration)}, and then when done, close it up by

* doing something like the following:

* <pre>

* {@code

* Configuration newConfig = new Configuration(originalConf);

* HConnection connection = HConnectionManager.getConnection(newConfig);

* // Use the connection to your hearts' delight and then when done...

* HConnectionManager.deleteConnection(newConfig, true);

* }

* </pre>

* <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd

* register a shutdown hook that called {@link #deleteAllConnections()}

* on its way out but the order in which shutdown hooks run is not defined so

* were problematic for clients of HConnection that wanted to register their

* own shutdown hooks so we removed ours though this shifts the onus for

* cleanup to the client.

*/

  // An LRU Map of HConnectionKey -> HConnection (TableServer).  All

  // access must be synchronized.  This map is not private because tests

  // need to be able to tinker with it.

  static final Map<HConnectionKey, HConnectionImplementation> HBASE_INSTANCES;

  public static final int MAX_CACHED_HBASE_INSTANCES;

  private static Log LOG = LogFactory.getLog(HConnectionManager.class);

  static {

    // We set instances to one more than the value specified for {@link

    // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max

    // connections to the ensemble from the one client is 30, so in that case we

    // should run into zk issues before the LRU hit this value of 31.

    MAX_CACHED_HBASE_INSTANCES = HBaseConfiguration.create().getInt(

        HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,

        HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;

    HBASE_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(

        (int) (MAX_CACHED_HBASE_INSTANCES / 0.75F) + 1, 0.75F, true) {

       @Override

      protected boolean removeEldestEntry(

          Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {

         return size() > MAX_CACHED_HBASE_INSTANCES;

       }

    };

  }

  /*

   * Non-instantiable.

   */

  protected HConnectionManager() {

    super();

  }

  /**

   * Get the connection that goes with the passed <code>conf</code>

   * configuration instance.

   * If no current connection exists, method creates a new connection for the

   * passed <code>conf</code> instance.

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection getConnection(Configuration conf)

  throws ZooKeeperConnectionException {

    HConnectionKey connectionKey = new HConnectionKey(conf);

    synchronized (HBASE_INSTANCES) {

      HConnectionImplementation connection = HBASE_INSTANCES.get(connectionKey);

      if (connection == null) {

        connection = new HConnectionImplementation(conf, true, null);

        HBASE_INSTANCES.put(connectionKey, connection);

      } else if (connection.isClosed()) {

        HConnectionManager.deleteConnection(connectionKey, true);

        connection = new HConnectionImplementation(conf, true, null);

        HBASE_INSTANCES.put(connectionKey, connection);

      }

      connection.incCount();

      return connection;

    }

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code>

   * instance.

   * Note: This bypasses the usual HConnection life cycle management.

   * The caller is responsible for calling {@link HConnection#close()}

   * on the returned connection instance.

   *

   * This is the recommended way to create HConnections.

   * {@code

   * HConnection connection = HConnectionManager.createConnection(conf);

   * HTableInterface table = connection.getTable("mytable");

   * table.get(...);

   * ...

   * table.close();

   * connection.close();

   * }

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf)

  throws ZooKeeperConnectionException {

    return new HConnectionImplementation(conf, false, null);

  }

  public static HConnection createConnection(Configuration conf, ExecutorService pool)

      throws IOException {

    return new HConnectionImplementation(conf, false, pool);

  }

  /**

   * Delete connection information for the instance specified by configuration.

   * If there are no more references to it, this will then close connection to

   * the zookeeper ensemble and let go of all resources.

   *

   * @param conf

   *          configuration whose identity is used to find {@link HConnection}

   *          instance.

   * @param stopProxy

   *          No longer used.  This parameter is ignored.

   * @deprecated use {@link #createConnection(org.apache.hadoop.conf.Configuration)} instead

   */

  @Deprecated

  public static void deleteConnection(Configuration conf, boolean stopProxy) {

    deleteConnection(conf);

  }

  /**

   * Delete connection information for the instance specified by configuration.

   * If there are no more references to it, this will then close connection to

   * the zookeeper ensemble and let go of all resources.

   *

   * @param conf

   *          configuration whose identity is used to find {@link HConnection}

   *          instance.

   */

  @Deprecated

  public static void deleteConnection(Configuration conf) {

    deleteConnection(new HConnectionKey(conf), false);

  }

  /**

   * Delete stale connection information for the instance specified by configuration.

   * This will then close connection to

   * the zookeeper ensemble and let go of all resources.

   *

   * @param connection

   */

  public static void deleteStaleConnection(HConnection connection) {

    deleteConnection(connection, true);

  }

  /**

   * Delete information for all connections.

   * @param stopProxy No longer used.  This parameter is ignored.

   * @deprecated use {@link #deleteAllConnections()} instead

   */

  @Deprecated

  public static void deleteAllConnections(boolean stopProxy) {

    deleteAllConnections();

  }

  /**

   * Delete information for all connections.

   * @throws IOException

   */

  @Deprecated

  public static void deleteAllConnections() {

    synchronized (HBASE_INSTANCES) {

      Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();

      connectionKeys.addAll(HBASE_INSTANCES.keySet());

      for (HConnectionKey connectionKey : connectionKeys) {

        deleteConnection(connectionKey, false);

      }

      HBASE_INSTANCES.clear();

    }

  }

  @Deprecated

  private static void deleteConnection(HConnection connection, boolean staleConnection) {

    synchronized (HBASE_INSTANCES) {

      for (Entry<HConnectionKey, HConnectionImplementation> connectionEntry : HBASE_INSTANCES

          .entrySet()) {

        if (connectionEntry.getValue() == connection) {

          deleteConnection(connectionEntry.getKey(), staleConnection);

          break;

        }

      }

    }

  }

  @Deprecated

  private static void deleteConnection(HConnectionKey connectionKey,

      boolean staleConnection) {

    synchronized (HBASE_INSTANCES) {

      HConnectionImplementation connection = HBASE_INSTANCES

          .get(connectionKey);

      if (connection != null) {

        connection.decCount();

        if (connection.isZeroReference() || staleConnection) {

          HBASE_INSTANCES.remove(connectionKey);

          connection.internalClose();

        }

      }else {

        LOG.error("Connection not found in the list, can't delete it "+

          "(connection key="+connectionKey+"). May be the key was modified?");

      }

    }

  }

  /**

   * It is provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return Number of cached regions for the table.

   * @throws ZooKeeperConnectionException

   */

  static int getCachedRegionCount(Configuration conf,

      final byte[] tableName)

  throws IOException {

    return execute(new HConnectable<Integer>(conf) {

      @Override

      public Integer connect(HConnection connection) {

        return ((HConnectionImplementation) connection)

            .getNumberOfCachedRegionLocations(tableName);

      }

    });

  }

  /**

   * It's provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return true if the region where the table and row reside is cached.

   * @throws ZooKeeperConnectionException

   */

  static boolean isRegionCached(Configuration conf,

      final byte[] tableName, final byte[] row) throws IOException {

    return execute(new HConnectable<Boolean>(conf) {

      @Override

      public Boolean connect(HConnection connection) {

        return ((HConnectionImplementation) connection).isRegionCached(tableName, row);

      }

    });

  }

  /**

   * This class makes it convenient for one to execute a command in the context

   * of a {@link HConnection} instance based on the given {@link Configuration}.

   *

   * <p>

   * If you find yourself wanting to use a {@link HConnection} for a relatively

   * short duration of time, and do not want to deal with the hassle of creating

   * and cleaning up that resource, then you should consider using this

   * convenience class.

   *

   * @param <T>

   *          the return type of the {@link HConnectable#connect(HConnection)}

   *          method.

   */

  public static abstract class HConnectable<T> {

    public Configuration conf;

    public HConnectable(Configuration conf) {

      this.conf = conf;

    }

    public abstract T connect(HConnection connection) throws IOException;

  }

  /**

   * This convenience method invokes the given {@link HConnectable#connect}

   * implementation using a {@link HConnection} instance that lasts just for the

   * duration of that invocation.

   *

   * @param <T> the return type of the connect method

   * @param connectable the {@link HConnectable} instance

   * @return the value returned by the connect method

   * @throws IOException

   */

  public static <T> T execute(HConnectable<T> connectable) throws IOException {

    if (connectable == null || connectable.conf == null) {

      return null;

    }

    Configuration conf = connectable.conf;

    HConnection connection = HConnectionManager.getConnection(conf);

    boolean connectSucceeded = false;

    try {

      T returnValue = connectable.connect(connection);

      connectSucceeded = true;

      return returnValue;

    } finally {

      try {

        connection.close();

      } catch (Exception e) {

        if (connectSucceeded) {

          throw new IOException("The connection to " + connection

              + " could not be deleted.", e);

        }

      }

    }

  }

  /**

   * Denotes a unique key to a {@link HConnection} instance.

   *

   * In essence, this class captures the properties in {@link Configuration}

   * that may be used in the process of establishing a connection. In light of

   * that, if any new such properties are introduced into the mix, they must be

   * added to the {@link HConnectionKey#properties} list.

   *

   */

  public static class HConnectionKey {

    public static String[] CONNECTION_PROPERTIES = new String[] {

        HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT,

        HConstants.ZOOKEEPER_CLIENT_PORT,

        HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME,

        HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIENT_RETRIES_NUMBER,

        HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,

        HConstants.HBASE_RPC_TIMEOUT_KEY,

        HConstants.HBASE_CLIENT_PREFETCH_LIMIT,

        HConstants.HBASE_META_SCANNER_CACHING,

        HConstants.HBASE_CLIENT_INSTANCE_ID };

    private Map<String, String> properties;

    private String username;

    public HConnectionKey(Configuration conf) {

      Map<String, String> m = new HashMap<String, String>();

      if (conf != null) {

        for (String property : CONNECTION_PROPERTIES) {

          String value = conf.get(property);

          if (value != null) {

            m.put(property, value);

          }

        }

      }

      this.properties = Collections.unmodifiableMap(m);

      try {

        User currentUser = User.getCurrent();

        if (currentUser != null) {

          username = currentUser.getName();

        }

      } catch (IOException ioe) {

        LOG.warn("Error obtaining current user, skipping username in HConnectionKey",

            ioe);

      }

    }

    @Override

    public int hashCode() {

      final int prime = 31;

      int result = 1;

      if (username != null) {

        result = username.hashCode();

      }

      for (String property : CONNECTION_PROPERTIES) {

        String value = properties.get(property);

        if (value != null) {

          result = prime * result + value.hashCode();

        }

      }

      return result;

    }

    @Override

    public boolean equals(Object obj) {

      if (this == obj)

        return true;

      if (obj == null)

        return false;

      if (getClass() != obj.getClass())

        return false;

      HConnectionKey that = (HConnectionKey) obj;

      if (this.username != null && !this.username.equals(that.username)) {

        return false;

      } else if (this.username == null && that.username != null) {

        return false;

      }

      if (this.properties == null) {

        if (that.properties != null) {

          return false;

        }

      } else {

        if (that.properties == null) {

          return false;

        }

        for (String property : CONNECTION_PROPERTIES) {

          String thisValue = this.properties.get(property);

          String thatValue = that.properties.get(property);

          if (thisValue == thatValue) {

            continue;

          }

          if (thisValue == null || !thisValue.equals(thatValue)) {

            return false;

          }

        }

      }

      return true;

    }

    @Override

    public String toString() {

      return "HConnectionKey{" +

        "properties=" + properties +

        ", username='" + username + '\'' +

        '}';

    }

  }

  /* Encapsulates connection to zookeeper and regionservers.*/

  static class HConnectionImplementation implements HConnection, Closeable {

    static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);

    private final Class<? extends HRegionInterface> serverInterfaceClass;

    private final long pause;

    private final int numRetries;

    private final int maxRPCAttempts;

    private final int rpcTimeout;

    private final int prefetchRegionLimit;

    private final Object masterLock = new Object();

    private volatile boolean closed;

    private volatile boolean aborted;

    private volatile boolean resetting;

    private volatile HMasterInterface master;

    // ZooKeeper reference

    private volatile ZooKeeperWatcher zooKeeper;

    // ZooKeeper-based master address tracker

    private volatile MasterAddressTracker masterAddressTracker;

    private volatile RootRegionTracker rootRegionTracker;

    private volatile ClusterId clusterId;

    private final Object metaRegionLock = new Object();

    private final Object userRegionLock = new Object();

 

    private final Object resetLock = new Object();

    // thread executor shared by all HTableInterface instances created

    // by this connection

    private volatile ExecutorService batchPool = null;

    private volatile boolean cleanupPool = false;

    private final Configuration conf;

    private RpcEngine rpcEngine;

    // Known region HServerAddress.toString() -> HRegionInterface

    private final Map<String, HRegionInterface> servers =

      new ConcurrentHashMap<String, HRegionInterface>();

    private final ConcurrentHashMap<String, String> connectionLock =

      new ConcurrentHashMap<String, String>();

    /**

     * Map of table to table {@link HRegionLocation}s.  The table key is made

     * by doing a {@link Bytes#mapKey(byte[])} of the table's name.

     */

    private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>

      cachedRegionLocations =

        new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();

    // The presence of a server in the map implies it's likely that there is an

    // entry in cachedRegionLocations that map to this server; but the absence

    // of a server in this map guarentees that there is no entry in cache that

    // maps to the absent server.

    private final Set<String> cachedServers =

        new HashSet<String>();

    // region cache prefetch is enabled by default. this set contains all

    // tables whose region cache prefetch are disabled.

    private final Set<Integer> regionCachePrefetchDisabledTables =

      new CopyOnWriteArraySet<Integer>();

    private int refCount;

    // indicates whether this connection's life cycle is managed

    private final boolean managed;

    /**

     * constructor

     * @param conf Configuration object

     */

    @SuppressWarnings("unchecked")

    public HConnectionImplementation(Configuration conf, boolean managed, ExecutorService pool)

    throws ZooKeeperConnectionException {

      this.conf = conf;

      this.batchPool = pool;

      this.managed = managed;

      String serverClassName = conf.get(HConstants.REGION_SERVER_CLASS,

        HConstants.DEFAULT_REGION_SERVER_CLASS);

      this.closed = false;

      try {

        this.serverInterfaceClass =

          (Class<? extends HRegionInterface>) Class.forName(serverClassName);

      } catch (ClassNotFoundException e) {

        throw new UnsupportedOperationException(

            "Unable to find region server interface " + serverClassName, e);

      }

      this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,

          HConstants.DEFAULT_HBASE_CLIENT_PAUSE);

      this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,

          HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);

      this.maxRPCAttempts = conf.getInt(

          HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,

          HConstants.DEFAULT_HBASE_CLIENT_RPC_MAXATTEMPTS);

      this.rpcTimeout = conf.getInt(

          HConstants.HBASE_RPC_TIMEOUT_KEY,

          HConstants.DEFAULT_HBASE_RPC_TIMEOUT);

      this.prefetchRegionLimit = conf.getInt(

          HConstants.HBASE_CLIENT_PREFETCH_LIMIT,

          HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);

      this.master = null;

      this.resetting = false;

    }

    @Override

    public HTableInterface getTable(String tableName) throws IOException {

      return getTable(Bytes.toBytes(tableName));

    }

    @Override

    public HTableInterface getTable(byte[] tableName) throws IOException {

      return getTable(tableName, getBatchPool());

    }

    @Override

    public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {

      return getTable(Bytes.toBytes(tableName), pool);

    }

    @Override

    public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {

      if (managed) {

        throw new IOException("The connection has to be unmanaged.");

      }

      return new HTable(tableName, this, pool);

    }

    private ExecutorService getBatchPool() {

      if (batchPool == null) {

        // shared HTable thread executor not yet initialized

        synchronized (this) {

          if (batchPool == null) {

            int maxThreads = conf.getInt("hbase.hconnection.threads.max", Integer.MAX_VALUE);

            if (maxThreads == 0) {

              maxThreads = Runtime.getRuntime().availableProcessors();

            }

            long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60);

            this.batchPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(),

                maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),

                Threads.newDaemonThreadFactory("hbase-connection-shared-executor"));

            ((ThreadPoolExecutor) this.batchPool).allowCoreThreadTimeOut(true);

          }

          this.cleanupPool = true;

        }

      }

      return this.batchPool;

    }

    protected ExecutorService getCurrentBatchPool() {

      return batchPool;

    }

    private void shutdownBatchPool() {

      if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {

        this.batchPool.shutdown();

        try {

          if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {

            this.batchPool.shutdownNow();

          }

        } catch (InterruptedException e) {

          this.batchPool.shutdownNow();

        }

      }

    }

    private synchronized void ensureZookeeperTrackers()

        throws ZooKeeperConnectionException {

      // initialize zookeeper and master address manager

      if (zooKeeper == null) {

        zooKeeper = getZooKeeperWatcher();

      }

      if (clusterId == null) {

        clusterId = new ClusterId(zooKeeper, this);

        if (clusterId.hasId()) {

          conf.set(HConstants.CLUSTER_ID, clusterId.getId());

        }

      }

      if (masterAddressTracker == null) {

        masterAddressTracker = new MasterAddressTracker(zooKeeper, this);

        masterAddressTracker.start();

      }

      if (rootRegionTracker == null) {

        rootRegionTracker = new RootRegionTracker(zooKeeper, this);

        rootRegionTracker.start();

      }

      // RpcEngine needs access to zookeeper data, like cluster ID

      if (rpcEngine == null) {

        this.rpcEngine = HBaseRPC.getProtocolEngine(conf);

      }

    }

    private synchronized void resetZooKeeperTrackers() {

      if (masterAddressTracker != null) {

        masterAddressTracker.stop();

        masterAddressTracker = null;

      }

      if (rootRegionTracker != null) {

        rootRegionTracker.stop();

        rootRegionTracker = null;

      }

      clusterId = null;

      if (zooKeeper != null) {

        zooKeeper.close();

        zooKeeper = null;

      }

    }

    public Configuration getConfiguration() {

      return this.conf;

    }

    /**

     * Log failure of getMaster attempt

     * @return true if should retry

     */

    private boolean shouldRetryGetMaster(int tries, Exception e) {

      if (tries == numRetries - 1) {

        // This was our last chance - don't bother sleeping

        LOG.info("getMaster attempt " + tries + " of " + numRetries +

          " failed; no more retrying.", e);

        return false;

      }

      LOG.info("getMaster attempt " + tries + " of " + numRetries +

        " failed; retrying after sleep of " +

        ConnectionUtils.getPauseTime(this.pause, tries), e);

      return true;

    }

    public HMasterInterface getMaster()

    throws MasterNotRunningException, ZooKeeperConnectionException {

      // TODO: REMOVE.  MOVE TO HBaseAdmin and redo as a Callable!!!

      // Check if we already have a good master connection

      try {

        if (master != null && master.isMasterRunning()) {

          return master;

        }

      } catch (UndeclaredThrowableException ute) {

        // log, but ignore, the loop below will attempt to reconnect

        LOG.info("Exception contacting master. Retrying...", ute.getCause());

      }

      ensureZookeeperTrackers();

      checkIfBaseNodeAvailable();

      ServerName sn = null;

      synchronized (this.masterLock) {

        try {

          if (master != null && master.isMasterRunning()) {

            return master;

          }

        } catch (UndeclaredThrowableException ute) {

          // log, but ignore, the loop below will attempt to reconnect

          LOG.info("Exception contacting master. Retrying...", ute.getCause());

        }

        this.master = null;

        for (int tries = 0;

          !this.closed && this.master == null && tries < numRetries;

        tries++) {

          try {

            sn = masterAddressTracker.getMasterAddress();

            if (sn == null) {

              LOG.info("ZooKeeper available but no active master location found");

              throw new MasterNotRunningException();

            }

            InetSocketAddress isa =

              new InetSocketAddress(sn.getHostname(), sn.getPort());

            HMasterInterface tryMaster = rpcEngine.getProxy(

                HMasterInterface.class, HMasterInterface.VERSION, isa, this.conf,

                this.rpcTimeout);

            if (tryMaster.isMasterRunning()) {

              this.master = tryMaster;

              this.masterLock.notifyAll();

              break;

            }

          } catch (IOException e) {

            if (!shouldRetryGetMaster(tries, e)) break;

          } catch (UndeclaredThrowableException ute) {

            if (!shouldRetryGetMaster(tries, ute)) break;

          }

          // Cannot connect to master or it is not running. Sleep & retry

          try {

            this.masterLock.wait(ConnectionUtils.getPauseTime(this.pause, tries));

          } catch (InterruptedException e) {

            Thread.currentThread().interrupt();

            throw new RuntimeException("Thread was interrupted while trying to connect to master.");

          }

        }

        if (this.master == null) {

          if (sn == null) {

            throw new MasterNotRunningException();

          }

          throw new MasterNotRunningException(sn.toString());

        }

        return this.master;

      }

    }

    private void checkIfBaseNodeAvailable() throws MasterNotRunningException {

      if (false == masterAddressTracker.checkIfBaseNodeAvailable()) {

        String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. "

            + "There could be a mismatch with the one configured in the master.";

        LOG.error(errorMsg);

        throw new MasterNotRunningException(errorMsg);

      }

    }

    public boolean isMasterRunning()

    throws MasterNotRunningException, ZooKeeperConnectionException {

      if (this.master == null) {

        getMaster();

      }

      boolean isRunning = master.isMasterRunning();

      if(isRunning) {

        return true;

      }

      throw new MasterNotRunningException();

    }

    public HRegionLocation getRegionLocation(final byte [] name,

        final byte [] row, boolean reload)

    throws IOException {

      return reload? relocateRegion(name, row): locateRegion(name, row);

    }

    public boolean isTableEnabled(byte[] tableName) throws IOException {

      return testTableOnlineState(tableName, true);

    }

    public boolean isTableDisabled(byte[] tableName) throws IOException {

      return testTableOnlineState(tableName, false);

    }

    public boolean isTableAvailable(final byte[] tableName) throws IOException {

      final AtomicBoolean available = new AtomicBoolean(true);

      final AtomicInteger regionCount = new AtomicInteger(0);

      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {

        @Override

        public boolean processRow(Result row) throws IOException {

          byte[] value = row.getValue(HConstants.CATALOG_FAMILY,

              HConstants.REGIONINFO_QUALIFIER);

          HRegionInfo info = Writables.getHRegionInfoOrNull(value);

          if (info != null) {

            if (Bytes.equals(tableName, info.getTableName())) {

              value = row.getValue(HConstants.CATALOG_FAMILY,

                  HConstants.SERVER_QUALIFIER);

              if (value == null) {

                available.set(false);

                return false;

              }

              regionCount.incrementAndGet();

            }

          }

          return true;

        }

      };

      MetaScanner.metaScan(conf, this, visitor, null);

      return available.get() && (regionCount.get() > 0);

    }

    /*

     * @param True if table is online

     */

    private boolean testTableOnlineState(byte [] tableName, boolean online)

    throws IOException {

      if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {

        // The root region is always enabled

        return online;

      }

      ZooKeeperWatcher zkw = getZooKeeperWatcher();

      String tableNameStr = Bytes.toString(tableName);

      try {

        if (online) {

          return ZKTableReadOnly.isEnabledTable(zkw, tableNameStr);

        }

        return ZKTableReadOnly.isDisabledTable(zkw, tableNameStr);

      } catch (KeeperException e) {

        throw new IOException("Enable/Disable failed", e);

      }

    }

    @Override

    public HRegionLocation locateRegion(final byte[] regionName) throws IOException {

      return locateRegion(HRegionInfo.getTableName(regionName),

        HRegionInfo.getStartKey(regionName), false, true);

    }

    @Override

    public List<HRegionLocation> locateRegions(final byte[] tableName)

    throws IOException {

      return locateRegions(tableName, false, true);

    }

    @Override

    public List<HRegionLocation> locateRegions(final byte[] tableName, final boolean useCache,

        final boolean offlined) throws IOException {

      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,

          tableName, offlined);

      final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();

      for (HRegionInfo regionInfo : regions.keySet()) {

        locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));

      }

      return locations;

    }

    public HRegionLocation locateRegion(final byte [] tableName,

        final byte [] row)

    throws IOException{

      return locateRegion(tableName, row, true, true);

    }

    public HRegionLocation relocateRegion(final byte [] tableName,

        final byte [] row)

    throws IOException{

      // Since this is an explicit request not to use any caching, finding

      // disabled tables should not be desirable.  This will ensure that an exception is thrown when

      // the first time a disabled table is interacted with.

      if (isTableDisabled(tableName)) {

        throw new DoNotRetryIOException(Bytes.toString(tableName) + " is disabled.");

      }

      return locateRegion(tableName, row, false, true);

    }

    private HRegionLocation locateRegion(final byte [] tableName,

      final byte [] row, boolean useCache, boolean retry)

    throws IOException {

      if (this.closed) throw new IOException(toString() + " closed");

      if (tableName == null || tableName.length == 0) {

        throw new IllegalArgumentException(

            "table name cannot be null or zero length");

      }

      ensureZookeeperTrackers();

      if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {

        try {

          ServerName servername = this.rootRegionTracker.waitRootRegionLocation(this.rpcTimeout);

          LOG.debug("Looked up root region location, connection=" + this +

            "; serverName=" + ((servername == null)? "": servername.toString()));

          if (servername == null) return null;

          return new HRegionLocation(HRegionInfo.ROOT_REGIONINFO,

            servername.getHostname(), servername.getPort());

        } catch (InterruptedException e) {

          Thread.currentThread().interrupt();

          return null;

        }

      } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {

        return locateRegionInMeta(HConstants.ROOT_TABLE_NAME, tableName, row,

            useCache, metaRegionLock, retry);

      } else {

        // Region not in the cache - have to go to the meta RS

        return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row,

            useCache, userRegionLock, retry);

      }

    }

    /*

     * Search .META. for the HRegionLocation info that contains the table and

     * row we're seeking. It will prefetch certain number of regions info and

     * save them to the global region cache.

     */

    private void prefetchRegionCache(final byte[] tableName,

        final byte[] row) {

      // Implement a new visitor for MetaScanner, and use it to walk through

      // the .META.

      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {

        public boolean processRow(Result result) throws IOException {

          try {

            byte[] value = result.getValue(HConstants.CATALOG_FAMILY,

                HConstants.REGIONINFO_QUALIFIER);

            HRegionInfo regionInfo = null;

            if (value != null) {

              // convert the row result into the HRegionLocation we need!

              regionInfo = Writables.getHRegionInfo(value);

              // possible we got a region of a different table...

              if (!Bytes.equals(regionInfo.getTableName(),

                  tableName)) {

                return false; // stop scanning

              }

              if (regionInfo.isOffline()) {

                // don't cache offline regions

                return true;

              }

              value = result.getValue(HConstants.CATALOG_FAMILY,

                  HConstants.SERVER_QUALIFIER);

              if (value == null) {

                return true;  // don't cache it

              }

              final String hostAndPort = Bytes.toString(value);

              String hostname = Addressing.parseHostname(hostAndPort);

              int port = Addressing.parsePort(hostAndPort);

              value = result.getValue(HConstants.CATALOG_FAMILY,

                  HConstants.STARTCODE_QUALIFIER);

              // instantiate the location

              HRegionLocation loc =

                new HRegionLocation(regionInfo, hostname, port);

              // cache this meta entry

              cacheLocation(tableName, loc);

            }

            return true;

          } catch (RuntimeException e) {

            throw new IOException(e);

          }

        }

      };

      try {

        // pre-fetch certain number of regions info at region cache.

        MetaScanner.metaScan(conf, this, visitor, tableName, row,

            this.prefetchRegionLimit, HConstants.META_TABLE_NAME);

      } catch (IOException e) {

        LOG.warn("Encountered problems when prefetch META table: ", e);

      }

    }

    /*

      * Search one of the meta tables (-ROOT- or .META.) for the HRegionLocation

      * info that contains the table and row we're seeking.

      */

    private HRegionLocation locateRegionInMeta(final byte [] parentTable,

      final byte [] tableName, final byte [] row, boolean useCache,

      Object regionLockObject, boolean retry)

    throws IOException {

      HRegionLocation location;

      // If we are supposed to be using the cache, look in the cache to see if

      // we already have the region.

      if (useCache) {

        location = getCachedLocation(tableName, row);

        if (location != null) {

          return location;

        }

      }

      int localNumRetries = retry ? numRetries : 1;