*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A non-instantiable class that manages creation of {@link HConnection}s.

* <p>The simplest way to use this class is by using {@link #createConnection(Configuration)}.

* This creates a new {@link HConnection} to the cluster that is managed by the caller.

* From this {@link HConnection} {@link HTableInterface} implementations are retrieved

* with {@link HConnection#getTable(byte[])}. Example:

* <pre>

* {@code

* HConnection connection = HConnectionManager.createConnection(config);

* HTableInterface table = connection.getTable("table1");

* try {

*   // Use the table as needed, for a single operation and a single thread

* } finally {

*   table.close();

*   connection.close();

* }

* }</pre>

* <p>The following logic and API will be removed in the future:

* <p>This class has a static Map of {@link HConnection} instances keyed by

* {@link Configuration}; all invocations of {@link #getConnection(Configuration)}

* that pass the same {@link Configuration} instance will be returned the same

* {@link  HConnection} instance (Adding properties to a Configuration

* instance does not change its object identity; for more on how this is done see

* {@link HConnectionKey}).  Sharing {@link HConnection}

* instances is usually what you want; all clients of the {@link HConnection}

* instances share the HConnections' cache of Region locations rather than each

* having to discover for itself the location of meta, etc.  It makes

* sense for the likes of the pool of HTables class {@link HTablePool}, for

* instance (If concerned that a single {@link HConnection} is insufficient

* for sharing amongst clients in say an heavily-multithreaded environment,

* in practise its not proven to be an issue.  Besides, {@link HConnection} is

* implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a

* connection per cluster-member, exclusively).

*

* <p>But sharing connections makes clean up of {@link HConnection} instances a little awkward.

* Currently, clients cleanup by calling {@link #deleteConnection(Configuration)}. This will

* shutdown the zookeeper connection the HConnection was using and clean up all

* HConnection resources as well as stopping proxies to servers out on the

* cluster. Not running the cleanup will not end the world; it'll

* just stall the closeup some and spew some zookeeper connection failed

* messages into the log.  Running the cleanup on a {@link HConnection} that is

* subsequently used by another will cause breakage so be careful running

* cleanup.

* <p>To create a {@link HConnection} that is not shared by others, you can

* create a new {@link Configuration} instance, pass this new instance to

* {@link #getConnection(Configuration)}, and then when done, close it up by

* doing something like the following:

* <pre>

* {@code

* Configuration newConfig = new Configuration(originalConf);

* HConnection connection = HConnectionManager.getConnection(newConfig);

* // Use the connection to your hearts' delight and then when done...

* HConnectionManager.deleteConnection(newConfig, true);

* }

* </pre>

* <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd

* register a shutdown hook that called {@link #deleteAllConnections()}

* on its way out but the order in which shutdown hooks run is not defined so

* were problematic for clients of HConnection that wanted to register their

* own shutdown hooks so we removed ours though this shifts the onus for

* cleanup to the client.

*/

@InterfaceStability.Evolving

  static final Log LOG = LogFactory.getLog(HConnectionManager.class);

  public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";

  private static final String CLIENT_NONCES_ENABLED_KEY = "hbase.client.nonces.enabled";

  // An LRU Map of HConnectionKey -> HConnection (TableServer).  All

  // access must be synchronized.  This map is not private because tests

  // need to be able to tinker with it.

  static final Map<HConnectionKey, HConnectionImplementation> CONNECTION_INSTANCES;

  public static final int MAX_CACHED_CONNECTION_INSTANCES;

  /**

   * Global nonceGenerator shared per client.Currently there's no reason to limit its scope.

   * Once it's set under nonceGeneratorCreateLock, it is never unset or changed.

   */

  private static volatile NonceGenerator nonceGenerator = null;

  /** The nonce generator lock. Only taken when creating HConnection, which gets a private copy. */

  private static Object nonceGeneratorCreateLock = new Object();

  static {

    // We set instances to one more than the value specified for {@link

    // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max

    // connections to the ensemble from the one client is 30, so in that case we

    // should run into zk issues before the LRU hit this value of 31.

    MAX_CACHED_CONNECTION_INSTANCES = HBaseConfiguration.create().getInt(

      HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;

    CONNECTION_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(

        (int) (MAX_CACHED_CONNECTION_INSTANCES / 0.75F) + 1, 0.75F, true) {

      @Override

      protected boolean removeEldestEntry(

          Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {

         return size() > MAX_CACHED_CONNECTION_INSTANCES;

       }

    };

  }

  /*

   * Non-instantiable.

   */

  private HConnectionManager() {

    super();

  }

  /**

   * @param conn The connection for which to replace the generator.

   * @param cnm Replaces the nonce generator used, for testing.

   * @return old nonce generator.

   */

  @VisibleForTesting

  public static NonceGenerator injectNonceGeneratorForTesting(

      HConnection conn, NonceGenerator cnm) {

    NonceGenerator ng = conn.getNonceGenerator();

    LOG.warn("Nonce generator is being replaced by test code for " + cnm.getClass().getName());

    ((HConnectionImplementation)conn).nonceGenerator = cnm;

    return ng;

  }

  /**

   * Get the connection that goes with the passed <code>conf</code> configuration instance.

   * If no current connection exists, method creates a new connection and keys it using

   * connection-specific properties from the passed {@link Configuration}; see

   * {@link HConnectionKey}.

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  @Deprecated

  public static HConnection getConnection(final Configuration conf)

  throws IOException {

    HConnectionKey connectionKey = new HConnectionKey(conf);

    synchronized (CONNECTION_INSTANCES) {

      HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);

      if (connection == null) {

        connection = (HConnectionImplementation)createConnection(conf, true);

        CONNECTION_INSTANCES.put(connectionKey, connection);

      } else if (connection.isClosed()) {

        HConnectionManager.deleteConnection(connectionKey, true);

        connection = (HConnectionImplementation)createConnection(conf, true);

        CONNECTION_INSTANCES.put(connectionKey, connection);

      }

      connection.incCount();

      return connection;

    }

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code> instance.

   * <p>Note: This bypasses the usual HConnection life cycle management done by

   * {@link #getConnection(Configuration)}. The caller is responsible for

   * calling {@link HConnection#close()} on the returned connection instance.

   *

   * This is the recommended way to create HConnections.

   * {@code

   * HConnection connection = HConnectionManager.createConnection(conf);

   * HTableInterface table = connection.getTable("mytable");

   * table.get(...);

   * ...

   * table.close();

   * connection.close();

   * }

   *

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf)

  throws IOException {

    UserProvider provider = UserProvider.instantiate(conf);

    return createConnection(conf, false, null, provider.getCurrent());

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code> instance.

   * <p>Note: This bypasses the usual HConnection life cycle management done by

   * {@link #getConnection(Configuration)}. The caller is responsible for

   * calling {@link HConnection#close()} on the returned connection instance.

   * This is the recommended way to create HConnections.

   * {@code

   * ExecutorService pool = ...;

   * HConnection connection = HConnectionManager.createConnection(conf, pool);

   * HTableInterface table = connection.getTable("mytable");

   * table.get(...);

   * ...

   * table.close();

   * connection.close();

   * }

   * @param conf configuration

   * @param pool the thread pool to use for batch operation in HTables used via this HConnection

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf, ExecutorService pool)

  throws IOException {

    UserProvider provider = UserProvider.instantiate(conf);

    return createConnection(conf, false, pool, provider.getCurrent());

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code> instance.

   * <p>Note: This bypasses the usual HConnection life cycle management done by

   * {@link #getConnection(Configuration)}. The caller is responsible for

   * calling {@link HConnection#close()} on the returned connection instance.

   * This is the recommended way to create HConnections.

   * {@code

   * ExecutorService pool = ...;

   * HConnection connection = HConnectionManager.createConnection(conf, pool);

   * HTableInterface table = connection.getTable("mytable");

   * table.get(...);

   * ...

   * table.close();

   * connection.close();

   * }

   * @param conf configuration

   * @param user the user the connection is for

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf, User user)

  throws IOException {

    return createConnection(conf, false, null, user);

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code> instance.

   * <p>Note: This bypasses the usual HConnection life cycle management done by

   * {@link #getConnection(Configuration)}. The caller is responsible for

   * calling {@link HConnection#close()} on the returned connection instance.

   * This is the recommended way to create HConnections.

   * {@code

   * ExecutorService pool = ...;

   * HConnection connection = HConnectionManager.createConnection(conf, pool);

   * HTableInterface table = connection.getTable("mytable");

   * table.get(...);

   * ...

   * table.close();

   * connection.close();

   * }

   * @param conf configuration

   * @param pool the thread pool to use for batch operation in HTables used via this HConnection

   * @param user the user the connection is for

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf, ExecutorService pool, User user)

  throws IOException {

    return createConnection(conf, false, pool, user);

  }

  @Deprecated

  static HConnection createConnection(final Configuration conf, final boolean managed)

      throws IOException {

    UserProvider provider = UserProvider.instantiate(conf);

    return createConnection(conf, managed, null, provider.getCurrent());

  }

  @Deprecated

  static HConnection createConnection(final Configuration conf, final boolean managed,

      final ExecutorService pool, final User user)

  throws IOException {

    String className = conf.get("hbase.client.connection.impl",

      HConnectionManager.HConnectionImplementation.class.getName());

    Class<?> clazz = null;

    try {

      clazz = Class.forName(className);

    } catch (ClassNotFoundException e) {

      throw new IOException(e);

    }

    try {

      // Default HCM#HCI is not accessible; make it so before invoking.

      Constructor<?> constructor =

        clazz.getDeclaredConstructor(Configuration.class,

          boolean.class, ExecutorService.class, User.class);

      constructor.setAccessible(true);

      return (HConnection) constructor.newInstance(conf, managed, pool, user);

    } catch (Exception e) {

      throw new IOException(e);

    }

  }

  /**

   * Delete connection information for the instance specified by passed configuration.

   * If there are no more references to the designated connection connection, this method will

   * then close connection to the zookeeper ensemble and let go of all associated resources.

   *

   * @param conf configuration whose identity is used to find {@link HConnection} instance.

   * @deprecated

   */

  public static void deleteConnection(Configuration conf) {

    deleteConnection(new HConnectionKey(conf), false);

  }

  /**

   * Cleanup a known stale connection.

   * This will then close connection to the zookeeper ensemble and let go of all resources.

   *

   * @param connection

   * @deprecated

   */

  public static void deleteStaleConnection(HConnection connection) {

    deleteConnection(connection, true);

  }

  /**

   * Delete information for all connections. Close or not the connection, depending on the

   *  staleConnection boolean and the ref count. By default, you should use it with

   *  staleConnection to true.

   * @deprecated

   */

  public static void deleteAllConnections(boolean staleConnection) {

    synchronized (CONNECTION_INSTANCES) {

      Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();

      connectionKeys.addAll(CONNECTION_INSTANCES.keySet());

      for (HConnectionKey connectionKey : connectionKeys) {

        deleteConnection(connectionKey, staleConnection);

      }

      CONNECTION_INSTANCES.clear();

    }

  }

  /**

   * Delete information for all connections..

   * @deprecated kept for backward compatibility, but the behavior is broken. HBASE-8983

   */

  @Deprecated

  public static void deleteAllConnections() {

    deleteAllConnections(false);

  }

  @Deprecated

  private static void deleteConnection(HConnection connection, boolean staleConnection) {

    synchronized (CONNECTION_INSTANCES) {

      for (Entry<HConnectionKey, HConnectionImplementation> e: CONNECTION_INSTANCES.entrySet()) {

        if (e.getValue() == connection) {

          deleteConnection(e.getKey(), staleConnection);

          break;

        }

      }

    }

  }

  @Deprecated

  private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {

    synchronized (CONNECTION_INSTANCES) {

      HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);

      if (connection != null) {

        connection.decCount();

        if (connection.isZeroReference() || staleConnection) {

          CONNECTION_INSTANCES.remove(connectionKey);

          connection.internalClose();

        }

      } else {

        LOG.error("Connection not found in the list, can't delete it "+

          "(connection key=" + connectionKey + "). May be the key was modified?", new Exception());

      }

    }

  }

  /**

   * It is provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return Number of cached regions for the table.

   * @throws ZooKeeperConnectionException

   */

  static int getCachedRegionCount(Configuration conf, final TableName tableName)

  throws IOException {

    return execute(new HConnectable<Integer>(conf) {

      @Override

      public Integer connect(HConnection connection) {

        return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName);

      }

    });

  }

  /**

   * It's provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return true if the region where the table and row reside is cached.

   * @throws ZooKeeperConnectionException

   */

  static boolean isRegionCached(Configuration conf,

                                final TableName tableName,

                                final byte[] row)

  throws IOException {

    return execute(new HConnectable<Boolean>(conf) {

      @Override

      public Boolean connect(HConnection connection) {

        return ((HConnectionImplementation) connection).isRegionCached(tableName, row);

      }

    });

  }

  /**

   * This convenience method invokes the given {@link HConnectable#connect}

   * implementation using a {@link HConnection} instance that lasts just for the

   * duration of the invocation.

   *

   * @param <T> the return type of the connect method

   * @param connectable the {@link HConnectable} instance

   * @return the value returned by the connect method

   * @throws IOException

   */

  @InterfaceAudience.Private

  public static <T> T execute(HConnectable<T> connectable) throws IOException {

    if (connectable == null || connectable.conf == null) {

      return null;

    }

    Configuration conf = connectable.conf;

    HConnection connection = HConnectionManager.getConnection(conf);

    boolean connectSucceeded = false;

    try {

      T returnValue = connectable.connect(connection);

      connectSucceeded = true;

      return returnValue;

    } finally {

      try {

        connection.close();

      } catch (Exception e) {

        ExceptionUtil.rethrowIfInterrupt(e);

        if (connectSucceeded) {

          throw new IOException("The connection to " + connection

              + " could not be deleted.", e);

        }

      }

    }

  }

  @edu.umd.cs.findbugs.annotations.SuppressWarnings(

      value="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION",

      justification="Access to the conncurrent hash map is under a lock so should be fine.")

  static class HConnectionImplementation implements HConnection, Closeable {

    static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);

    private final long pause;

    private final int numTries;

    final int rpcTimeout;

    private NonceGenerator nonceGenerator = null;

    private final int prefetchRegionLimit;

    private volatile boolean closed;

    private volatile boolean aborted;

    // package protected for the tests

    ClusterStatusListener clusterStatusListener;

    private final Object userRegionLock = new Object();

    // We have a single lock for master & zk to prevent deadlocks. Having

    //  one lock for ZK and one lock for master is not possible:

    //  When creating a connection to master, we need a connection to ZK to get

    //  its address. But another thread could have taken the ZK lock, and could

    //  be waiting for the master lock => deadlock.

    private final Object masterAndZKLock = new Object();

    private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;

    private final DelayedClosing delayedClosing =

      DelayedClosing.createAndStart(this);

    // thread executor shared by all HTableInterface instances created

    // by this connection

    private volatile ExecutorService batchPool = null;

    private volatile boolean cleanupPool = false;

    private final Configuration conf;

    // Client rpc instance.

    private RpcClient rpcClient;

    /**

      * Map of table to table {@link HRegionLocation}s.

      */

    private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>

        cachedRegionLocations =

      new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>();

    // The presence of a server in the map implies it's likely that there is an

    // entry in cachedRegionLocations that map to this server; but the absence

    // of a server in this map guarentees that there is no entry in cache that

    // maps to the absent server.

    // The access to this attribute must be protected by a lock on cachedRegionLocations

    private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();

    // region cache prefetch is enabled by default. this set contains all

    // tables whose region cache prefetch are disabled.

    private final Set<Integer> regionCachePrefetchDisabledTables =

      new CopyOnWriteArraySet<Integer>();

    private int refCount;

    // indicates whether this connection's life cycle is managed (by us)

    private boolean managed;

    private User user;

    /**

     * Cluster registry of basic info such as clusterid and meta region location.

     */

     Registry registry;

     HConnectionImplementation(Configuration conf, boolean managed) throws IOException {

       this(conf, managed, null, null);

     }

    /**

     * constructor

     * @param conf Configuration object

     * @param managed If true, does not do full shutdown on close; i.e. cleanup of connection

     * to zk and shutdown of all services; we just close down the resources this connection was

     * responsible for and decrement usage counters.  It is up to the caller to do the full

     * cleanup.  It is set when we want have connection sharing going on -- reuse of zk connection,

     * and cached region locations, established regionserver connections, etc.  When connections

     * are shared, we have reference counting going on and will only do full cleanup when no more

     * users of an HConnectionImplementation instance.

     */

    HConnectionImplementation(Configuration conf, boolean managed,

        ExecutorService pool, User user) throws IOException {

      this(conf);

      this.user = user;

      this.batchPool = pool;

      this.managed = managed;

      this.registry = setupRegistry();

      retrieveClusterId();

      this.rpcClient = new RpcClient(this.conf, this.clusterId);

      // Do we publish the status?

      boolean shouldListen = conf.getBoolean(HConstants.STATUS_PUBLISHED,

          HConstants.STATUS_PUBLISHED_DEFAULT);

      Class<? extends ClusterStatusListener.Listener> listenerClass =

          conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,

              ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,

              ClusterStatusListener.Listener.class);

      if (shouldListen) {

        if (listenerClass == null) {

          LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +

              ClusterStatusListener.STATUS_LISTENER_CLASS + " is not set - not listening status");

        } else {

          clusterStatusListener = new ClusterStatusListener(

              new ClusterStatusListener.DeadServerHandler() {

                @Override

                public void newDead(ServerName sn) {

                  clearCaches(sn);

                  rpcClient.cancelConnections(sn.getHostname(), sn.getPort(),

                      new SocketException(sn.getServerName() +

                          " is dead: closing its connection."));

                }

              }, conf, listenerClass);

        }

      }

    }

    /** Dummy nonce generator for disabled nonces. */

    private static class NoNonceGenerator implements NonceGenerator {

      @Override

      public long getNonceGroup() {

        return HConstants.NO_NONCE;

      }

      @Override

      public long newNonce() {

        return HConstants.NO_NONCE;

      }

    }

    /**

     * For tests.

     */

    protected HConnectionImplementation(Configuration conf) {

      this.conf = conf;

      this.closed = false;

      this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,

          HConstants.DEFAULT_HBASE_CLIENT_PAUSE);

      this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,

          HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);

      this.rpcTimeout = conf.getInt(

          HConstants.HBASE_RPC_TIMEOUT_KEY,

          HConstants.DEFAULT_HBASE_RPC_TIMEOUT);

      if (conf.getBoolean(CLIENT_NONCES_ENABLED_KEY, true)) {

        synchronized (HConnectionManager.nonceGeneratorCreateLock) {

          if (HConnectionManager.nonceGenerator == null) {

            HConnectionManager.nonceGenerator = new PerClientRandomNonceGenerator();

          }

          this.nonceGenerator = HConnectionManager.nonceGenerator;

        }

      } else {

        this.nonceGenerator = new NoNonceGenerator();

      }

      this.prefetchRegionLimit = conf.getInt(

          HConstants.HBASE_CLIENT_PREFETCH_LIMIT,

          HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);

    }

    @Override

    public HTableInterface getTable(String tableName) throws IOException {

      return getTable(TableName.valueOf(tableName));

    }

    @Override

    public HTableInterface getTable(byte[] tableName) throws IOException {

      return getTable(TableName.valueOf(tableName));

    }

    @Override

    public HTableInterface getTable(TableName tableName) throws IOException {

      return getTable(tableName, getBatchPool());

    }

    @Override