* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* TODO:pc remove application logic to a separate interface.

*/

  public static final Log LOG = LogFactory.getLog(

    HiveMetaStore.class);

  private static HadoopThriftAuthBridge.Server saslServer;

      ThriftHiveMetastore.Iface {

    public static final Log LOG = HiveMetaStore.LOG;

    private static boolean createDefaultDB = false;

    private String rawStoreClassName;

    private final HiveConf hiveConf; // stores datastore (jpox) properties,

                                     // right now they come from jpox.properties

    private Warehouse wh; // hdfs warehouse

    private final ThreadLocal<RawStore> threadLocalMS =

      new ThreadLocal<RawStore>() {

      @Override

      protected synchronized RawStore initialValue() {

        return null;

      }

    };

    // Thread local configuration is needed as many threads could make changes

    // to the conf using the connection hook

    private final ThreadLocal<Configuration> threadLocalConf =

      new ThreadLocal<Configuration>() {

      @Override

      protected synchronized Configuration initialValue() {

        return null;

      }

    };

    // The next serial number to be assigned

    private boolean checkForDefaultDb;

    private static int nextSerialNum = 0;

    private static ThreadLocal<Integer> threadLocalId = new ThreadLocal() {

      @Override

      protected synchronized Object initialValue() {

        return new Integer(nextSerialNum++);

      }

    };

    // Used for retrying JDO calls on datastore failures

    private int retryInterval = 0;

    private int retryLimit = 0;

    private JDOConnectionURLHook urlHook = null;

    private String urlHookClassName = "";

    public static Integer get() {

      return threadLocalId.get();

    }

    public HMSHandler(String name) throws MetaException {

      super(name);

      hiveConf = new HiveConf(this.getClass());

      init();

    }

    public HMSHandler(String name, HiveConf conf) throws MetaException {

      super(name);

      hiveConf = conf;

      init();

    }

    public HiveConf getHiveConf() {

      return hiveConf;

    }

    private ClassLoader classLoader;

    private AlterHandler alterHandler;

    {

      classLoader = Thread.currentThread().getContextClassLoader();

      if (classLoader == null) {

        classLoader = Configuration.class.getClassLoader();

      }

    }

    private boolean init() throws MetaException {

      rawStoreClassName = hiveConf.get("hive.metastore.rawstore.impl");

      checkForDefaultDb = hiveConf.getBoolean(

          "hive.metastore.checkForDefaultDb", true);

      String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",

          HiveAlterHandler.class.getName());

      alterHandler = (AlterHandler) ReflectionUtils.newInstance(getClass(

          alterHandlerName, AlterHandler.class), hiveConf);

      wh = new Warehouse(hiveConf);

      retryInterval = HiveConf.getIntVar(hiveConf,

          HiveConf.ConfVars.METASTOREINTERVAL);

      retryLimit = HiveConf.getIntVar(hiveConf,

          HiveConf.ConfVars.METASTOREATTEMPTS);

      // Using the hook on startup ensures that the hook always has priority

      // over settings in *.xml. We can use hiveConf as only a single thread

      // will be calling the constructor.

      updateConnectionURL(hiveConf, null);

      createDefaultDB();

      return true;

    }

    private String addPrefix(String s) {

      return threadLocalId.get() + ": " + s;

    }

    /**

     * A Command is a closure used to pass a block of code from individual

     * functions to executeWithRetry, which centralizes connection error

     * handling. Command is parameterized on the return type of the function.

     *

     * The general transformation is:

     *

     * From:

     * String foo(int a) throws ExceptionB {

     *   <block of code>

     * }

     *

     * To:

     * String foo(final int a) throws ExceptionB {

     *   String ret =  null;

     *   try {

     *     ret = executeWithRetry(new Command<Boolean>() {

     *       String run(RawStore ms) {

     *         <block of code>

     *       }

     *     }

     *   } catch (ExceptionB e) {

     *     throw e;

     *   } catch (Exception e) {

     *     // Since run is only supposed to throw ExceptionB it could only

     *     // be a runtime exception

     *     throw (RuntimeException)e;

     *   }

     * }

     *

     * The catch blocks are used to ensure that the exceptions thrown by the

     * <block of code> follow the function definition.

     */

    private static class Command<T> {

      T run(RawStore ms) throws Exception {

        return null;

      }

    }

    private <T> T executeWithRetry(Command<T> cmd) throws Exception {

      T ret = null;

      boolean gotNewConnectUrl = false;

      boolean reloadConf = HiveConf.getBoolVar(hiveConf,

          HiveConf.ConfVars.METASTOREFORCERELOADCONF);

      if (reloadConf) {

        updateConnectionURL(getConf(), null);

      }

      int retryCount = 0;

      Exception caughtException = null;

      while(true) {

        try {

          RawStore ms = getMS(reloadConf || gotNewConnectUrl);

          ret = cmd.run(ms);

          break;

        } catch (javax.jdo.JDOFatalDataStoreException e) {

          caughtException = e;

        } catch (javax.jdo.JDODataStoreException e) {

          caughtException = e;

        }

        if (retryCount >= retryLimit) {

          throw caughtException;

        }

        assert(retryInterval >= 0);

        retryCount++;

        LOG.error(

            String.format(

                "JDO datastore error. Retrying metastore command " +

                "after %d ms (attempt %d of %d)", retryInterval, retryCount, retryLimit));

        Thread.sleep(retryInterval);

        // If we have a connection error, the JDO connection URL hook might

        // provide us with a new URL to access the datastore.

        String lastUrl = getConnectionURL(getConf());

        gotNewConnectUrl = updateConnectionURL(getConf(), lastUrl);

      }

      return ret;

    }

    private Configuration getConf() {

      Configuration conf = threadLocalConf.get();

      if (conf == null) {

        conf = new Configuration(hiveConf);

        threadLocalConf.set(conf);

      }

      return conf;

    }

    /**

     * Get a cached RawStore.

     *

     * @return

     * @throws MetaException

     */

    private RawStore getMS(boolean reloadConf) throws MetaException {

      RawStore ms = threadLocalMS.get();

      if (ms == null) {

        LOG.info(addPrefix("Opening raw store with implemenation class:"

            + rawStoreClassName));

        ms = (RawStore) ReflectionUtils.newInstance(getClass(rawStoreClassName,

            RawStore.class), getConf());

        threadLocalMS.set(ms);

        ms = threadLocalMS.get();

      }

      if (reloadConf) {

        ms.setConf(getConf());

      }

      return ms;

    }

    /**

     * Updates the connection URL in hiveConf using the hook

     * @return true if a new connection URL was loaded into the thread local

     * configuration

     */

    private boolean updateConnectionURL(Configuration conf, String badUrl)

        throws MetaException {

      String connectUrl = null;

      String currentUrl = getConnectionURL(conf);

      try {

        // We always call init because the hook name in the configuration could

        // have changed.

        initConnectionUrlHook();

        if (urlHook != null) {

          if (badUrl != null) {

            urlHook.notifyBadConnectionUrl(badUrl);

          }

          connectUrl = urlHook.getJdoConnectionUrl(hiveConf);

        }

      } catch (Exception e) {

        LOG.error("Exception while getting connection URL from the hook: " +

            e);

      }

      if (connectUrl != null && !connectUrl.equals(currentUrl)) {

        LOG.error(addPrefix(

            String.format("Overriding %s with %s",

                HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),

                connectUrl)));

        conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),

            connectUrl);

        return true;

      }

      return false;

    }

    private static String getConnectionURL(Configuration conf) {

      return conf.get(

          HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),"");

    }

    // Multiple threads could try to initialize at the same time.

    synchronized private void initConnectionUrlHook()

        throws ClassNotFoundException {

      String className =

        hiveConf.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim();

      if (className.equals("")){

        urlHookClassName = "";

        urlHook = null;

        return;

      }

      boolean urlHookChanged = !urlHookClassName.equals(className);

      if (urlHook == null || urlHookChanged) {

        urlHookClassName = className.trim();

        Class <?> urlHookClass = Class.forName(urlHookClassName, true,

            JavaUtils.getClassLoader());

        urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);

      }

      return;

    }

    private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException {

      try {

        ms.getDatabase(DEFAULT_DATABASE_NAME);

      } catch (NoSuchObjectException e) {

        ms.createDatabase(

            new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT,

                wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null));

      }

      HMSHandler.createDefaultDB = true;

    }

    /**

     * create default database if it doesn't exist

     *

     * @throws MetaException

     */

    private void createDefaultDB() throws MetaException {

      if (HMSHandler.createDefaultDB || !checkForDefaultDb) {

        return;

      }

      try {

        executeWithRetry(new Command<Boolean>() {

          @Override

          Boolean run(RawStore ms) throws Exception {

            createDefaultDB_core(ms);

            return Boolean.TRUE;

          }

        });

      } catch (InvalidObjectException e) {

        throw new MetaException(e.getMessage());

      } catch (MetaException e) {

        throw e;

      } catch (Exception e) {

        assert(e instanceof RuntimeException);

        throw (RuntimeException)e;

      }

    }

    private Class<?> getClass(String rawStoreClassName, Class<?> class1)

        throws MetaException {

      try {

        return Class.forName(rawStoreClassName, true, classLoader);

      } catch (ClassNotFoundException e) {

        throw new MetaException(rawStoreClassName + " class not found");

      }

    }

    private void logStartFunction(String m) {

      LOG.info(threadLocalId.get().toString() + ": " + m);

    }

    private void logStartTableFunction(String f, String db, String tbl) {

      LOG.info(threadLocalId.get().toString() + ": " + f + " : db=" + db + " tbl=" + tbl);

    }

    private void logStartPartitionFunction(String f, String db, String tbl, List<String> partVals) {

      LOG.info(threadLocalId.get().toString() + ": " + f + " : db=" + db + " tbl=" + tbl

          + "[" + join(partVals, ",") + "]");

    }

    @Override

    public fb_status getStatus() {

      return fb_status.ALIVE;

    }

    @Override

    public void shutdown() {

      logStartFunction("Shutting down the object store...");

      RawStore ms = threadLocalMS.get();

      if (ms != null) {

        ms.shutdown();

      }

      System.exit(0);

    }

    private void create_database_core(RawStore ms, final Database db)

        throws AlreadyExistsException, InvalidObjectException, MetaException {

      if (!validateName(db.getName())) {

        throw new InvalidObjectException(db.getName() + " is not a valid database name");

      }

      boolean success = false;

      try {

        ms.openTransaction();

        if (null == db.getLocationUri()) {

          db.setLocationUri(wh.getDefaultDatabasePath(db.getName()).toString());

        }

        ms.createDatabase(db);

        success = ms.commitTransaction();

      } finally {

        if (!success) {

          ms.rollbackTransaction();

        } else {

          wh.mkdirs(new Path(db.getLocationUri()));

        }

      }

    }

    public void create_database(final Database db)

        throws AlreadyExistsException, InvalidObjectException, MetaException {

      incrementCounter("create_database");

      logStartFunction("create_database: "

                       + db.getName() + " "

                       + db.getLocationUri() + " "

                       + db.getDescription());

      try {

        try {

          if(null != get_database(db.getName())) {

            throw new AlreadyExistsException("Database " + db.getName() + " already exists");

          }

        } catch (NoSuchObjectException e) {

          // expected

        }

        executeWithRetry(new Command<Boolean>() {

          @Override

          Boolean run(RawStore ms) throws Exception {

            create_database_core(ms, db);

            return Boolean.TRUE;

          }

        });

      } catch (AlreadyExistsException e) {

        throw e;

      } catch (InvalidObjectException e) {

        throw e;

      } catch (MetaException e) {

        throw e;

      } catch (Exception e) {

        assert(e instanceof RuntimeException);

        throw (RuntimeException)e;

      }

    }

    public Database get_database(final String name) throws NoSuchObjectException,

        MetaException {

      incrementCounter("get_database");

      logStartFunction("get_database: " + name);

      Database db = null;

      try {

        db = executeWithRetry(new Command<Database>() {

          @Override

          Database run(RawStore ms) throws Exception {

            return ms.getDatabase(name);

          }

        });

      } catch (MetaException e) {

        throw e;

      } catch (NoSuchObjectException e) {

        throw e;

      } catch (Exception e) {

        assert(e instanceof RuntimeException);

        throw (RuntimeException)e;

      }

      return db;

    }

    public void alter_database(final String dbName, final Database db)

        throws NoSuchObjectException, TException, MetaException {

      incrementCounter("alter_database");

      logStartFunction("alter_database" + dbName);

      try {

        executeWithRetry(new Command<Boolean>() {

          @Override

          Boolean run(RawStore ms) throws Exception {

            return ms.alterDatabase(dbName, db);

          }