* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This class has functions that implement meta data/DDL operations using calls

* to the metastore.

* It has a metastore client instance it uses to communicate with the metastore.

*

* It is a thread local variable, and the instances is accessed using static

* get methods in this class.

*/

  static final private Log LOG = LogFactory.getLog("hive.ql.metadata.Hive");

  private HiveConf conf = null;

  private IMetaStoreClient metaStoreClient;

  private static ThreadLocal<Hive> hiveDB = new ThreadLocal<Hive>() {

    @Override

    protected synchronized Hive initialValue() {

      return null;

    }

    @Override

    public synchronized void remove() {

      if (this.get() != null) {

        this.get().close();

      }

      super.remove();

    }

  };

  /**

   * Gets hive object for the current thread. If one is not initialized then a

   * new one is created If the new configuration is different in metadata conf

   * vars then a new one is created.

   *

   * @param c

   *          new Hive Configuration

   * @return Hive object for current thread

   * @throws HiveException

   *

   */

  public static Hive get(HiveConf c) throws HiveException {

    boolean needsRefresh = false;

    Hive db = hiveDB.get();

    if (db != null) {

      for (HiveConf.ConfVars oneVar : HiveConf.metaVars) {

        // Since metaVars are all of different types, use string for comparison

        String oldVar = db.getConf().get(oneVar.varname, "");

        String newVar = c.get(oneVar.varname, "");

        if (oldVar.compareToIgnoreCase(newVar) != 0) {

          needsRefresh = true;

          break;

        }

      }

    }

    return get(c, needsRefresh);

  }

  /**

   * get a connection to metastore. see get(HiveConf) function for comments

   *

   * @param c

   *          new conf

   * @param needsRefresh

   *          if true then creates a new one

   * @return The connection to the metastore

   * @throws HiveException

   */

  public static Hive get(HiveConf c, boolean needsRefresh) throws HiveException {

    Hive db = hiveDB.get();

    if (db == null || needsRefresh) {

      closeCurrent();

      c.set("fs.scheme.class", "dfs");

      Hive newdb = new Hive(c);

      hiveDB.set(newdb);

      return newdb;

    }

    db.conf = c;

    return db;

  }

  public static Hive get() throws HiveException {

    Hive db = hiveDB.get();

    if (db == null) {

      db = new Hive(new HiveConf(Hive.class));

      hiveDB.set(db);

    }

    return db;

  }

  public static void set(Hive hive) {

    hiveDB.set(hive);

  }

  public static void closeCurrent() {

    hiveDB.remove();

  }

  /**

   * Hive

   *

   * @param argFsRoot

   * @param c

   *

   */

  private Hive(HiveConf c) throws HiveException {

    conf = c;

  }

  /**

   * closes the connection to metastore for the calling thread

   */

  private void close() {

    LOG.debug("Closing current thread's connection to Hive Metastore.");

    if (metaStoreClient != null) {

      metaStoreClient.close();

      metaStoreClient = null;

    }

  }

  /**

   * Create a database

   * @param db

   * @param ifNotExist if true, will ignore AlreadyExistsException exception

   * @throws AlreadyExistsException

   * @throws HiveException

   */

  public void createDatabase(Database db, boolean ifNotExist)

      throws AlreadyExistsException, HiveException {

    try {

      getMSC().createDatabase(db);

    } catch (AlreadyExistsException e) {

      if (!ifNotExist) {

        throw e;

      }

    } catch (Exception e) {

      throw new HiveException(e);

    }

  }

  /**

   * Create a Database. Raise an error if a database with the same name already exists.

   * @param db

   * @throws AlreadyExistsException

   * @throws HiveException

   */

  public void createDatabase(Database db) throws AlreadyExistsException, HiveException {

    createDatabase(db, false);

  }

  /**

   * Drop a database.

   * @param name

   * @throws NoSuchObjectException

   * @throws HiveException

   * @see org.apache.hadoop.hive.metastore.HiveMetaStoreClient#dropDatabase(java.lang.String)

   */

  public void dropDatabase(String name) throws HiveException, NoSuchObjectException {

    dropDatabase(name, true, false, false);

  }

  /**

   * Drop a database

   * @param name

   * @param deleteData

   * @param ignoreUnknownDb if true, will ignore NoSuchObjectException

   * @throws HiveException

   * @throws NoSuchObjectException

   */

  public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownDb)

      throws HiveException, NoSuchObjectException {

    dropDatabase(name, deleteData, ignoreUnknownDb, false);

  }

  /**

   * Drop a database

   * @param name

   * @param deleteData

   * @param ignoreUnknownDb if true, will ignore NoSuchObjectException

   * @param cascade           if true, delete all tables on the DB if exists. Othewise, the query

   *                        will fail if table still exists.

   * @throws HiveException

   * @throws NoSuchObjectException

   */

  public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownDb, boolean cascade)

      throws HiveException, NoSuchObjectException {

    try {

      getMSC().dropDatabase(name, deleteData, ignoreUnknownDb, cascade);

    } catch (NoSuchObjectException e) {

      throw e;

    } catch (Exception e) {

      throw new HiveException(e);

    }

  }

  /**

   * Creates a table metdata and the directory for the table data

   *

   * @param tableName

   *          name of the table

   * @param columns

   *          list of fields of the table

   * @param partCols

   *          partition keys of the table

   * @param fileInputFormat

   *          Class of the input format of the table data file

   * @param fileOutputFormat

   *          Class of the output format of the table data file

   * @throws HiveException

   *           thrown if the args are invalid or if the metadata or the data

   *           directory couldn't be created

   */

  public void createTable(String tableName, List<String> columns,

      List<String> partCols, Class<? extends InputFormat> fileInputFormat,

      Class<?> fileOutputFormat) throws HiveException {

    this.createTable(tableName, columns, partCols, fileInputFormat,

        fileOutputFormat, -1, null);

  }

  /**

   * Creates a table metdata and the directory for the table data

   *

   * @param tableName

   *          name of the table

   * @param columns

   *          list of fields of the table

   * @param partCols

   *          partition keys of the table

   * @param fileInputFormat

   *          Class of the input format of the table data file

   * @param fileOutputFormat

   *          Class of the output format of the table data file

   * @param bucketCount

   *          number of buckets that each partition (or the table itself) should

   *          be divided into

   * @throws HiveException

   *           thrown if the args are invalid or if the metadata or the data

   *           directory couldn't be created

   */

  public void createTable(String tableName, List<String> columns,

      List<String> partCols, Class<? extends InputFormat> fileInputFormat,

      Class<?> fileOutputFormat, int bucketCount, List<String> bucketCols)

      throws HiveException {

    if (columns == null) {

      throw new HiveException("columns not specified for table " + tableName);

    }

    Table tbl = newTable(tableName);

    tbl.setInputFormatClass(fileInputFormat.getName());

    tbl.setOutputFormatClass(fileOutputFormat.getName());

    for (String col : columns) {

      FieldSchema field = new FieldSchema(col, STRING_TYPE_NAME, "default");

      tbl.getCols().add(field);

    }

    if (partCols != null) {

      for (String partCol : partCols) {

        FieldSchema part = new FieldSchema();

        part.setName(partCol);

        part.setType(STRING_TYPE_NAME); // default partition key

        tbl.getPartCols().add(part);

      }

    }

    tbl.setSerializationLib(LazySimpleSerDe.class.getName());

    tbl.setNumBuckets(bucketCount);

    tbl.setBucketCols(bucketCols);

    createTable(tbl);

  }

  /**

   * Updates the existing table metadata with the new metadata.

   *

   * @param tblName

   *          name of the existing table

   * @param newTbl

   *          new name of the table. could be the old name

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void alterTable(String tblName, Table newTbl)

      throws InvalidOperationException, HiveException {

    Table t = newTable(tblName);

    try {

      // Remove the DDL_TIME so it gets refreshed

      if (newTbl.getParameters() != null) {

        newTbl.getParameters().remove(hive_metastoreConstants.DDL_TIME);

      }

      getMSC().alter_table(t.getDbName(), t.getTableName(), newTbl.getTTable());

    } catch (MetaException e) {

      throw new HiveException("Unable to alter table.", e);

    } catch (TException e) {

      throw new HiveException("Unable to alter table.", e);

    }

  }

  /**

   * Updates the existing index metadata with the new metadata.

   *

   * @param idxName

   *          name of the existing index

   * @param newIdx

   *          new name of the index. could be the old name

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void alterIndex(String dbName, String baseTblName, String idxName, Index newIdx)

      throws InvalidOperationException, HiveException {

    try {

      getMSC().alter_index(dbName, baseTblName, idxName, newIdx);

    } catch (MetaException e) {

      throw new HiveException("Unable to alter index.", e);

    } catch (TException e) {

      throw new HiveException("Unable to alter index.", e);

    }

  }

  /**

   * Updates the existing partition metadata with the new metadata.

   *

   * @param tblName

   *          name of the existing table

   * @param newPart

   *          new partition

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void alterPartition(String tblName, Partition newPart)

      throws InvalidOperationException, HiveException {

    Table t = newTable(tblName);

    alterPartition(t.getDbName(), t.getTableName(), newPart);

  }

  /**

   * Updates the existing partition metadata with the new metadata.

   *

   * @param dbName

   *          name of the exiting table's database

   * @param tblName

   *          name of the existing table

   * @param newPart

   *          new partition

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void alterPartition(String dbName, String tblName, Partition newPart)

      throws InvalidOperationException, HiveException {

    try {

      // Remove the DDL time so that it gets refreshed

      if (newPart.getParameters() != null) {

        newPart.getParameters().remove(hive_metastoreConstants.DDL_TIME);

      }

      getMSC().alter_partition(dbName, tblName, newPart.getTPartition());

    } catch (MetaException e) {

      throw new HiveException("Unable to alter partition.", e);

    } catch (TException e) {

      throw new HiveException("Unable to alter partition.", e);

    }

  }

  /**

   * Updates the existing table metadata with the new metadata.

   *

   * @param tblName

   *          name of the existing table

   * @param newParts

   *          new partitions

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void alterPartitions(String tblName, List<Partition> newParts)

      throws InvalidOperationException, HiveException {

    Table t = newTable(tblName);

    List<org.apache.hadoop.hive.metastore.api.Partition> newTParts =

      new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>();

    try {

      // Remove the DDL time so that it gets refreshed

      for (Partition tmpPart: newParts) {

        if (tmpPart.getParameters() != null) {

          tmpPart.getParameters().remove(hive_metastoreConstants.DDL_TIME);

        }

        newTParts.add(tmpPart.getTPartition());

      }

      getMSC().alter_partitions(t.getDbName(), t.getTableName(), newTParts);

    } catch (MetaException e) {

      throw new HiveException("Unable to alter partition.", e);

    } catch (TException e) {

      throw new HiveException("Unable to alter partition.", e);

    }

  }

  /**

   * Rename a old partition to new partition

   *

   * @param tbl

   *          existing table

   * @param oldPartSpec

   *          spec of old partition

   * @param newPart

   *          new partition

   * @throws InvalidOperationException

   *           if the changes in metadata is not acceptable

   * @throws TException

   */

  public void renamePartition(Table tbl, Map<String, String> oldPartSpec, Partition newPart)

      throws HiveException {

    try {

      Map<String, String> newPartSpec = newPart.getSpec();

      if (oldPartSpec.keySet().size() != tbl.getPartCols().size()

          || newPartSpec.keySet().size() != tbl.getPartCols().size()) {

        throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. ");

      }

      if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){

        throw new HiveException("Unable to rename partition to the same name: old and new partition cols don't match. ");

      }

      List<String> pvals = new ArrayList<String>();

      for (FieldSchema field : tbl.getPartCols()) {

        String val = oldPartSpec.get(field.getName());

        if (val == null || val.length() == 0) {

          throw new HiveException("get partition: Value for key "

              + field.getName() + " is null or empty");

        } else if (val != null){

          pvals.add(val);

        }

      }

      getMSC().renamePartition(tbl.getDbName(), tbl.getTableName(), pvals,

          newPart.getTPartition());

    } catch (InvalidOperationException e){

      throw new HiveException("Unable to rename partition.", e);

    } catch (MetaException e) {

      throw new HiveException("Unable to rename partition.", e);

    } catch (TException e) {

      throw new HiveException("Unable to rename partition.", e);

    }

  }

  public void alterDatabase(String dbName, Database db)

      throws HiveException {

    try {

      getMSC().alterDatabase(dbName, db);

    } catch (MetaException e) {

      throw new HiveException("Unable to alter database " + dbName, e);

    } catch (NoSuchObjectException e) {

      throw new HiveException("Database " + dbName + " does not exists.", e);

    } catch (TException e) {

      throw new HiveException("Unable to alter database " + dbName, e);

    }

  }

  /**

   * Creates the table with the give objects

   *

   * @param tbl

   *          a table object

   * @throws HiveException

   */

  public void createTable(Table tbl) throws HiveException {

    createTable(tbl, false);

  }

  /**

   * Creates the table with the give objects

   *

   * @param tbl

   *          a table object

   * @param ifNotExists

   *          if true, ignore AlreadyExistsException

   * @throws HiveException

   */

  public void createTable(Table tbl, boolean ifNotExists) throws HiveException {

    try {

      if (tbl.getDbName() == null || "".equals(tbl.getDbName().trim())) {

        tbl.setDbName(SessionState.get().getCurrentDatabase());

      }

      if (tbl.getCols().size() == 0) {

        tbl.setFields(MetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(),

            tbl.getDeserializer()));

      }

      tbl.checkValidity();

      if (tbl.getParameters() != null) {

        tbl.getParameters().remove(hive_metastoreConstants.DDL_TIME);

      }

      org.apache.hadoop.hive.metastore.api.Table tTbl = tbl.getTTable();

      PrincipalPrivilegeSet principalPrivs = new PrincipalPrivilegeSet();

      SessionState ss = SessionState.get();

      if (ss != null) {

        CreateTableAutomaticGrant grants = ss.getCreateTableGrants();

        if (grants != null) {

          principalPrivs.setUserPrivileges(grants.getUserGrants());

          principalPrivs.setGroupPrivileges(grants.getGroupGrants());

          principalPrivs.setRolePrivileges(grants.getRoleGrants());

          tTbl.setPrivileges(principalPrivs);

        }

      }

      getMSC().createTable(tTbl);

    } catch (AlreadyExistsException e) {

      if (!ifNotExists) {

        throw new HiveException(e);

      }

    } catch (Exception e) {

      throw new HiveException(e);

    }

  }

  /**

   *

   * @param tableName

   *          table name

   * @param indexName

   *          index name

   * @param indexHandlerClass

   *          index handler class

   * @param indexedCols

   *          index columns

   * @param indexTblName

   *          index table's name

   * @param deferredRebuild

   *          referred build index table's data

   * @param inputFormat

   *          input format

   * @param outputFormat

   *          output format

   * @param serde

   * @param storageHandler

   *          index table's storage handler

   * @param location

   *          location

   * @param idxProps

   *          idx

   * @param serdeProps

   *          serde properties

   * @param collItemDelim

   * @param fieldDelim

   * @param fieldEscape

   * @param lineDelim

   * @param mapKeyDelim

   * @throws HiveException

   */

  public void createIndex(String tableName, String indexName, String indexHandlerClass,

      List<String> indexedCols, String indexTblName, boolean deferredRebuild,

      String inputFormat, String outputFormat, String serde,

      String storageHandler, String location,

      Map<String, String> idxProps, Map<String, String> tblProps, Map<String, String> serdeProps,

      String collItemDelim, String fieldDelim, String fieldEscape,

      String lineDelim, String mapKeyDelim, String indexComment)

      throws HiveException {

    try {

      String dbName = SessionState.get().getCurrentDatabase();

      Index old_index = null;

      try {

        old_index = getIndex(dbName, tableName, indexName);

      } catch (Exception e) {

      }

      if (old_index != null) {

        throw new HiveException("Index " + indexName + " already exists on table " + tableName + ", db=" + dbName);

      }

      org.apache.hadoop.hive.metastore.api.Table baseTbl = getMSC().getTable(dbName, tableName);

      if (baseTbl.getTableType() == TableType.VIRTUAL_VIEW.toString()) {

        throw new HiveException("tableName="+ tableName +" is a VIRTUAL VIEW. Index on VIRTUAL VIEW is not supported.");

      }

      if (indexTblName == null) {

        indexTblName = MetaStoreUtils.getIndexTableName(dbName, tableName, indexName);

      } else {

        org.apache.hadoop.hive.metastore.api.Table temp = null;

        try {

          temp = getMSC().getTable(dbName, indexTblName);

        } catch (Exception e) {

        }

        if (temp != null) {

          throw new HiveException("Table name " + indexTblName + " already exists. Choose another name.");

        }

      }

      org.apache.hadoop.hive.metastore.api.StorageDescriptor storageDescriptor = baseTbl.getSd().deepCopy();

      SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo();

      if(serde != null) {

        serdeInfo.setSerializationLib(serde);

      } else {

        if (storageHandler == null) {

          serdeInfo.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());

        } else {

          HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler);

          String serDeClassName = sh.getSerDeClass().getName();

          serdeInfo.setSerializationLib(serDeClassName);

        }

      }

      if (fieldDelim != null) {

        serdeInfo.getParameters().put(FIELD_DELIM, fieldDelim);

        serdeInfo.getParameters().put(SERIALIZATION_FORMAT, fieldDelim);

      }

      if (fieldEscape != null) {

        serdeInfo.getParameters().put(ESCAPE_CHAR, fieldEscape);

      }

      if (collItemDelim != null) {

        serdeInfo.getParameters().put(COLLECTION_DELIM, collItemDelim);

      }

      if (mapKeyDelim != null) {

        serdeInfo.getParameters().put(MAPKEY_DELIM, mapKeyDelim);

      }

      if (lineDelim != null) {

        serdeInfo.getParameters().put(LINE_DELIM, lineDelim);

      }

      if (serdeProps != null) {

        Iterator<Entry<String, String>> iter = serdeProps.entrySet()

          .iterator();

        while (iter.hasNext()) {

          Entry<String, String> m = iter.next();

          serdeInfo.getParameters().put(m.getKey(), m.getValue());

        }

      }

      storageDescriptor.setLocation(null);

      if (location != null) {

        storageDescriptor.setLocation(location);

      }

      storageDescriptor.setInputFormat(inputFormat);

      storageDescriptor.setOutputFormat(outputFormat);

      Map<String, String> params = new HashMap<String,String>();

      List<FieldSchema> indexTblCols = new ArrayList<FieldSchema>();

      List<Order> sortCols = new ArrayList<Order>();

      storageDescriptor.setBucketCols(null);

      int k = 0;

      Table metaBaseTbl = new Table(baseTbl);

      for (int i = 0; i < metaBaseTbl.getCols().size(); i++) {

        FieldSchema col = metaBaseTbl.getCols().get(i);

        if (indexedCols.contains(col.getName())) {

          indexTblCols.add(col);

          sortCols.add(new Order(col.getName(), 1));

          k++;

        }

      }

      if (k != indexedCols.size()) {

        throw new RuntimeException(

            "Check the index columns, they should appear in the table being indexed.");

      }

      storageDescriptor.setCols(indexTblCols);

      storageDescriptor.setSortCols(sortCols);

      int time = (int) (System.currentTimeMillis() / 1000);

      org.apache.hadoop.hive.metastore.api.Table tt = null;

      HiveIndexHandler indexHandler = HiveUtils.getIndexHandler(this.getConf(), indexHandlerClass);

      if (indexHandler.usesIndexTable()) {

        tt = new org.apache.hadoop.hive.ql.metadata.Table(dbName, indexTblName).getTTable();

        List<FieldSchema> partKeys = baseTbl.getPartitionKeys();

        tt.setPartitionKeys(partKeys);

        tt.setTableType(TableType.INDEX_TABLE.toString());

        if (tblProps != null) {

          for (Entry<String, String> prop : tblProps.entrySet()) {

            tt.putToParameters(prop.getKey(), prop.getValue());

          }

        }

      }

      if(!deferredRebuild) {

        throw new RuntimeException("Please specify deferred rebuild using \" WITH DEFERRED REBUILD \".");

      }

      Index indexDesc = new Index(indexName, indexHandlerClass, dbName, tableName, time, time, indexTblName,

          storageDescriptor, params, deferredRebuild);

      if (indexComment != null) {

        indexDesc.getParameters().put("comment", indexComment);

      }

      if (idxProps != null)

      {

        indexDesc.getParameters().putAll(idxProps);

      }

      indexHandler.analyzeIndexDefinition(baseTbl, indexDesc, tt);

      this.getMSC().createIndex(indexDesc, tt);

    } catch (Exception e) {

      throw new HiveException(e);

    }

  }

  public Index getIndex(String qualifiedIndexName) throws HiveException {

    String[] names = getQualifiedNames(qualifiedIndexName);

    switch (names.length) {

    case 3:

      return getIndex(names[0], names[1], names[2]);

    case 2:

      return getIndex(SessionState.get().getCurrentDatabase(),

          names[0], names[1]);