* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  protected static HiveMetaStoreClient client;

  protected static HiveConf hiveConf;

  protected static Warehouse warehouse;

  protected static boolean isThriftClient = false;

  private static final String TEST_DB1_NAME = "testdb1";

  private static final String TEST_DB2_NAME = "testdb2";

  @Override

  protected void setUp() throws Exception {

    hiveConf = new HiveConf(this.getClass());

    warehouse = new Warehouse(hiveConf);

    // set some values to use for getting conf. vars

    hiveConf.set("hive.metastore.metrics.enabled","true");

    hiveConf.set("hive.key1", "value1");

    hiveConf.set("hive.key2", "http://www.example.com");

    hiveConf.set("hive.key3", "");

    hiveConf.set("hive.key4", "0");

  }

  public void testNameMethods() {

    Map<String, String> spec = new LinkedHashMap<String, String>();

    spec.put("ds", "2008-07-01 14:13:12");

    spec.put("hr", "14");

    List<String> vals = new ArrayList<String>();

    for(String v : spec.values()) {

      vals.add(v);

    }

    String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";

    try {

      List<String> testVals = client.partitionNameToVals(partName);

      assertTrue("Values from name are incorrect", vals.equals(testVals));

      Map<String, String> testSpec = client.partitionNameToSpec(partName);

      assertTrue("Spec from name is incorrect", spec.equals(testSpec));

      List<String> emptyVals = client.partitionNameToVals("");

      assertTrue("Values should be empty", emptyVals.size() == 0);

      Map<String, String> emptySpec =  client.partitionNameToSpec("");

      assertTrue("Spec should be empty", emptySpec.size() == 0);

    } catch (Exception e) {

      assert(false);

    }

  }

  /**

   * tests create table and partition and tries to drop the table without

   * droppping the partition

   *

   * @throws Exception

   */

  public void testPartition() throws Exception {

    partitionTester(client, hiveConf);

  }

  public static void partitionTester(HiveMetaStoreClient client, HiveConf hiveConf)

    throws Exception {

    try {

      String dbName = "compdb";

      String tblName = "comptbl";

      String typeName = "Person";

      List<String> vals = makeVals("2008-07-01 14:13:12", "14");

      List<String> vals2 = makeVals("2008-07-01 14:13:12", "15");

      List<String> vals3 = makeVals("2008-07-02 14:13:12", "15");

      List<String> vals4 = makeVals("2008-07-03 14:13:12", "151");

      client.dropTable(dbName, tblName);

      silentDropDatabase(dbName);

      Database db = new Database();

      db.setName(dbName);

      client.createDatabase(db);

      db = client.getDatabase(dbName);

      Path dbPath = new Path(db.getLocationUri());

      FileSystem fs = FileSystem.get(dbPath.toUri(), hiveConf);

      boolean inheritPerms = hiveConf.getBoolVar(

          HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);

      FsPermission dbPermission = fs.getFileStatus(dbPath).getPermission();

      if (inheritPerms) {

         //Set different perms for the database dir for further tests

         dbPermission = new FsPermission((short)488);

         fs.setPermission(dbPath, dbPermission);

      }

      client.dropType(typeName);

      Type typ1 = new Type();

      typ1.setName(typeName);

      typ1.setFields(new ArrayList<FieldSchema>(2));

      typ1.getFields().add(

          new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));

      typ1.getFields().add(

          new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));

      client.createType(typ1);

      Table tbl = new Table();

      tbl.setDbName(dbName);

      tbl.setTableName(tblName);

      StorageDescriptor sd = new StorageDescriptor();

      tbl.setSd(sd);

      sd.setCols(typ1.getFields());

      sd.setCompressed(false);

      sd.setNumBuckets(1);

      sd.setParameters(new HashMap<String, String>());

      sd.getParameters().put("test_param_1", "Use this for comments etc");

      sd.setBucketCols(new ArrayList<String>(2));

      sd.getBucketCols().add("name");

      sd.setSerdeInfo(new SerDeInfo());

      sd.getSerdeInfo().setName(tbl.getTableName());

      sd.getSerdeInfo().setParameters(new HashMap<String, String>());

      sd.getSerdeInfo().getParameters()

          .put(serdeConstants.SERIALIZATION_FORMAT, "1");

      sd.setSortCols(new ArrayList<Order>());

      sd.setStoredAsSubDirectories(false);

      //skewed information

      SkewedInfo skewInfor = new SkewedInfo();

      skewInfor.setSkewedColNames(Arrays.asList("name"));

      List<String> skv = Arrays.asList("1");

      skewInfor.setSkewedColValues(Arrays.asList(skv));

      Map<List<String>, String> scvlm = new HashMap<List<String>, String>();

      scvlm.put(skv, "location1");

      skewInfor.setSkewedColValueLocationMaps(scvlm);

      sd.setSkewedInfo(skewInfor);

      tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));

      tbl.getPartitionKeys().add(

          new FieldSchema("ds", serdeConstants.STRING_TYPE_NAME, ""));

      tbl.getPartitionKeys().add(

          new FieldSchema("hr", serdeConstants.STRING_TYPE_NAME, ""));

      client.createTable(tbl);

      if (isThriftClient) {

        // the createTable() above does not update the location in the 'tbl'

        // object when the client is a thrift client and the code below relies

        // on the location being present in the 'tbl' object - so get the table

        // from the metastore

        tbl = client.getTable(dbName, tblName);

      }

      assertEquals(dbPermission, fs.getFileStatus(new Path(tbl.getSd().getLocation()))

          .getPermission());

      Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1");

      Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2");

      Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3");

      Partition part4 = makePartitionObject(dbName, tblName, vals4, tbl, "/part4");

      // check if the partition exists (it shouldn't)

      boolean exceptionThrown = false;

      try {

        Partition p = client.getPartition(dbName, tblName, vals);

      } catch(Exception e) {

        assertEquals("partition should not have existed",

            NoSuchObjectException.class, e.getClass());

        exceptionThrown = true;

      }

      assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown);

      Partition retp = client.add_partition(part);

      assertNotNull("Unable to create partition " + part, retp);

      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))

          .getPermission());

      Partition retp2 = client.add_partition(part2);

      assertNotNull("Unable to create partition " + part2, retp2);

      assertEquals(dbPermission, fs.getFileStatus(new Path(retp2.getSd().getLocation()))

          .getPermission());

      Partition retp3 = client.add_partition(part3);

      assertNotNull("Unable to create partition " + part3, retp3);

      assertEquals(dbPermission, fs.getFileStatus(new Path(retp3.getSd().getLocation()))

          .getPermission());

      Partition retp4 = client.add_partition(part4);

      assertNotNull("Unable to create partition " + part4, retp4);

      assertEquals(dbPermission, fs.getFileStatus(new Path(retp4.getSd().getLocation()))

          .getPermission());

      Partition part_get = client.getPartition(dbName, tblName, part.getValues());

      if(isThriftClient) {

        // since we are using thrift, 'part' will not have the create time and

        // last DDL time set since it does not get updated in the add_partition()

        // call - likewise part2 and part3 - set it correctly so that equals check

        // doesn't fail

        adjust(client, part, dbName, tblName);

        adjust(client, part2, dbName, tblName);

        adjust(client, part3, dbName, tblName);

      }

      assertTrue("Partitions are not same", part.equals(part_get));

      String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";

      String part2Name = "ds=2008-07-01 14%3A13%3A12/hr=15";

      String part3Name ="ds=2008-07-02 14%3A13%3A12/hr=15";

      String part4Name ="ds=2008-07-03 14%3A13%3A12/hr=151";

      part_get = client.getPartition(dbName, tblName, partName);

      assertTrue("Partitions are not the same", part.equals(part_get));

      // Test partition listing with a partial spec - ds is specified but hr is not

      List<String> partialVals = new ArrayList<String>();

      partialVals.add(vals.get(0));

      Set<Partition> parts = new HashSet<Partition>();

      parts.add(part);

      parts.add(part2);

      List<Partition> partial = client.listPartitions(dbName, tblName, partialVals,

          (short) -1);

      assertTrue("Should have returned 2 partitions", partial.size() == 2);

      assertTrue("Not all parts returned", partial.containsAll(parts));

      Set<String> partNames = new HashSet<String>();

      partNames.add(partName);

      partNames.add(part2Name);

      List<String> partialNames = client.listPartitionNames(dbName, tblName, partialVals,

          (short) -1);

      assertTrue("Should have returned 2 partition names", partialNames.size() == 2);

      assertTrue("Not all part names returned", partialNames.containsAll(partNames));

      partNames.add(part3Name);

      partNames.add(part4Name);

      partialVals.clear();

      partialVals.add("");

      partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);

      assertTrue("Should have returned 4 partition names", partialNames.size() == 4);

      assertTrue("Not all part names returned", partialNames.containsAll(partNames));

      // Test partition listing with a partial spec - hr is specified but ds is not

      parts.clear();

      parts.add(part2);

      parts.add(part3);

      partialVals.clear();

      partialVals.add("");

      partialVals.add(vals2.get(1));

      partial = client.listPartitions(dbName, tblName, partialVals, (short) -1);

      assertEquals("Should have returned 2 partitions", 2, partial.size());

      assertTrue("Not all parts returned", partial.containsAll(parts));

      partNames.clear();

      partNames.add(part2Name);

      partNames.add(part3Name);

      partialNames = client.listPartitionNames(dbName, tblName, partialVals,

          (short) -1);

      assertEquals("Should have returned 2 partition names", 2, partialNames.size());

      assertTrue("Not all part names returned", partialNames.containsAll(partNames));

      // Verify escaped partition names don't return partitions

      exceptionThrown = false;

      try {

        String badPartName = "ds=2008-07-01 14%3A13%3A12/hrs=14";

        client.getPartition(dbName, tblName, badPartName);

      } catch(NoSuchObjectException e) {

        exceptionThrown = true;

      }

      assertTrue("Bad partition spec should have thrown an exception", exceptionThrown);

      Path partPath = new Path(part.getSd().getLocation());

      assertTrue(fs.exists(partPath));

      client.dropPartition(dbName, tblName, part.getValues(), true);

      assertFalse(fs.exists(partPath));

      // Test append_partition_by_name

      client.appendPartition(dbName, tblName, partName);

      Partition part5 = client.getPartition(dbName, tblName, part.getValues());

      assertTrue("Append partition by name failed", part5.getValues().equals(vals));;

      Path part5Path = new Path(part5.getSd().getLocation());

      assertTrue(fs.exists(part5Path));

      // Test drop_partition_by_name

      assertTrue("Drop partition by name failed",

          client.dropPartition(dbName, tblName, partName, true));

      assertFalse(fs.exists(part5Path));

      // add the partition again so that drop table with a partition can be

      // tested

      retp = client.add_partition(part);

      assertNotNull("Unable to create partition " + part, retp);

      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))

          .getPermission());

      // test add_partitions

      List<String> mvals1 = makeVals("2008-07-04 14:13:12", "14641");

      List<String> mvals2 = makeVals("2008-07-04 14:13:12", "14642");

      List<String> mvals3 = makeVals("2008-07-04 14:13:12", "14643");

      List<String> mvals4 = makeVals("2008-07-04 14:13:12", "14643"); // equal to 3

      List<String> mvals5 = makeVals("2008-07-04 14:13:12", "14645");

      Exception savedException;

      // add_partitions(empty list) : ok, normal operation

      client.add_partitions(new ArrayList<Partition>());

      // add_partitions(1,2,3) : ok, normal operation

      Partition mpart1 = makePartitionObject(dbName, tblName, mvals1, tbl, "/mpart1");

      Partition mpart2 = makePartitionObject(dbName, tblName, mvals2, tbl, "/mpart2");

      Partition mpart3 = makePartitionObject(dbName, tblName, mvals3, tbl, "/mpart3");

      client.add_partitions(Arrays.asList(mpart1,mpart2,mpart3));

      if(isThriftClient) {

        // do DDL time munging if thrift mode

        adjust(client, mpart1, dbName, tblName);

        adjust(client, mpart2, dbName, tblName);

        adjust(client, mpart3, dbName, tblName);

      }

      verifyPartitionsPublished(client, dbName, tblName,

          Arrays.asList(mvals1.get(0)),

          Arrays.asList(mpart1,mpart2,mpart3));

      Partition mpart4 = makePartitionObject(dbName, tblName, mvals4, tbl, "/mpart4");

      Partition mpart5 = makePartitionObject(dbName, tblName, mvals5, tbl, "/mpart5");

      // create dir for /mpart5

      Path mp5Path = new Path(mpart5.getSd().getLocation());

      warehouse.mkdirs(mp5Path);

      assertTrue(fs.exists(mp5Path));

      assertEquals(dbPermission, fs.getFileStatus(mp5Path).getPermission());

      // add_partitions(5,4) : err = duplicate keyvals on mpart4

      savedException = null;

      try {

        client.add_partitions(Arrays.asList(mpart5,mpart4));

      } catch (Exception e) {

        savedException = e;

      } finally {

        assertNotNull(savedException);

      }

      // check that /mpart4 does not exist, but /mpart5 still does.

      assertTrue(fs.exists(mp5Path));

      assertFalse(fs.exists(new Path(mpart4.getSd().getLocation())));

      // add_partitions(5) : ok

      client.add_partitions(Arrays.asList(mpart5));

      if(isThriftClient) {

        // do DDL time munging if thrift mode

        adjust(client, mpart5, dbName, tblName);

      }

      verifyPartitionsPublished(client, dbName, tblName,

          Arrays.asList(mvals1.get(0)),

          Arrays.asList(mpart1,mpart2,mpart3,mpart5));

      //// end add_partitions tests

      client.dropTable(dbName, tblName);

      client.dropType(typeName);

      // recreate table as external, drop partition and it should

      // still exist

      tbl.setParameters(new HashMap<String, String>());

      tbl.getParameters().put("EXTERNAL", "TRUE");

      client.createTable(tbl);

      retp = client.add_partition(part);

      assertTrue(fs.exists(partPath));

      client.dropPartition(dbName, tblName, part.getValues(), true);

      assertTrue(fs.exists(partPath));

      for (String tableName : client.getTables(dbName, "*")) {

        client.dropTable(dbName, tableName);

      }

      client.dropDatabase(dbName);

    } catch (Exception e) {

      System.err.println(StringUtils.stringifyException(e));

      System.err.println("testPartition() failed.");

      throw e;

    }

  }

  private static void verifyPartitionsPublished(HiveMetaStoreClient client,

      String dbName, String tblName, List<String> partialSpec,

      List<Partition> expectedPartitions)

          throws NoSuchObjectException, MetaException, TException {

    // Test partition listing with a partial spec

    List<Partition> mpartial = client.listPartitions(dbName, tblName, partialSpec,

        (short) -1);

    assertEquals("Should have returned "+expectedPartitions.size()+

        " partitions, returned " + mpartial.size(),

        expectedPartitions.size(), mpartial.size());

    assertTrue("Not all parts returned", mpartial.containsAll(expectedPartitions));

  }

  private static List<String> makeVals(String ds, String id) {

    List <String> vals4 = new ArrayList<String>(2);

    vals4 = new ArrayList<String>(2);

    vals4.add(ds);

    vals4.add(id);

    return vals4;

  }

  private static Partition makePartitionObject(String dbName, String tblName,

      List<String> ptnVals, Table tbl, String ptnLocationSuffix) {

    Partition part4 = new Partition();

    part4.setDbName(dbName);

    part4.setTableName(tblName);

    part4.setValues(ptnVals);

    part4.setParameters(new HashMap<String, String>());

    part4.setSd(tbl.getSd().deepCopy());

    part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());

    part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix);

    return part4;

  }

  public void testListPartitions() throws Throwable {

    // create a table with multiple partitions

    String dbName = "compdb";

    String tblName = "comptbl";

    String typeName = "Person";

    cleanUp(dbName, tblName, typeName);

    List<List<String>> values = new ArrayList<List<String>>();

    values.add(makeVals("2008-07-01 14:13:12", "14"));

    values.add(makeVals("2008-07-01 14:13:12", "15"));

    values.add(makeVals("2008-07-02 14:13:12", "15"));

    values.add(makeVals("2008-07-03 14:13:12", "151"));

    createMultiPartitionTableSchema(dbName, tblName, typeName, values);

    List<Partition> partitions = client.listPartitions(dbName, tblName, (short)-1);

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() +

      " partitions", values.size(), partitions.size());

    partitions = client.listPartitions(dbName, tblName, (short)(values.size()/2));

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() / 2 +

      " partitions",values.size() / 2, partitions.size());

    partitions = client.listPartitions(dbName, tblName, (short) (values.size() * 2));

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() +

      " partitions",values.size(), partitions.size());

    cleanUp(dbName, tblName, typeName);

  }

  public void testListPartitionNames() throws Throwable {

    // create a table with multiple partitions

    String dbName = "compdb";

    String tblName = "comptbl";

    String typeName = "Person";

    cleanUp(dbName, tblName, typeName);

    List<List<String>> values = new ArrayList<List<String>>();

    values.add(makeVals("2008-07-01 14:13:12", "14"));

    values.add(makeVals("2008-07-01 14:13:12", "15"));

    values.add(makeVals("2008-07-02 14:13:12", "15"));

    values.add(makeVals("2008-07-03 14:13:12", "151"));

    createMultiPartitionTableSchema(dbName, tblName, typeName, values);

    List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() +

      " partitions", values.size(), partitions.size());

    partitions = client.listPartitionNames(dbName, tblName, (short)(values.size()/2));

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() / 2 +

      " partitions",values.size() / 2, partitions.size());

    partitions = client.listPartitionNames(dbName, tblName, (short) (values.size() * 2));

    assertNotNull("should have returned partitions", partitions);

    assertEquals(" should have returned " + values.size() +

      " partitions",values.size(), partitions.size());

    cleanUp(dbName, tblName, typeName);

  }

  public void testDropTable() throws Throwable {

    // create a table with multiple partitions

    String dbName = "compdb";

    String tblName = "comptbl";

    String typeName = "Person";

    cleanUp(dbName, tblName, typeName);

    List<List<String>> values = new ArrayList<List<String>>();

    values.add(makeVals("2008-07-01 14:13:12", "14"));

    values.add(makeVals("2008-07-01 14:13:12", "15"));

    values.add(makeVals("2008-07-02 14:13:12", "15"));

    values.add(makeVals("2008-07-03 14:13:12", "151"));

    createMultiPartitionTableSchema(dbName, tblName, typeName, values);

    client.dropTable(dbName, tblName);

    client.dropType(typeName);

    boolean exceptionThrown = false;

    try {

      client.getTable(dbName, tblName);

    } catch(Exception e) {