Examples of org.apache.hadoop.hive.metastore.api.Partition

Package org.apache.hadoop.hive.metastore.api

Examples of org.apache.hadoop.hive.metastore.api.Partition

org.apache.hadoop.hive.metastore.api.Partition

      }


      assertEquals(dbPermission, fs.getFileStatus(new Path(tbl.getSd().getLocation()))
          .getPermission());


      Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1");
      Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2");
      Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3");
      Partition part4 = makePartitionObject(dbName, tblName, vals4, tbl, "/part4");


      // check if the partition exists (it shouldn't)
      boolean exceptionThrown = false;
      try {
        Partition p = client.getPartition(dbName, tblName, vals);
      } catch(Exception e) {
        assertEquals("partition should not have existed",
            NoSuchObjectException.class, e.getClass());
        exceptionThrown = true;
      }
      assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown);
      Partition retp = client.add_partition(part);
      assertNotNull("Unable to create partition " + part, retp);
      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))
          .getPermission());
      Partition retp2 = client.add_partition(part2);
      assertNotNull("Unable to create partition " + part2, retp2);
      assertEquals(dbPermission, fs.getFileStatus(new Path(retp2.getSd().getLocation()))
          .getPermission());
      Partition retp3 = client.add_partition(part3);
      assertNotNull("Unable to create partition " + part3, retp3);
      assertEquals(dbPermission, fs.getFileStatus(new Path(retp3.getSd().getLocation()))
          .getPermission());
      Partition retp4 = client.add_partition(part4);
      assertNotNull("Unable to create partition " + part4, retp4);
      assertEquals(dbPermission, fs.getFileStatus(new Path(retp4.getSd().getLocation()))
          .getPermission());


      Partition part_get = client.getPartition(dbName, tblName, part.getValues());
      if(isThriftClient) {
        // since we are using thrift, 'part' will not have the create time and
        // last DDL time set since it does not get updated in the add_partition()
        // call - likewise part2 and part3 - set it correctly so that equals check
        // doesn't fail
        adjust(client, part, dbName, tblName);
        adjust(client, part2, dbName, tblName);
        adjust(client, part3, dbName, tblName);
      }
      assertTrue("Partitions are not same", part.equals(part_get));


      String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";
      String part2Name = "ds=2008-07-01 14%3A13%3A12/hr=15";
      String part3Name ="ds=2008-07-02 14%3A13%3A12/hr=15";
      String part4Name ="ds=2008-07-03 14%3A13%3A12/hr=151";


      part_get = client.getPartition(dbName, tblName, partName);
      assertTrue("Partitions are not the same", part.equals(part_get));


      // Test partition listing with a partial spec - ds is specified but hr is not
      List<String> partialVals = new ArrayList<String>();
      partialVals.add(vals.get(0));
      Set<Partition> parts = new HashSet<Partition>();
      parts.add(part);
      parts.add(part2);


      List<Partition> partial = client.listPartitions(dbName, tblName, partialVals,
          (short) -1);
      assertTrue("Should have returned 2 partitions", partial.size() == 2);
      assertTrue("Not all parts returned", partial.containsAll(parts));


      Set<String> partNames = new HashSet<String>();
      partNames.add(partName);
      partNames.add(part2Name);
      List<String> partialNames = client.listPartitionNames(dbName, tblName, partialVals,
          (short) -1);
      assertTrue("Should have returned 2 partition names", partialNames.size() == 2);
      assertTrue("Not all part names returned", partialNames.containsAll(partNames));


      partNames.add(part3Name);
      partNames.add(part4Name);
      partialVals.clear();
      partialVals.add("");
      partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);
      assertTrue("Should have returned 4 partition names", partialNames.size() == 4);
      assertTrue("Not all part names returned", partialNames.containsAll(partNames));


      // Test partition listing with a partial spec - hr is specified but ds is not
      parts.clear();
      parts.add(part2);
      parts.add(part3);


      partialVals.clear();
      partialVals.add("");
      partialVals.add(vals2.get(1));


      partial = client.listPartitions(dbName, tblName, partialVals, (short) -1);
      assertEquals("Should have returned 2 partitions", 2, partial.size());
      assertTrue("Not all parts returned", partial.containsAll(parts));


      partNames.clear();
      partNames.add(part2Name);
      partNames.add(part3Name);
      partialNames = client.listPartitionNames(dbName, tblName, partialVals,
          (short) -1);
      assertEquals("Should have returned 2 partition names", 2, partialNames.size());
      assertTrue("Not all part names returned", partialNames.containsAll(partNames));


      // Verify escaped partition names don't return partitions
      exceptionThrown = false;
      try {
        String badPartName = "ds=2008-07-01 14%3A13%3A12/hrs=14";
        client.getPartition(dbName, tblName, badPartName);
      } catch(NoSuchObjectException e) {
        exceptionThrown = true;
      }
      assertTrue("Bad partition spec should have thrown an exception", exceptionThrown);


      Path partPath = new Path(part.getSd().getLocation());




      assertTrue(fs.exists(partPath));
      client.dropPartition(dbName, tblName, part.getValues(), true);
      assertFalse(fs.exists(partPath));


      // Test append_partition_by_name
      client.appendPartition(dbName, tblName, partName);
      Partition part5 = client.getPartition(dbName, tblName, part.getValues());
      assertTrue("Append partition by name failed", part5.getValues().equals(vals));;
      Path part5Path = new Path(part5.getSd().getLocation());
      assertTrue(fs.exists(part5Path));


      // Test drop_partition_by_name
      assertTrue("Drop partition by name failed",
          client.dropPartition(dbName, tblName, partName, true));
      assertFalse(fs.exists(part5Path));


      // add the partition again so that drop table with a partition can be
      // tested
      retp = client.add_partition(part);
      assertNotNull("Unable to create partition " + part, retp);
      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))
          .getPermission());


      // test add_partitions


      List<String> mvals1 = makeVals("2008-07-04 14:13:12", "14641");
      List<String> mvals2 = makeVals("2008-07-04 14:13:12", "14642");
      List<String> mvals3 = makeVals("2008-07-04 14:13:12", "14643");
      List<String> mvals4 = makeVals("2008-07-04 14:13:12", "14643"); // equal to 3
      List<String> mvals5 = makeVals("2008-07-04 14:13:12", "14645");


      Exception savedException;


      // add_partitions(empty list) : ok, normal operation
      client.add_partitions(new ArrayList<Partition>());


      // add_partitions(1,2,3) : ok, normal operation
      Partition mpart1 = makePartitionObject(dbName, tblName, mvals1, tbl, "/mpart1");
      Partition mpart2 = makePartitionObject(dbName, tblName, mvals2, tbl, "/mpart2");
      Partition mpart3 = makePartitionObject(dbName, tblName, mvals3, tbl, "/mpart3");
      client.add_partitions(Arrays.asList(mpart1,mpart2,mpart3));


      if(isThriftClient) {
        // do DDL time munging if thrift mode
        adjust(client, mpart1, dbName, tblName);
        adjust(client, mpart2, dbName, tblName);
        adjust(client, mpart3, dbName, tblName);
      }
      verifyPartitionsPublished(client, dbName, tblName,
          Arrays.asList(mvals1.get(0)),
          Arrays.asList(mpart1,mpart2,mpart3));


      Partition mpart4 = makePartitionObject(dbName, tblName, mvals4, tbl, "/mpart4");
      Partition mpart5 = makePartitionObject(dbName, tblName, mvals5, tbl, "/mpart5");


      // create dir for /mpart5
      Path mp5Path = new Path(mpart5.getSd().getLocation());
      warehouse.mkdirs(mp5Path);
      assertTrue(fs.exists(mp5Path));
      assertEquals(dbPermission, fs.getFileStatus(mp5Path).getPermission());


      // add_partitions(5,4) : err = duplicate keyvals on mpart4

View Full Code Here

    return vals4;
  }


  private static Partition makePartitionObject(String dbName, String tblName,
      List<String> ptnVals, Table tbl, String ptnLocationSuffix) {
    Partition part4 = new Partition();
    part4.setDbName(dbName);
    part4.setTableName(tblName);
    part4.setValues(ptnVals);
    part4.setParameters(new HashMap<String, String>());
    part4.setSd(tbl.getSd().deepCopy());
    part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
    part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix);
    return part4;
  }

View Full Code Here

    }


    List<String> vals = new ArrayList<String>(1);
    vals.add("abc");


    Partition part = new Partition();
    part.setDbName(dbName);
    part.setTableName(viewName);
    part.setValues(vals);
    part.setParameters(new HashMap<String, String>());


    client.add_partition(part);


    Partition part2 = client.getPartition(dbName, viewName, part.getValues());


    part2.getParameters().put("a", "b");


    client.alter_partition(dbName, viewName, part2);


    Partition part3 = client.getPartition(dbName, viewName, part.getValues());
    assertEquals("couldn't view alter partition", part3.getParameters().get(
        "a"), "b");


    client.dropTable(dbName, viewName);


    client.dropTable(dbName, tblName);

View Full Code Here

        // on the location being present in the 'tbl' object - so get the table
        // from the metastore
        tbl = client.getTable(dbName, tblName);
      }


      Partition part = new Partition();
      part.setDbName(dbName);
      part.setTableName(tblName);
      part.setValues(vals);
      part.setParameters(new HashMap<String, String>());
      part.setSd(tbl.getSd());
      part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
      part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");


      client.add_partition(part);


      Partition part2 = client.getPartition(dbName, tblName, part.getValues());


      part2.getParameters().put("retention", "10");
      part2.getSd().setNumBuckets(12);
      part2.getSd().getSerdeInfo().getParameters().put("abc", "1");
      client.alter_partition(dbName, tblName, part2);


      Partition part3 = client.getPartition(dbName, tblName, part.getValues());
      assertEquals("couldn't alter partition", part3.getParameters().get(
          "retention"), "10");
      assertEquals("couldn't alter partition", part3.getSd().getSerdeInfo()
          .getParameters().get("abc"), "1");
      assertEquals("couldn't alter partition", part3.getSd().getNumBuckets(),
          12);


      client.dropTable(dbName, tblName);


      client.dropDatabase(dbName);

View Full Code Here

        // on the location being present in the 'tbl' object - so get the table
        // from the metastore
        tbl = client.getTable(dbName, tblName);
      }


      Partition part = new Partition();
      part.setDbName(dbName);
      part.setTableName(tblName);
      part.setValues(vals);
      part.setParameters(new HashMap<String, String>());
      part.setSd(tbl.getSd().deepCopy());
      part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
      part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");
      part.getParameters().put("retention", "10");
      part.getSd().setNumBuckets(12);
      part.getSd().getSerdeInfo().getParameters().put("abc", "1");


      client.add_partition(part);


      part.setValues(tmp_vals);
      client.renamePartition(dbName, tblName, vals, part);


      boolean exceptionThrown = false;
      try {
        Partition p = client.getPartition(dbName, tblName, vals);
      } catch(Exception e) {
        assertEquals("partition should not have existed",
            NoSuchObjectException.class, e.getClass());
        exceptionThrown = true;
      }
      assertTrue("Expected NoSuchObjectException", exceptionThrown);


      Partition part3 = client.getPartition(dbName, tblName, tmp_vals);
      assertEquals("couldn't rename partition", part3.getParameters().get(
          "retention"), "10");
      assertEquals("couldn't rename partition", part3.getSd().getSerdeInfo()
          .getParameters().get("abc"), "1");
      assertEquals("couldn't rename partition", part3.getSd().getNumBuckets(),
          12);
      assertEquals("new partition sd matches", part3.getSd().getLocation(),
          tbl.getSd().getLocation() + part2_path);


      part.setValues(vals);
      client.renamePartition(dbName, tblName, tmp_vals, part);


      exceptionThrown = false;
      try {
        Partition p = client.getPartition(dbName, tblName, tmp_vals);
      } catch(Exception e) {
        assertEquals("partition should not have existed",
            NoSuchObjectException.class, e.getClass());
        exceptionThrown = true;
      }

View Full Code Here

    String partLocnRoot, Map<String, String> partKVs,
    HCatSchema outputSchema, Map<String, String> params,
    Table table, FileSystem fs,
    String grpName, FsPermission perms) throws IOException {


    Partition partition = new Partition();
    partition.setDbName(table.getDbName());
    partition.setTableName(table.getTableName());
    partition.setSd(new StorageDescriptor(table.getTTable().getSd()));


    List<FieldSchema> fields = new ArrayList<FieldSchema>();
    for (HCatFieldSchema fieldSchema : outputSchema.getFields()) {
      fields.add(HCatSchemaUtils.getFieldSchema(fieldSchema));
    }


    partition.getSd().setCols(fields);


    partition.setValues(FileOutputFormatContainer.getPartitionValueList(table, partKVs));


    partition.setParameters(params);


    // Sets permissions and group name on partition dirs and files.


    Path partPath;
    if (Boolean.valueOf((String)table.getProperty("EXTERNAL"))
         && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) {
      // honor external table that specifies the location
      partPath = new Path(jobInfo.getLocation());
    } else {
      partPath = new Path(partLocnRoot);
      int i = 0;
      for (FieldSchema partKey : table.getPartitionKeys()) {
        if (i++ != 0) {
          applyGroupAndPerms(fs, partPath, perms, grpName, false);
        }
        partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
      }
    }


    // Apply the group and permissions to the leaf partition and files.
    // Need not bother in case of HDFS as permission is taken care of by setting UMask
    if (!ShimLoader.getHadoopShims().getHCatShim().isFileInHDFS(fs, partPath)) {
      applyGroupAndPerms(fs, partPath, perms, grpName, true);
    }


    // Set the location in the StorageDescriptor
    if (dynamicPartitioningUsed) {
      String dynamicPartitionDestination = getFinalDynamicPartitionDestination(table, partKVs);
      if (harProcessor.isEnabled()) {
        harProcessor.exec(context, partition, partPath);
        partition.getSd().setLocation(
          harProcessor.getProcessedLocation(new Path(dynamicPartitionDestination)));
      } else {
        partition.getSd().setLocation(dynamicPartitionDestination);
      }
    } else {
      partition.getSd().setLocation(partPath.toString());
    }
    return partition;
  }

View Full Code Here

  }


  public Partition getPartition(String dbName, String tableName,
      List<String> part_vals) throws NoSuchObjectException, MetaException {
    openTransaction();
    Partition part = convertToPart(getMPartition(dbName, tableName, part_vals));
    commitTransaction();
    if(part == null) {
      throw new NoSuchObjectException("partition values="
          + part_vals.toString());
    }
    part.setValues(part_vals);
    return part;
  }

View Full Code Here


  private Partition convertToPart(MPartition mpart) throws MetaException {
    if (mpart == null) {
      return null;
    }
    return new Partition(mpart.getValues(), mpart.getTable().getDatabase()
        .getName(), mpart.getTable().getTableName(), mpart.getCreateTime(),
        mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()),
        mpart.getParameters());
  }

View Full Code Here

  private Partition convertToPart(String dbName, String tblName, MPartition mpart)
      throws MetaException {
    if (mpart == null) {
      return null;
    }
    return new Partition(mpart.getValues(), dbName, tblName, mpart.getCreateTime(),
        mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false),
        mpart.getParameters());
  }

View Full Code Here

      List<MPartition> mparts = listMPartitions(dbName, tblName, max);
      List<Partition> parts = new ArrayList<Partition>(mparts.size());
      if (mparts != null && mparts.size()>0) {
        for (MPartition mpart : mparts) {
          MTable mtbl = mpart.getTable();
          Partition part = convertToPart(mpart);
          parts.add(part);


          if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
            String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl
                .getPartitionKeys()), part.getValues());
            PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(dbName,
                tblName, partName, userName, groupNames);
            part.setPrivileges(partAuth);
          }
        }
      }
      success =  commitTransaction();
      return parts;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.metastore.api.Partition

com.facebook.giraph.hive.impl.output.HiveApiOutputCommitter

com.facebook.giraph.hive.output.HiveApiOutputCommitter

com.facebook.hiveio.common.HiveUtils

com.facebook.hiveio.output.HiveApiOutputCommitter

com.facebook.presto.hive.HiveClient

com.facebook.presto.hive.metastore.MockHiveMetastoreClient

com.facebook.presto.hive.MockHiveMetastoreClient

com.facebook.presto.hive.shaded.org.apache.commons.lang.builder.HashCodeBuilder

com.facebook.presto.hive.shaded.org.apache.thrift.protocol.TTupleProtocol

com.taobao.zeus.jobs.sub.tool.HiveOutputCheckJob

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.