Examples of org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy$PartitionIterator

Package org.apache.hadoop.hive.metastore.partition.spec

Examples of org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy$PartitionIterator

org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy
Iterator to iterate over Partitions corresponding to a PartitionSpec.

        RawStore ms, String dbName, String tblName, List<PartitionSpec> partSpecs, boolean ifNotExists)
        throws TException {
      boolean success = false;
      // Ensures that the list doesn't have dups, and keeps track of directories we have created.
      Map<PartValEqWrapperLite, Boolean> addedPartitions = new HashMap<PartValEqWrapperLite, Boolean>();
      PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partSpecs);
      PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator();
      Table tbl = null;
      try {
        ms.openTransaction();
        tbl = ms.getTable(dbName, tblName);
        if (tbl == null) {

View Full Code Here

      clearAndRecreateDB(hmsc);
      createTable(hmsc, enablePartitionGrouping);
      Table table = hmsc.getTable(dbName, tableName);
      populatePartitions(hmsc, table, Arrays.asList("isLocatedInTablePath", "isLocatedOutsideTablePath"));


      PartitionSpecProxy partitionSpecProxy = hmsc.listPartitionSpecs(dbName, tableName, -1);
      Assert.assertEquals( "Unexpected number of partitions.", nDates * 2, partitionSpecProxy.size());


      Map<String, List<String>> locationToDateMap = new HashMap<String, List<String>>();
      locationToDateMap.put("isLocatedInTablePath",  new ArrayList<String>());
      locationToDateMap.put("isLocatedOutsideTablePath", new ArrayList<String>());
      PartitionSpecProxy.PartitionIterator iterator = partitionSpecProxy.getPartitionIterator();


      while (iterator.hasNext()) {
        Partition partition = iterator.next();
        locationToDateMap.get(partition.getValues().get(1)).add(partition.getValues().get(0));
      }


      List<String> expectedDates = new ArrayList<String>(nDates);
      for (int i=0; i<nDates; ++i) {
        expectedDates.add(datePrefix + i);
      }


      Assert.assertArrayEquals("Unexpected date-values.", expectedDates.toArray(), locationToDateMap.get("isLocatedInTablePath").toArray());
      Assert.assertArrayEquals("Unexpected date-values.", expectedDates.toArray(), locationToDateMap.get("isLocatedOutsideTablePath").toArray());


      partitionSpecProxy = hmsc.listPartitionSpecsByFilter(dbName, tableName, "blurb = \"isLocatedOutsideTablePath\"", -1);
      locationToDateMap.get("isLocatedInTablePath").clear();
      locationToDateMap.get("isLocatedOutsideTablePath").clear();
      iterator = partitionSpecProxy.getPartitionIterator();


      while (iterator.hasNext()) {
        Partition partition = iterator.next();
        locationToDateMap.get(partition.getValues().get(1)).add(partition.getValues().get(0));
      }

View Full Code Here

      targetTableSd.setLocation(
          targetTableSd.getLocation().replace( tableName, targetTableName));
      hmsc.createTable(targetTable);


      // Get partition-list from source.
      PartitionSpecProxy partitionsForAddition
          = hmsc.listPartitionSpecsByFilter(dbName, tableName, "blurb = \"isLocatedInTablePath\"", -1);
      partitionsForAddition.setTableName(targetTableName);
      partitionsForAddition.setRootLocation(targetTableSd.getLocation());


      Assert.assertEquals("Unexpected number of partitions added. ",
          partitionsForAddition.size(), hmsc.add_partitions_pspec(partitionsForAddition));


      // Check that the added partitions are as expected.
      PartitionSpecProxy clonedPartitions = hmsc.listPartitionSpecs(dbName, targetTableName, -1);
      Assert.assertEquals("Unexpected number of partitions returned. ",
          partitionsForAddition.size(), clonedPartitions.size());


      PartitionSpecProxy.PartitionIterator sourceIterator = partitionsForAddition.getPartitionIterator(),
                                           targetIterator = clonedPartitions.getPartitionIterator();


      while (targetIterator.hasNext()) {
        Partition sourcePartition = sourceIterator.next(),
                  targetPartition = targetIterator.next();
        Assert.assertEquals("Mismatched values.",

View Full Code Here

      // Add partitions with new schema.
      // Mark Partitions with new schema with different blurb.
      populatePartitions(hmsc, table, Arrays.asList("hasNewColumn"));


      // Retrieve *all* partitions from the table.
      PartitionSpecProxy partitionSpecProxy = hmsc.listPartitionSpecs(dbName, tableName, -1);
      Assert.assertEquals("Unexpected number of partitions.", nDates * 3, partitionSpecProxy.size());


      // Confirm grouping.
      Assert.assertTrue("Unexpected type of PartitionSpecProxy.", partitionSpecProxy instanceof CompositePartitionSpecProxy);
      CompositePartitionSpecProxy compositePartitionSpecProxy = (CompositePartitionSpecProxy)partitionSpecProxy;
      List<PartitionSpec> partitionSpecs = compositePartitionSpecProxy.toPartitionSpec();
      Assert.assertTrue("PartitionSpec[0] should have been a SharedSDPartitionSpec.",
          partitionSpecs.get(0).isSetSharedSDPartitionSpec());
      Assert.assertEquals("PartitionSpec[0] should use the table-path as the common root location. ",
          table.getSd().getLocation(), partitionSpecs.get(0).getRootPath());
      Assert.assertTrue("PartitionSpec[1] should have been a SharedSDPartitionSpec.",
          partitionSpecs.get(1).isSetSharedSDPartitionSpec());
      Assert.assertEquals("PartitionSpec[1] should use the table-path as the common root location. ",
          table.getSd().getLocation(), partitionSpecs.get(1).getRootPath());
      Assert.assertTrue("PartitionSpec[2] should have been a ListComposingPartitionSpec.",
          partitionSpecs.get(2).isSetPartitionList());


      // Categorize the partitions returned, and confirm that all partitions are accounted for.
      PartitionSpecProxy.PartitionIterator iterator = partitionSpecProxy.getPartitionIterator();
      Map<String, List<Partition>> blurbToPartitionList = new HashMap<String, List<Partition>>(3);
      while (iterator.hasNext()) {


        Partition partition = iterator.next();
        String blurb = partition.getValues().get(1);

View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy$PartitionIterator

cascading.tuple.TupleEntrySchemeIterator

org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler

org.apache.hadoop.hive.metastore.TestHiveMetaStorePartitionSpecs

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.