Examples of org.apache.hadoop.hive.metastore.Warehouse

Package org.apache.hadoop.hive.metastore

Examples of org.apache.hadoop.hive.metastore.Warehouse

org.apache.hadoop.hive.metastore.Warehouse
This class represents a warehouse where data of Hive tables is stored

        String username = ShimLoader.getHadoopShims().getShortUserName(ugi);


        whPath = new Path(whDir);
        whFs = whPath.getFileSystem(conf);


        wh = new Warehouse(conf);
        hive = Hive.get(conf);


        //clean up mess in HMS
        HcatTestUtils.cleanupHMS(hive, wh, perm700);

View Full Code Here


        hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName());
        hcatConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
        hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
        hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
        clientWH = new Warehouse(hcatConf);
        msc = new HiveMetaStoreClient(hcatConf, null);
        System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
        System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
    }

View Full Code Here


    @Override
    public void setConf(Configuration conf) {
        super.setConf(conf);
        try {
            this.wh = new Warehouse(conf);
        } catch (MetaException ex) {
            throw new RuntimeException(ex);
        }
    }

View Full Code Here

                // We dont set perms or groups for default dir.
                return 0;
            } else {
                try {
                    Hive db = Hive.get();
                    Path dbPath = new Warehouse(conf).getDatabasePath(db.getDatabase(dbName));
                    FileSystem fs = dbPath.getFileSystem(conf);
                    if (perms != null) {
                        fs.setPermission(dbPath, perms);
                    }
                    if (null != grp) {

View Full Code Here

    }
  }


  private void deleteDir(Path dir) throws HiveException {
    try {
      Warehouse wh = new Warehouse(conf);
      wh.deleteDir(dir, true);
    } catch (MetaException e) {
      throw new HiveException(e);
    }
  }

View Full Code Here

              // allocate a temporary output dir on the location of the table
              String tableName = getUnescapedName((ASTNode)ast.getChild(0));
              Table newTable = db.newTable(tableName);
              Path location;
              try {
                Warehouse wh = new Warehouse(conf);
                location = wh.getDatabasePath(db.getDatabase(newTable.getDbName()));
              } catch (MetaException e) {
                throw new SemanticException(e);
              }
              try {
                fname = ctx.getExternalTmpFileURI(

View Full Code Here

              dumpTable = db.newTable(qb.getTableDesc().getTableName());
              if (!db.databaseExists(dumpTable.getDbName())) {
                throw new SemanticException("ERROR: The database " + dumpTable.getDbName()
                    + " does not exist.");
              }
              Warehouse wh = new Warehouse(conf);
              targetPath = wh.getTablePath(db.getDatabase(dumpTable.getDbName()), dumpTable
                  .getTableName());
            } catch (HiveException e) {
              throw new SemanticException(e);
            } catch (MetaException e) {
              throw new SemanticException(e);

View Full Code Here

    StatsAggregator statsAggregator = null;
    int ret = 0;


    try {
      // Stats setup:
      Warehouse wh = new Warehouse(conf);
      FileSystem fileSys;
      FileStatus[] fileStatus;


      if (!this.getWork().getNoStatsAggregator()) {
        String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
        StatsFactory.setImplementation(statsImplementationClass, conf);
        statsAggregator = StatsFactory.getStatsAggregator();
        // manufacture a StatsAggregator
        if (!statsAggregator.connect(conf)) {
          throw new HiveException("StatsAggregator connect failed " + statsImplementationClass);
        }
      }


      TableStatistics tblStats = new TableStatistics();


      org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
      Map<String, String> parameters = tTable.getParameters();


      boolean tableStatsExist = this.existStats(parameters);


      for (String statType : supportedStats) {
        if (parameters.containsKey(statType)) {
          tblStats.setStat(statType, Long.parseLong(parameters.get(statType)));
        }
      }


      if (parameters.containsKey(StatsSetupConst.NUM_PARTITIONS)) {
        tblStats.setNumPartitions(Integer.parseInt(parameters.get(StatsSetupConst.NUM_PARTITIONS)));
      }


      List<Partition> partitions = getPartitionsList();
      boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC);


      if (partitions == null) {
        // non-partitioned tables:
        if (!tableStatsExist && atomic) {
          return 0;
        }
        Path tablePath = wh.getTablePath(db.getDatabase(table.getDbName()), table.getTableName());
        fileSys = tablePath.getFileSystem(conf);
        fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys);


        tblStats.setStat(StatsSetupConst.NUM_FILES, fileStatus.length);
        long tableSize = 0L;

View Full Code Here

    }
  }


  private void deleteDir(Path dir) throws HiveException {
    try {
      Warehouse wh = new Warehouse(conf);
      wh.deleteDir(dir, true);
    } catch (MetaException e) {
      throw new HiveException(e);
    }
  }

View Full Code Here

      } catch (HiveException e) {
        System.err.println(StringUtils.stringifyException(e));
        assertTrue("Unable to create table: " + tableName, false);
      }
      // get table
      Warehouse wh = new Warehouse(hiveConf);
      Table ft = null;
      try {
        ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        assertNotNull("Unable to fetch table", ft);
        ft.checkValidity();
        assertEquals("Table names didn't match for table: " + tableName, tbl.getName(), ft.getName());
        assertEquals("Table owners didn't match for table: " + tableName, tbl.getOwner(), ft.getOwner());
        assertEquals("Table retention didn't match for table: " + tableName, tbl.getRetention(), ft.getRetention());
        assertEquals("Data location is not set correctly", wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName).toString(), ft.getDataLocation().toString());
        // now that URI is set correctly, set the original table's uri and then compare the two tables
        tbl.setDataLocation(ft.getDataLocation());
        assertTrue("Tables  doesn't match: " + tableName, ft.getTTable().equals(tbl.getTTable()));
        assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), ThriftDeserializer.class.getName());
        assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.metastore.Warehouse

org.apache.cassandra.hadoop.hive.metastore.SchemaManagerService

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.ContentSummary

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Trash

org.apache.hadoop.hive.metastore.api.SkewedInfo

org.apache.hadoop.hive.ql.exec.DDLTask

org.apache.hadoop.hive.ql.exec.StatsTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.