Examples of org.apache.hadoop.hive.metastore.Warehouse

Package org.apache.hadoop.hive.metastore

Examples of org.apache.hadoop.hive.metastore.Warehouse

org.apache.hadoop.hive.metastore.Warehouse
This class represents a warehouse where data of Hive tables is stored

  }


  private int aggregateStats() {
    try {
      // Stats setup:
      Warehouse wh = new Warehouse(conf);
      FileSystem fileSys;
      FileStatus[] fileStatus;


      // manufacture a StatsAggregator
      StatsAggregator statsAggregator;
      String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
      StatsFactory.setImplementation(statsImplementationClass, conf);
      statsAggregator = StatsFactory.getStatsAggregator();
      if (!statsAggregator.connect(conf)) {
        // this should not fail the whole job, return 0 so that the job won't fail.
        console.printInfo("[WARNING] Could not update table/partition level stats.",
            "StatsAggregator.connect() failed: stats class = " +
            statsImplementationClass);
        return 0;
      }




      TableStatistics tblStats = new TableStatistics();


      //
      // For partitioned table get the old table statistics for incremental update
      //
      if (table.isPartitioned()) {
        org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
        Map<String, String> parameters = tTable.getParameters();
        if (parameters.containsKey(StatsSetupConst.ROW_COUNT)) {
          tblStats.setNumRows(Long.parseLong(parameters.get(StatsSetupConst.ROW_COUNT)));
        }
        if (parameters.containsKey(StatsSetupConst.NUM_PARTITIONS)) {
          tblStats.setNumPartitions(Integer.parseInt(parameters.get(StatsSetupConst.NUM_PARTITIONS)));
        }
        if (parameters.containsKey(StatsSetupConst.NUM_FILES)) {
          tblStats.setNumFiles(Integer.parseInt(parameters.get(StatsSetupConst.NUM_FILES)));
        }
        if (parameters.containsKey(StatsSetupConst.TOTAL_SIZE)) {
          tblStats.setSize(Long.parseLong(parameters.get(StatsSetupConst.TOTAL_SIZE)));
        }
      }


      List<Partition> partitions = getPartitionsList();


      if (partitions == null) {
        // non-partitioned tables:


        Path tablePath = wh.getDefaultTablePath(table.getDbName(), table.getTableName());
        fileSys = tablePath.getFileSystem(conf);
        fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys);
        tblStats.setNumFiles(fileStatus.length);
        long tableSize = 0L;
        for (int i = 0; i < fileStatus.length; i++) {

View Full Code Here

    }
  }


  private void deleteDir(Path dir) throws HiveException {
    try {
      Warehouse wh = new Warehouse(conf);
      wh.deleteDir(dir, true);
    } catch (MetaException e) {
      throw new HiveException(e);
    }
  }

View Full Code Here


    howlConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HowlSemanticAnalyzer.class.getName());
    howlConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    howlConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    howlConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    clientWH = new Warehouse(howlConf);
    msc = new HiveMetaStoreClient(howlConf,null);
    System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
    System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
  }

View Full Code Here

    }
    catch(Exception e){
      assertTrue(e instanceof ExitException);
      assertEquals(((ExitException)e).getStatus(), 0);
    }
    Warehouse wh = new Warehouse(conf);
    Path dfsPath = wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, "testNoPartTbl");
    FileSystem fs = dfsPath.getFileSystem(conf);
    assertEquals(fs.getFileStatus(dfsPath).getPermission(),FsPermission.valueOf("drwx-wx---"));


    pig.setBatchOn();
    pig.registerQuery("A  = load 'build.xml' as (line:chararray);");
    pig.registerQuery("store A into 'testNoPartTbl' using "+HowlStorer.class.getName()+"();");
    pig.executeBatch();
    FileStatus[] status = fs.listStatus(dfsPath,hiddenFileFilter);


    assertEquals(status.length, 1);
    assertEquals(FsPermission.valueOf("drwx-wx---"),status[0].getPermission());


    try{
      HowlCli.main(new String[]{"-e","create table testPartTbl (line string)  partitioned by (a string) stored as RCFILE", "-p","rwx-wx--x"});
    }
    catch(Exception e){
      assertTrue(e instanceof ExitException);
      assertEquals(((ExitException)e).getStatus(), 0);
    }


    dfsPath = wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, "testPartTbl");
    assertEquals(fs.getFileStatus(dfsPath).getPermission(),FsPermission.valueOf("drwx-wx--x"));


    pig.setBatchOn();
    pig.registerQuery("A  = load 'build.xml' as (line:chararray);");
    pig.registerQuery("store A into 'testPartTbl' using "+HowlStorer.class.getName()+"('a=part');");

View Full Code Here

  private void authorize(HiveSemanticAnalyzerHookContext context, String loc) throws SemanticException{


    Path tblDir;
    Configuration conf = context.getConf();
    try {
      Warehouse wh = new Warehouse(conf);
      if (loc == null || loc.isEmpty()){
        tblDir = wh.getDnsPath(wh.getDefaultTablePath(context.getHive().getCurrentDatabase(), tableName).getParent());
      }
      else{
        tblDir = wh.getDnsPath(new Path(loc));
      }


      try {
        AuthUtils.authorize(tblDir, FsAction.WRITE, conf);
      } catch (HowlException e) {

View Full Code Here


  private void authorize(String name, HiveSemanticAnalyzerHookContext cntxt, FsAction action, boolean isDBOp)
                                                      throws MetaException, HiveException, HowlException{




    Warehouse wh = new Warehouse(cntxt.getConf());
    if(!isDBOp){
      // Do validations for table path.
      Table tbl;
      try{
        tbl = cntxt.getHive().getTable(name);
      }
      catch(InvalidTableException ite){
        // Table itself doesn't exist in metastore, nothing to validate.
        return;
      }
      Path path = tbl.getPath();
      if(path != null){
        AuthUtils.authorize(wh.getDnsPath(path), action, cntxt.getConf());
      } else{
        // This will happen, if table exists in metastore for a given
        // tablename, but has no path associated with it, so there is nothing to check.
        // In such cases, do no checks and allow whatever hive behavior is for it.
        return;
      }
    } else{
      // Else, its a DB operation.
      AuthUtils.authorize(wh.getDefaultDatabasePath(name), action, cntxt.getConf());
    }
  }

View Full Code Here

        // We dont set perms or groups for default dir.
        return 0;
      }
      else{
        try{
          Path dbPath = new Warehouse(conf).getDefaultDatabasePath(dbName);
          FileSystem fs = dbPath.getFileSystem(conf);
          if(perms != null){
            fs.setPermission(dbPath, perms);
          }
          if(null != grp){

View Full Code Here


    hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName());
    hcatConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    clientWH = new Warehouse(hcatConf);
    msc = new HiveMetaStoreClient(hcatConf, null);
    System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
    System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
  }

View Full Code Here

    }
  }


  private void deleteDir(Path dir) throws HiveException {
    try {
      Warehouse wh = new Warehouse(conf);
      wh.deleteDir(dir, true);
    } catch (MetaException e) {
      throw new HiveException(e);
    }
  }

View Full Code Here

              // allocate a temporary output dir on the location of the table
              String tableName = getUnescapedName((ASTNode) ast.getChild(0));
              Table newTable = db.newTable(tableName);
              Path location;
              try {
                Warehouse wh = new Warehouse(conf);
                location = wh.getDatabasePath(db.getDatabase(newTable.getDbName()));
              } catch (MetaException e) {
                throw new SemanticException(e);
              }
              try {
                fname = ctx.getExternalTmpFileURI(

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.metastore.Warehouse

org.apache.cassandra.hadoop.hive.metastore.SchemaManagerService

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.ContentSummary

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Trash

org.apache.hadoop.hive.metastore.api.SkewedInfo

org.apache.hadoop.hive.ql.exec.DDLTask

org.apache.hadoop.hive.ql.exec.StatsTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.