Package org.apache.hadoop.hive.metastore

Examples of org.apache.hadoop.hive.metastore.Warehouse

267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
    StatsAggregator statsAggregator = null;

    try {
      // Stats setup:
      Warehouse wh = new Warehouse(conf);
      FileSystem fileSys;
      FileStatus[] fileStatus;

      if (!this.getWork().getNoStatsAggregator()) {
        String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
        StatsFactory.setImplementation(statsImplementationClass, conf);
        statsAggregator = StatsFactory.getStatsAggregator();
        // manufacture a StatsAggregator
        if (!statsAggregator.connect(conf)) {
          throw new HiveException("StatsAggregator connect failed " + statsImplementationClass);
        }
      }

      TableStatistics tblStats = new TableStatistics();

      org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
      Map<String, String> parameters = tTable.getParameters();

      boolean tableStatsExist = this.existStats(parameters);

      for (String statType : supportedStats) {
        if (parameters.containsKey(statType)) {
          tblStats.setStat(statType, Long.parseLong(parameters.get(statType)));
        }
      }

      if (parameters.containsKey(StatsSetupConst.NUM_PARTITIONS)) {
        tblStats.setNumPartitions(Integer.parseInt(parameters.get(StatsSetupConst.NUM_PARTITIONS)));
      }

      List<Partition> partitions = getPartitionsList();
      boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC);

      if (partitions == null) {
        // non-partitioned tables:
        if (!tableStatsExist && atomic) {
          return 0;
        }
        Path tablePath = wh.getTablePath(db.getDatabase(table.getDbName()), table.getTableName());
        fileSys = tablePath.getFileSystem(conf);
        fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys);

        tblStats.setStat(StatsSetupConst.NUM_FILES, fileStatus.length);
        long tableSize = 0L;
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        assertTrue("Unable to create table: " + tableName, false);
      }

      // get table
      Table ft = null;
      Warehouse wh = new Warehouse(hiveConf);
      try {
        ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        ft.checkValidity();
        assertEquals("Table names didn't match for table: " + tableName, tbl
            .getTableName(), ft.getTableName());
        assertEquals("Table owners didn't match for table: " + tableName, tbl
            .getOwner(), ft.getOwner());
        assertEquals("Table retention didn't match for table: " + tableName,
            tbl.getRetention(), ft.getRetention());
        assertEquals("Data location is not set correctly",
            wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(),
            ft.getDataLocation().toString());
        // now that URI is set correctly, set the original table's uri and then
        // compare the two tables
        tbl.setDataLocation(ft.getDataLocation());
        assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals(
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
      } catch (HiveException e) {
        System.err.println(StringUtils.stringifyException(e));
        assertTrue("Unable to create table: " + tableName, false);
      }
      // get table
      Warehouse wh = new Warehouse(hiveConf);
      Table ft = null;
      try {
        ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        assertNotNull("Unable to fetch table", ft);
        ft.checkValidity();
        assertEquals("Table names didn't match for table: " + tableName, tbl
            .getTableName(), ft.getTableName());
        assertEquals("Table owners didn't match for table: " + tableName, tbl
            .getOwner(), ft.getOwner());
        assertEquals("Table retention didn't match for table: " + tableName,
            tbl.getRetention(), ft.getRetention());
        assertEquals("Data location is not set correctly",
            wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(),
            ft.getDataLocation().toString());
        // now that URI is set correctly, set the original table's uri and then
        // compare the two tables
        tbl.setDataLocation(ft.getDataLocation());
        assertTrue("Tables  doesn't match: " + tableName, ft.getTTable()
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    String username = ShimLoader.getHadoopShims().getShortUserName(ugi);

    whPath = new Path(whDir);
    whFs = whPath.getFileSystem(conf);

    wh = new Warehouse(conf);
    hive = Hive.get(conf);

    //clean up mess in HMS
    HcatTestUtils.cleanupHMS(hive, wh, perm700);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
        // We dont set perms or groups for default dir.
        return 0;
      } else {
        try {
          Hive db = Hive.get();
          Path dbPath = new Warehouse(conf).getDatabasePath(db.getDatabase(dbName));
          FileSystem fs = dbPath.getFileSystem(conf);
          if (perms != null) {
            fs.setPermission(dbPath, perms);
          }
          if (null != grp) {
View Full Code Here
83
84
85
86
87
88
89
90
91
92
  @Override
  public void setConf(Configuration conf) {
    super.setConf(conf);
    try {
      this.wh = new Warehouse(conf);
    } catch (MetaException ex) {
      throw new RuntimeException(ex);
    }
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
    hcatConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hcatConf.set(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.varname, "60");
    hcatConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    clientWH = new Warehouse(hcatConf);
    msc = new HiveMetaStoreClient(hcatConf, null);
    System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
    System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
  }
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
        // We dont set perms or groups for default dir.
        return 0;
      } else {
        try {
          Hive db = Hive.get();
          Path dbPath = new Warehouse(conf).getDatabasePath(db.getDatabase(dbName));
          FileSystem fs = dbPath.getFileSystem(conf);
          if (perms != null) {
            fs.setPermission(dbPath, perms);
          }
          if (null != grp) {
View Full Code Here
95
96
97
98
99
100
101
102
103
104
    hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName());
    hcatConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    clientWH = new Warehouse(hcatConf);
    msc = new HiveMetaStoreClient(hcatConf, null);
    System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
    System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
  }
View Full Code Here
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
   * @param tbl
   * @param tableName
   * @throws MetaException
   */
  private void validateTable(Table tbl, String tableName) throws MetaException {
    Warehouse wh = new Warehouse(hiveConf);
    Table ft = null;
    try {
      // hm.getTable result will not have privileges set (it does not retrieve
      // that part from metastore), so unset privileges to null before comparing
      // (create table sets it to empty (non null) structures)
      tbl.getTTable().setPrivilegesIsSet(false);

      ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
      assertNotNull("Unable to fetch table", ft);
      ft.checkValidity();
      assertEquals("Table names didn't match for table: " + tableName, tbl
          .getTableName(), ft.getTableName());
      assertEquals("Table owners didn't match for table: " + tableName, tbl
          .getOwner(), ft.getOwner());
      assertEquals("Table retention didn't match for table: " + tableName,
          tbl.getRetention(), ft.getRetention());
      assertEquals("Data location is not set correctly",
          wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(),
          ft.getDataLocation().toString());
      // now that URI and times are set correctly, set the original table's uri and times
      // and then compare the two tables
      tbl.setDataLocation(ft.getDataLocation());
      tbl.setCreateTime(ft.getTTable().getCreateTime());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.metastore.Warehouse

  • org.apache.cassandra.hadoop.hive.metastore.SchemaManagerService
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.ContentSummary
  • org.apache.hadoop.fs.FileStatus
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.Trash
  • org.apache.hadoop.hive.metastore.api.SkewedInfo
  • org.apache.hadoop.hive.ql.exec.DDLTask
  • org.apache.hadoop.hive.ql.exec.StatsTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.