Package org.apache.hadoop.hive.metastore.api

Examples of org.apache.hadoop.hive.metastore.api.Table

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
  public void testAlterTableSetFF() throws IOException, MetaException, TException, NoSuchObjectException{

    howlDriver.run("drop table junit_sem_analysis");
    howlDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");

    Table tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
    assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    Map<String,String> tblParams = tbl.getParameters();
    assertEquals(RCFileInputDriver.class.getName(), tblParams.get("howl.isd"));
    assertEquals(RCFileOutputDriver.class.getName(), tblParams.get("howl.osd"));

    howlDriver.run("alter table junit_sem_analysis set fileformat INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
        "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver'");
    howlDriver.run("desc extended junit_sem_analysis");

    tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
    assertEquals("org.apache.hadoop.hive.ql.io.RCFileInputFormat",tbl.getSd().getInputFormat());
    assertEquals("org.apache.hadoop.hive.ql.io.RCFileOutputFormat",tbl.getSd().getOutputFormat());
    tblParams = tbl.getParameters();
    assertEquals("mydriver", tblParams.get("howl.isd"));
    assertEquals("yourdriver", tblParams.get("howl.osd"));

    howlDriver.run("drop table junit_sem_analysis");
  }
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    query =  "create table junit_sem_analysis (a int) partitioned by (b string)  stored as " +
        "INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
        "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver' ";
    assertEquals(0,howlDriver.run(query).getResponseCode());

    Table tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
    assertEquals("org.apache.hadoop.hive.ql.io.RCFileInputFormat",tbl.getSd().getInputFormat());
    assertEquals("org.apache.hadoop.hive.ql.io.RCFileOutputFormat",tbl.getSd().getOutputFormat());
    Map<String, String> tblParams = tbl.getParameters();
    assertEquals("mydriver", tblParams.get("howl.isd"));
    assertEquals("yourdriver", tblParams.get("howl.osd"));

    howlDriver.run("drop table junit_sem_analysis");
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    assertNotNull((client.getDatabase(dbName).getLocationUri()));

    List<FieldSchema> fields = new ArrayList<FieldSchema>();
    fields.add(new FieldSchema("colname", Constants.STRING_TYPE_NAME, ""));

    Table tbl = new Table();
    tbl.setDbName(dbName);
    tbl.setTableName(tblName);
    StorageDescriptor sd = new StorageDescriptor();
    sd.setCols(fields);
    tbl.setSd(sd);

    //sd.setLocation("hdfs://tmp");
    sd.setParameters(new HashMap<String, String>());
    sd.getParameters().put("test_param_1", "Use this for comments etc");
    sd.setBucketCols(new ArrayList<String>(2));
    sd.getBucketCols().add("name");
    sd.setSerdeInfo(new SerDeInfo());
    sd.getSerdeInfo().setName(tbl.getTableName());
    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    sd.getSerdeInfo().getParameters().put(
        org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
    sd.getSerdeInfo().setSerializationLib(
        org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
    tbl.setPartitionKeys(fields);

    Map<String, String> tableParams = new HashMap<String, String>();
    tableParams.put(HowlConstants.HOWL_OSD_CLASS, RCFileOutputDriver.class.getName());
    tableParams.put(HowlConstants.HOWL_ISD_CLASS, "testInputClass");
    tableParams.put("howl.testarg", "testArgValue");

    tbl.setParameters(tableParams);

    client.createTable(tbl);
    Path tblPath = new Path(client.getTable(dbName, tblName).getSd().getLocation());
    assertTrue(tblPath.getFileSystem(hiveConf).mkdirs(new Path(tblPath,"colname=p1")));
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
      client.dropTable(databaseName, tableName);
    } catch(Exception e) {
    } //can fail with NoSuchObjectException


    Table tbl = new Table();
    tbl.setDbName(databaseName);
    tbl.setTableName(tableName);
    tbl.setTableType("MANAGED_TABLE");
    StorageDescriptor sd = new StorageDescriptor();

    sd.setCols(getTableColumns());
    tbl.setPartitionKeys(getPartitionKeys());

    tbl.setSd(sd);

    sd.setBucketCols(new ArrayList<String>(2));
    sd.setSerdeInfo(new SerDeInfo());
    sd.getSerdeInfo().setName(tbl.getTableName());
    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    sd.getSerdeInfo().getParameters().put(
        org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
    sd.getSerdeInfo().setSerializationLib(serdeClass);
    sd.setInputFormat(inputFormat);
    sd.setOutputFormat(outputFormat);

    Map<String, String> tableParams = new HashMap<String, String>();
    tableParams.put(HowlConstants.HOWL_ISD_CLASS, inputSD);
    tableParams.put(HowlConstants.HOWL_OSD_CLASS, outputSD);
    tbl.setParameters(tableParams);

    client.createTable(tbl);
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    if(retCode != 0) {
      throw new IOException("Failed to create table.");
    }

    // assert that the table created has no howl instrumentation, and that we're still able to read it.
    Table table = client.getTable("default", "junit_unparted_noisd");
    assertFalse(table.getParameters().containsKey(HowlConstants.HOWL_ISD_CLASS));
    assertTrue(table.getSd().getInputFormat().equals(HowlConstants.HIVE_RCFILE_IF_CLASS));
   
    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.registerQuery("A = load '"+fullFileName+"' as (a:int);");
    server.registerQuery("store A into 'default.junit_unparted_noisd' using org.apache.howl.pig.HowlStorer();");
    server.registerQuery("B = load 'default.junit_unparted_noisd' using "+HowlLoader.class.getName()+"();");
    Iterator<Tuple> itr= server.openIterator("B");

    int i = 0;

    while(itr.hasNext()){
      Tuple t = itr.next();
      assertEquals(1, t.size());
      assertEquals(t.get(0), i);
      i++;
    }

    assertFalse(itr.hasNext());
    assertEquals(11, i);
   
    // assert that the table created still has no howl instrumentation
    Table table2 = client.getTable("default", "junit_unparted_noisd");
    assertFalse(table2.getParameters().containsKey(HowlConstants.HOWL_ISD_CLASS));
    assertTrue(table2.getSd().getInputFormat().equals(HowlConstants.HIVE_RCFILE_IF_CLASS));

    driver.run("drop table junit_unparted_noisd");
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    if(retCode != 0) {
      throw new IOException("Failed to create table.");
    }

    // assert that the table created has no howl instrumentation, and that we're still able to read it.
    Table table = client.getTable("default", "junit_parted_noisd");
   
    assertFalse(table.getParameters().containsKey(HowlConstants.HOWL_ISD_CLASS));
    assertTrue(table.getSd().getInputFormat().equals(HowlConstants.HIVE_RCFILE_IF_CLASS));

    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.registerQuery("A = load '"+fullFileName+"' as (a:int);");
    server.registerQuery("store A into 'default.junit_parted_noisd' using org.apache.howl.pig.HowlStorer('b=42');");
    server.registerQuery("B = load 'default.junit_parted_noisd' using "+HowlLoader.class.getName()+"();");
    Iterator<Tuple> itr= server.openIterator("B");

    int i = 0;

    while(itr.hasNext()){
      Tuple t = itr.next();
      assertEquals(2, t.size());
      assertEquals(t.get(0), i);
      assertEquals(t.get(1), "42");
      i++;
    }

    assertFalse(itr.hasNext());
    assertEquals(11, i);
   
    // assert that the table created still has no howl instrumentation
    Table table2 = client.getTable("default", "junit_parted_noisd");
    assertFalse(table2.getParameters().containsKey(HowlConstants.HOWL_ISD_CLASS));
    assertTrue(table2.getSd().getInputFormat().equals(HowlConstants.HIVE_RCFILE_IF_CLASS));
   
    // assert that there is one partition present, and it had howl instrumentation inserted when it was created.
    Partition ptn = client.getPartition("default", "junit_parted_noisd", Arrays.asList("42"));

    assertNotNull(ptn);
View Full Code Here
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
        throws NoSuchObjectException, MetaException {

      boolean success = false;
      boolean isExternal = false;
      Path tblPath = null;
      Table tbl = null;
      isExternal = false;
      boolean isIndexTable = false;
      try {
        ms.openTransaction();
        // drop any partitions
        tbl = get_table(dbname, name);
        if (tbl == null) {
          throw new NoSuchObjectException(name + " doesn't exist");
        }
        if (tbl.getSd() == null) {
          throw new MetaException("Table metadata is corrupted");
        }

        isIndexTable = isIndexTable(tbl);
        if (isIndexTable) {
          throw new RuntimeException(
              "The table " + name + " is an index table. Please do drop index instead.");
        }

        if (!isIndexTable) {
          try {
            List<Index> indexes = ms.getIndexes(dbname, name, Short.MAX_VALUE);
            while(indexes != null && indexes.size()>0) {
              for (Index idx : indexes) {
                this.drop_index_by_name(dbname, name, idx.getIndexName(), true);
              }
              indexes = ms.getIndexes(dbname, name, Short.MAX_VALUE);
            }
          } catch (TException e) {
            throw new MetaException(e.getMessage());
          }
        }
        isExternal = isExternal(tbl);
        if (tbl.getSd().getLocation() != null) {
          tblPath = new Path(tbl.getSd().getLocation());
        }

        if (!ms.dropTable(dbname, name)) {
          throw new MetaException("Unable to drop table");
        }
View Full Code Here
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
      return MetaStoreUtils.isIndexTable(table);
    }

    public Table get_table(final String dbname, final String name) throws MetaException,
        NoSuchObjectException {
      Table t = null;
      incrementCounter("get_table");
      logStartTableFunction("get_table", dbname, name);
      try {
        t = executeWithRetry(new Command<Table>() {
          @Override
          Table run(RawStore ms) throws Exception {
            Table t = ms.getTable(dbname, name);
            if (t == null) {
              throw new NoSuchObjectException(dbname + "." + name
                  + " table not found");
            }
            return t;
View Full Code Here
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
        part = new Partition();
        part.setDbName(dbName);
        part.setTableName(tableName);
        part.setValues(part_vals);

        Table tbl = ms.getTable(part.getDbName(), part.getTableName());
        if (tbl == null) {
          throw new InvalidObjectException(
              "Unable to add partition because table or database do not exist");
        }

        part.setSd(tbl.getSd());
        partLocation = new Path(tbl.getSd().getLocation(), Warehouse
            .makePartName(tbl.getPartitionKeys(), part_vals));
        part.getSd().setLocation(partLocation.toString());

        Partition old_part = null;
        try {
          old_part = get_partition(part.getDbName(), part
View Full Code Here
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
          old_part = null;
        }
        if (old_part != null) {
          throw new AlreadyExistsException("Partition already exists:" + part);
        }
        Table tbl = ms.getTable(part.getDbName(), part.getTableName());
        if (tbl == null) {
          throw new InvalidObjectException(
              "Unable to add partition because table or database do not exist");
        }

        String partLocationStr = part.getSd().getLocation();
        if (partLocationStr == null || partLocationStr.isEmpty()) {
          // set default location if not specified
          partLocation = new Path(tbl.getSd().getLocation(), Warehouse
              .makePartName(tbl.getPartitionKeys(), part.getValues()));

        } else {
          partLocation = wh.getDnsPath(new Path(partLocationStr));
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.metastore.api.Table

  • com.cloudera.cdk.data.hcatalog.HiveTestUtils
  • com.facebook.giraph.hive.input.HiveApiInputFormat
  • com.facebook.giraph.hive.output.HiveApiOutputCommitter
  • com.facebook.giraph.hive.output.HiveApiOutputFormat
  • com.facebook.giraph.hive.schema.HiveTableSchemas
  • com.facebook.hive.metastore.client.TestHiveMetastoreClient
  • com.facebook.hive.metastore.client.TestHiveMetastoreClientModule
  • com.facebook.hive.metastore.client.TestRetryingHiveMetastore
  • com.facebook.hiveio.input.HiveApiInputFormat
  • com.facebook.hiveio.output.HiveApiOutputCommitter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.