/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hcatalog.cli;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.mapreduce.HCatBaseTest;
import org.apache.thrift.TException;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestSemanticAnalysis extends HCatBaseTest {
private static final Logger LOG = LoggerFactory.getLogger(TestSemanticAnalysis.class);
private static final String TBL_NAME = "junit_sem_analysis";
private Driver hcatDriver = null;
private String query;
@Before
public void setUpHCatDriver() throws IOException {
if (hcatDriver == null) {
HiveConf hcatConf = new HiveConf(hiveConf);
hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
HCatSemanticAnalyzer.class.getName());
hcatDriver = new Driver(hcatConf);
SessionState.start(new CliSessionState(hcatConf));
}
}
@Test
public void testDescDB() throws CommandNeedRetryException, IOException {
hcatDriver.run("drop database mydb cascade");
assertEquals(0, hcatDriver.run("create database mydb").getResponseCode());
CommandProcessorResponse resp = hcatDriver.run("describe database mydb");
assertEquals(0, resp.getResponseCode());
ArrayList<String> result = new ArrayList<String>();
hcatDriver.getResults(result);
assertTrue(result.get(0).contains("mydb.db"));
hcatDriver.run("drop database mydb cascade");
}
@Test
public void testCreateTblWithLowerCasePartNames() throws CommandNeedRetryException, MetaException, TException, NoSuchObjectException {
driver.run("drop table junit_sem_analysis");
CommandProcessorResponse resp = driver.run("create table junit_sem_analysis (a int) partitioned by (B string) stored as TEXTFILE");
assertEquals(resp.getResponseCode(), 0);
assertEquals(null, resp.getErrorMessage());
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
assertEquals("Partition key name case problem", "b", tbl.getPartitionKeys().get(0).getName());
driver.run("drop table junit_sem_analysis");
}
@Test
public void testAlterTblFFpart() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
driver.run("drop table junit_sem_analysis");
driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE");
driver.run("alter table junit_sem_analysis add partition (b='2010-10-10')");
hcatDriver.run("alter table junit_sem_analysis partition (b='2010-10-10') set fileformat RCFILE");
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
assertEquals(TextInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(HiveIgnoreKeyTextOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
List<String> partVals = new ArrayList<String>(1);
partVals.add("2010-10-10");
Partition part = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals);
assertEquals(RCFileInputFormat.class.getName(), part.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), part.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testUsNonExistentDB() throws CommandNeedRetryException {
CommandProcessorResponse resp = hcatDriver.run("use no_such_db");
assertEquals(1, resp.getResponseCode());
}
@Test
public void testDatabaseOperations() throws MetaException, CommandNeedRetryException {
List<String> dbs = client.getAllDatabases();
String testDb1 = "testdatabaseoperatons1";
String testDb2 = "testdatabaseoperatons2";
if (dbs.contains(testDb1.toLowerCase())) {
assertEquals(0, hcatDriver.run("drop database " + testDb1).getResponseCode());
}
if (dbs.contains(testDb2.toLowerCase())) {
assertEquals(0, hcatDriver.run("drop database " + testDb2).getResponseCode());
}
assertEquals(0, hcatDriver.run("create database " + testDb1).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb1));
assertEquals(0, hcatDriver.run("create database if not exists " + testDb1).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb1));
assertEquals(0, hcatDriver.run("create database if not exists " + testDb2).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb2));
assertEquals(0, hcatDriver.run("drop database " + testDb1).getResponseCode());
assertEquals(0, hcatDriver.run("drop database " + testDb2).getResponseCode());
assertFalse(client.getAllDatabases().contains(testDb1));
assertFalse(client.getAllDatabases().contains(testDb2));
}
@Test
public void testCreateTableIfNotExists() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hcatDriver.run("drop table " + TBL_NAME);
hcatDriver.run("create table junit_sem_analysis (a int) stored as RCFILE");
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
List<FieldSchema> cols = tbl.getSd().getCols();
assertEquals(1, cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null)));
assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
CommandProcessorResponse resp = hcatDriver.run("create table if not exists junit_sem_analysis (a int) stored as RCFILE");
assertEquals(0, resp.getResponseCode());
assertNull(resp.getErrorMessage());
tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
cols = tbl.getSd().getCols();
assertEquals(1, cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null)));
assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAlterTblTouch() throws CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis touch");
assertEquals(0, response.getResponseCode());
hcatDriver.run("alter table junit_sem_analysis touch partition (b='12')");
assertEquals(0, response.getResponseCode());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testChangeColumns() throws CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis change a a1 int");
assertEquals(0, response.getResponseCode());
response = hcatDriver.run("alter table junit_sem_analysis change a1 a string");
assertEquals(0, response.getResponseCode());
response = hcatDriver.run("alter table junit_sem_analysis change a a int after c");
assertEquals(0, response.getResponseCode());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAddReplaceCols() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis replace columns (a1 tinyint)");
assertEquals(0, response.getResponseCode());
response = hcatDriver.run("alter table junit_sem_analysis add columns (d tinyint)");
assertEquals(0, response.getResponseCode());
assertNull(response.getErrorMessage());
response = hcatDriver.run("describe extended junit_sem_analysis");
assertEquals(0, response.getResponseCode());
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
List<FieldSchema> cols = tbl.getSd().getCols();
assertEquals(2, cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a1", "tinyint", null)));
assertTrue(cols.get(1).equals(new FieldSchema("d", "tinyint", null)));
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAlterTblClusteredBy() throws CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis clustered by (a) into 7 buckets");
assertEquals(0, response.getResponseCode());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAlterTableSetFF() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
hcatDriver.run("alter table junit_sem_analysis set fileformat INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
"'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver'");
hcatDriver.run("desc extended junit_sem_analysis");
tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAddPartFail() throws CommandNeedRetryException {
driver.run("drop table junit_sem_analysis");
driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location 'README.txt'");
assertEquals(0, response.getResponseCode());
driver.run("drop table junit_sem_analysis");
}
@Test
public void testAddPartPass() throws IOException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location '" + TEST_DATA_DIR + "'");
assertEquals(0, response.getResponseCode());
assertNull(response.getErrorMessage());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testCTAS() throws CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) as select * from tbl2";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(40000, response.getResponseCode());
assertTrue(response.getErrorMessage().contains("FAILED: SemanticException Operation not supported. Create table as Select is not a valid operation."));
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testStoredAs() throws CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int)";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(0, response.getResponseCode());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testAddDriverInfo() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as " +
"INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
"'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver' ";
assertEquals(0, hcatDriver.run(query).getResponseCode());
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
@Test
public void testInvalidateNonStringPartition() throws IOException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) partitioned by (b int) stored as RCFILE";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(40000, response.getResponseCode());
assertEquals("FAILED: SemanticException Operation not supported. HCatalog only supports partition columns of type string. For column: b Found type: int",
response.getErrorMessage());
}
@Test
public void testInvalidateSeqFileStoredAs() throws IOException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as SEQUENCEFILE";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(0, response.getResponseCode());
}
@Test
public void testInvalidateTextFileStoredAs() throws IOException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(0, response.getResponseCode());
}
@Test
public void testInvalidateClusteredBy() throws IOException, CommandNeedRetryException {
hcatDriver.run("drop table junit_sem_analysis");
query = "create table junit_sem_analysis (a int) partitioned by (b string) clustered by (a) into 10 buckets stored as TEXTFILE";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(0, response.getResponseCode());
}
@Test
public void testCTLFail() throws IOException, CommandNeedRetryException {
driver.run("drop table junit_sem_analysis");
driver.run("drop table like_table");
query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE";
driver.run(query);
query = "create table like_table like junit_sem_analysis";
CommandProcessorResponse response = hcatDriver.run(query);
assertEquals(0, response.getResponseCode());
}
@Test
public void testCTLPass() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
try {
hcatDriver.run("drop table junit_sem_analysis");
} catch (Exception e) {
LOG.error("Error in drop table.", e);
}
query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE";
hcatDriver.run(query);
String likeTbl = "like_table";
hcatDriver.run("drop table " + likeTbl);
query = "create table like_table like junit_sem_analysis";
CommandProcessorResponse resp = hcatDriver.run(query);
assertEquals(0, resp.getResponseCode());
// Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, likeTbl);
// assertEquals(likeTbl,tbl.getTableName());
// List<FieldSchema> cols = tbl.getSd().getCols();
// assertEquals(1, cols.size());
// assertEquals(new FieldSchema("a", "int", null), cols.get(0));
// assertEquals("org.apache.hadoop.hive.ql.io.RCFileInputFormat",tbl.getSd().getInputFormat());
// assertEquals("org.apache.hadoop.hive.ql.io.RCFileOutputFormat",tbl.getSd().getOutputFormat());
// Map<String, String> tblParams = tbl.getParameters();
// assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileInputStorageDriver", tblParams.get("hcat.isd"));
// assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileOutputStorageDriver", tblParams.get("hcat.osd"));
//
// hcatDriver.run("drop table junit_sem_analysis");
// hcatDriver.run("drop table "+likeTbl);
}
// This test case currently fails, since add partitions don't inherit anything from tables.
// public void testAddPartInheritDrivers() throws MetaException, TException, NoSuchObjectException{
//
// hcatDriver.run("drop table "+TBL_NAME);
// hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
// hcatDriver.run("alter table "+TBL_NAME+" add partition (b='2010-10-10')");
//
// List<String> partVals = new ArrayList<String>(1);
// partVals.add("2010-10-10");
//
// Map<String,String> map = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals).getParameters();
// assertEquals(map.get(InitializeInput.HOWL_ISD_CLASS), RCFileInputStorageDriver.class.getName());
// assertEquals(map.get(InitializeInput.HOWL_OSD_CLASS), RCFileOutputStorageDriver.class.getName());
// }
}