/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hive.service.cli.session;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.exec.FetchFormatter;
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.processors.SetProcessor;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.cli.*;
import org.apache.hive.service.cli.operation.ExecuteStatementOperation;
import org.apache.hive.service.cli.operation.GetCatalogsOperation;
import org.apache.hive.service.cli.operation.GetColumnsOperation;
import org.apache.hive.service.cli.operation.GetFunctionsOperation;
import org.apache.hive.service.cli.operation.GetSchemasOperation;
import org.apache.hive.service.cli.operation.GetTableTypesOperation;
import org.apache.hive.service.cli.operation.GetTypeInfoOperation;
import org.apache.hive.service.cli.operation.MetadataOperation;
import org.apache.hive.service.cli.operation.Operation;
import org.apache.hive.service.cli.operation.OperationManager;
import org.apache.hive.service.cli.thrift.TProtocolVersion;
import org.apache.hive.service.server.ThreadWithGarbageCleanup;
/**
* HiveSession
*
*/
public class HiveSessionImpl implements HiveSession {
private final SessionHandle sessionHandle;
private String username;
private final String password;
private HiveConf hiveConf;
private final SessionState sessionState;
private String ipAddress;
private static final String FETCH_WORK_SERDE_CLASS =
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
private static final Log LOG = LogFactory.getLog(HiveSessionImpl.class);
private SessionManager sessionManager;
private OperationManager operationManager;
private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
private boolean isOperationLogEnabled;
private File sessionLogDir;
private volatile long lastAccessTime;
public HiveSessionImpl(TProtocolVersion protocol, String username, String password,
HiveConf serverhiveConf, String ipAddress) {
this.username = username;
this.password = password;
this.sessionHandle = new SessionHandle(protocol);
this.hiveConf = new HiveConf(serverhiveConf);
this.ipAddress = ipAddress;
try {
// In non-impersonation mode, map scheduler queue to current user
// if fair scheduler is configured.
if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) {
ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username);
}
} catch (IOException e) {
LOG.warn("Error setting scheduler queue: " + e, e);
}
// Set an explicit session name to control the download directory name
hiveConf.set(ConfVars.HIVESESSIONID.varname,
sessionHandle.getHandleIdentifier().toString());
// Use thrift transportable formatter
hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER,
FetchFormatter.ThriftFormatter.class.getName());
hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue());
/**
* Create a new SessionState object that will be associated with this HiveServer2 session.
* When the server executes multiple queries in the same session,
* this SessionState object is reused across multiple queries.
*/
sessionState = new SessionState(hiveConf, username);
sessionState.setUserIpAddress(ipAddress);
sessionState.setIsHiveServerQuery(true);
lastAccessTime = System.currentTimeMillis();
SessionState.start(sessionState);
}
@Override
public void initialize(Map<String, String> sessionConfMap) throws Exception {
// Process global init file: .hiverc
processGlobalInitFile();
try {
sessionState.reloadAuxJars();
} catch (IOException e) {
String msg = "fail to load reloadable jar file path" + e;
LOG.error(msg, e);
throw new Exception(msg, e);
}
SessionState.setCurrentSessionState(sessionState);
// Set conf properties specified by user from client side
if (sessionConfMap != null) {
configureSession(sessionConfMap);
}
}
/**
* It is used for processing hiverc file from HiveServer2 side.
*/
private class GlobalHivercFileProcessor extends HiveFileProcessor {
@Override
protected BufferedReader loadFile(String fileName) throws IOException {
FileInputStream initStream = null;
BufferedReader bufferedReader = null;
initStream = new FileInputStream(fileName);
bufferedReader = new BufferedReader(new InputStreamReader(initStream));
return bufferedReader;
}
@Override
protected int processCmd(String cmd) {
int rc = 0;
String cmd_trimed = cmd.trim();
try {
executeStatementInternal(cmd_trimed, null, false);
} catch (HiveSQLException e) {
rc = -1;
LOG.warn("Failed to execute HQL command in global .hiverc file.", e);
}
return rc;
}
}
private void processGlobalInitFile() {
IHiveFileProcessor processor = new GlobalHivercFileProcessor();
try {
String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION);
if (hiverc != null) {
File hivercFile = new File(hiverc);
if (hivercFile.isDirectory()) {
hivercFile = new File(hivercFile, SessionManager.HIVERCFILE);
}
if (hivercFile.isFile()) {
LOG.info("Running global init file: " + hivercFile);
int rc = processor.processFile(hivercFile.getAbsolutePath());
if (rc != 0) {
LOG.error("Failed on initializing global .hiverc file");
}
} else {
LOG.debug("Global init file " + hivercFile + " does not exist");
}
}
} catch (IOException e) {
LOG.warn("Failed on initializing global .hiverc file", e);
}
}
private void configureSession(Map<String, String> sessionConfMap) throws Exception {
SessionState.setCurrentSessionState(sessionState);
for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
String key = entry.getKey();
if (key.startsWith("set:")) {
SetProcessor.setVariable(key.substring(4), entry.getValue());
} else if (key.startsWith("use:")) {
SessionState.get().setCurrentDatabase(entry.getValue());
} else {
hiveConf.verifyAndSet(key, entry.getValue());
}
}
}
@Override
public void setOperationLogSessionDir(File operationLogRootDir) {
sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString());
isOperationLogEnabled = true;
if (!sessionLogDir.exists()) {
if (!sessionLogDir.mkdir()) {
LOG.warn("Unable to create operation log session directory: " +
sessionLogDir.getAbsolutePath());
isOperationLogEnabled = false;
}
}
if (isOperationLogEnabled) {
LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath());
}
}
@Override
public boolean isOperationLogEnabled() {
return isOperationLogEnabled;
}
@Override
public File getOperationLogSessionDir() {
return sessionLogDir;
}
@Override
public TProtocolVersion getProtocolVersion() {
return sessionHandle.getProtocolVersion();
}
@Override
public SessionManager getSessionManager() {
return sessionManager;
}
@Override
public void setSessionManager(SessionManager sessionManager) {
this.sessionManager = sessionManager;
}
private OperationManager getOperationManager() {
return operationManager;
}
@Override
public void setOperationManager(OperationManager operationManager) {
this.operationManager = operationManager;
}
@Override
/**
* Opens a new HiveServer2 session for the client connection.
* Note that if doAs is true, this call goes through a proxy object,
* which wraps the method logic in a UserGroupInformation#doAs.
* That is why it is important to call SessionState#start here rather than the constructor.
*/
public void open() {
SessionState.start(sessionState);
}
protected synchronized void acquire(boolean userAccess) {
// Need to make sure that the this HiveServer2's session's session state is
// stored in the thread local for the handler thread.
SessionState.setCurrentSessionState(sessionState);
if (userAccess) {
lastAccessTime = System.currentTimeMillis();
}
}
/**
* 1. We'll remove the ThreadLocal SessionState as this thread might now serve
* other requests.
* 2. We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
* when this thread is garbage collected later.
* @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
*/
protected synchronized void release(boolean userAccess) {
SessionState.detachSession();
if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
ThreadWithGarbageCleanup currentThread =
(ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
currentThread.cacheThreadLocalRawStore();
}
if (userAccess) {
lastAccessTime = System.currentTimeMillis();
}
}
@Override
public SessionHandle getSessionHandle() {
return sessionHandle;
}
@Override
public String getUsername() {
return username;
}
@Override
public String getPassword() {
return password;
}
@Override
public HiveConf getHiveConf() {
hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE, FETCH_WORK_SERDE_CLASS);
return hiveConf;
}
@Override
public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
try {
return Hive.get(getHiveConf()).getMSC();
} catch (HiveException e) {
throw new HiveSQLException("Failed to get metastore connection", e);
} catch (MetaException e) {
throw new HiveSQLException("Failed to get metastore connection", e);
}
}
@Override
public GetInfoValue getInfo(GetInfoType getInfoType)
throws HiveSQLException {
acquire(true);
try {
switch (getInfoType) {
case CLI_SERVER_NAME:
return new GetInfoValue("Hive");
case CLI_DBMS_NAME:
return new GetInfoValue("Apache Hive");
case CLI_DBMS_VER:
return new GetInfoValue(HiveVersionInfo.getVersion());
case CLI_MAX_COLUMN_NAME_LEN:
return new GetInfoValue(128);
case CLI_MAX_SCHEMA_NAME_LEN:
return new GetInfoValue(128);
case CLI_MAX_TABLE_NAME_LEN:
return new GetInfoValue(128);
case CLI_TXN_CAPABLE:
default:
throw new HiveSQLException("Unrecognized GetInfoType value: " + getInfoType.toString());
}
} finally {
release(true);
}
}
@Override
public OperationHandle executeStatement(String statement, Map<String, String> confOverlay)
throws HiveSQLException {
return executeStatementInternal(statement, confOverlay, false);
}
@Override
public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay)
throws HiveSQLException {
return executeStatementInternal(statement, confOverlay, true);
}
private OperationHandle executeStatementInternal(String statement, Map<String, String> confOverlay,
boolean runAsync)
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
ExecuteStatementOperation operation = operationManager
.newExecuteStatementOperation(getSession(), statement, confOverlay, runAsync);
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
// Cleanup opHandle in case the query is synchronous
// Async query needs to retain and pass back the opHandle for error reporting
if (!runAsync) {
operationManager.closeOperation(opHandle);
}
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getTypeInfo()
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetTypeInfoOperation operation = operationManager.newGetTypeInfoOperation(getSession());
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getCatalogs()
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetCatalogsOperation operation = operationManager.newGetCatalogsOperation(getSession());
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getSchemas(String catalogName, String schemaName)
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetSchemasOperation operation =
operationManager.newGetSchemasOperation(getSession(), catalogName, schemaName);
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getTables(String catalogName, String schemaName, String tableName,
List<String> tableTypes)
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
MetadataOperation operation =
operationManager.newGetTablesOperation(getSession(), catalogName, schemaName, tableName, tableTypes);
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getTableTypes()
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetTableTypesOperation operation = operationManager.newGetTableTypesOperation(getSession());
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getColumns(String catalogName, String schemaName,
String tableName, String columnName) throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetColumnsOperation operation = operationManager.newGetColumnsOperation(getSession(),
catalogName, schemaName, tableName, columnName);
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public OperationHandle getFunctions(String catalogName, String schemaName, String functionName)
throws HiveSQLException {
acquire(true);
OperationManager operationManager = getOperationManager();
GetFunctionsOperation operation = operationManager
.newGetFunctionsOperation(getSession(), catalogName, schemaName, functionName);
OperationHandle opHandle = operation.getHandle();
try {
operation.run();
opHandleSet.add(opHandle);
return opHandle;
} catch (HiveSQLException e) {
operationManager.closeOperation(opHandle);
throw e;
} finally {
release(true);
}
}
@Override
public void close() throws HiveSQLException {
try {
acquire(true);
// Iterate through the opHandles and close their operations
for (OperationHandle opHandle : opHandleSet) {
operationManager.closeOperation(opHandle);
}
opHandleSet.clear();
// Cleanup session log directory.
cleanupSessionLogDir();
HiveHistory hiveHist = sessionState.getHiveHistory();
if (null != hiveHist) {
hiveHist.closeStream();
}
sessionState.close();
} catch (IOException ioe) {
throw new HiveSQLException("Failure to close", ioe);
} finally {
release(true);
}
}
private void cleanupSessionLogDir() {
if (isOperationLogEnabled) {
try {
FileUtils.forceDelete(sessionLogDir);
} catch (Exception e) {
LOG.error("Failed to cleanup session log dir: " + sessionHandle, e);
}
}
}
@Override
public SessionState getSessionState() {
return sessionState;
}
@Override
public String getUserName() {
return username;
}
@Override
public void setUserName(String userName) {
this.username = userName;
}
@Override
public long getLastAccessTime() {
return lastAccessTime;
}
@Override
public void closeExpiredOperations() {
OperationHandle[] handles = opHandleSet.toArray(new OperationHandle[opHandleSet.size()]);
if (handles.length > 0) {
List<Operation> operations = operationManager.removeExpiredOperations(handles);
if (!operations.isEmpty()) {
closeTimedOutOperations(operations);
}
}
}
private void closeTimedOutOperations(List<Operation> operations) {
acquire(false);
try {
for (Operation operation : operations) {
opHandleSet.remove(operation.getHandle());
try {
operation.close();
} catch (Exception e) {
LOG.warn("Exception is thrown closing timed-out operation " + operation.getHandle(), e);
}
}
} finally {
release(false);
}
}
@Override
public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
acquire(true);
try {
sessionManager.getOperationManager().cancelOperation(opHandle);
} finally {
release(true);
}
}
@Override
public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
acquire(true);
try {
operationManager.closeOperation(opHandle);
opHandleSet.remove(opHandle);
} finally {
release(true);
}
}
@Override
public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException {
acquire(true);
try {
return sessionManager.getOperationManager().getOperationResultSetSchema(opHandle);
} finally {
release(true);
}
}
@Override
public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
long maxRows, FetchType fetchType) throws HiveSQLException {
acquire(true);
try {
if (fetchType == FetchType.QUERY_OUTPUT) {
return operationManager.getOperationNextRowSet(opHandle, orientation, maxRows);
}
return operationManager.getOperationLogRowSet(opHandle, orientation, maxRows);
} finally {
release(true);
}
}
protected HiveSession getSession() {
return this;
}
@Override
public String getIpAddress() {
return ipAddress;
}
@Override
public void setIpAddress(String ipAddress) {
this.ipAddress = ipAddress;
}
@Override
public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer)
throws HiveSQLException {
HiveAuthFactory.verifyProxyAccess(getUsername(), owner, getIpAddress(), getHiveConf());
return authFactory.getDelegationToken(owner, renewer);
}
@Override
public void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
throws HiveSQLException {
HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
getIpAddress(), getHiveConf());
authFactory.cancelDelegationToken(tokenStr);
}
@Override
public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
throws HiveSQLException {
HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
getIpAddress(), getHiveConf());
authFactory.renewDelegationToken(tokenStr);
}
// extract the real user from the given token string
private String getUserFromToken(HiveAuthFactory authFactory, String tokenStr) throws HiveSQLException {
return authFactory.getUserFromToken(tokenStr);
}
}