/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.lockmgr;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnListImpl;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.ReflectionUtils;
import java.util.*;
/**
* An implementation of {@link HiveTxnManager} that does not support
* transactions. This provides default Hive behavior.
*/
class DummyTxnManager extends HiveTxnManagerImpl {
static final private Log LOG =
LogFactory.getLog(DummyTxnManager.class.getName());
private HiveLockManager lockMgr;
@Override
public void openTxn(String user) throws LockException {
// No-op
}
@Override
public HiveLockManager getLockManager() throws LockException {
if (lockMgr == null) {
boolean supportConcurrency =
conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
if (supportConcurrency) {
String lockMgrName =
conf.getVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER);
if ((lockMgrName == null) || (lockMgrName.isEmpty())) {
throw new LockException(ErrorMsg.LOCKMGR_NOT_SPECIFIED.getMsg());
}
try {
LOG.info("Creating lock manager of type " + lockMgrName);
lockMgr = (HiveLockManager)ReflectionUtils.newInstance(
conf.getClassByName(lockMgrName), conf);
lockMgr.setContext(new HiveLockManagerCtx(conf));
} catch (Exception e) {
// set hiveLockMgr to null just in case this invalid manager got set to
// next query's ctx.
if (lockMgr != null) {
try {
lockMgr.close();
} catch (LockException e1) {
//nothing can do here
}
lockMgr = null;
}
throw new LockException(ErrorMsg.LOCKMGR_NOT_INITIALIZED.getMsg() +
e.getMessage());
}
} else {
LOG.info("Concurrency mode is disabled, not creating a lock manager");
return null;
}
}
// Force a re-read of the configuration file. This is done because
// different queries in the session may be using the same lock manager.
lockMgr.refresh();
return lockMgr;
}
@Override
public void acquireLocks(QueryPlan plan, Context ctx, String username) throws LockException {
// Make sure we've built the lock manager
getLockManager();
// If the lock manager is still null, then it means we aren't using a
// lock manager
if (lockMgr == null) return;
List<HiveLockObj> lockObjects = new ArrayList<HiveLockObj>();
// Sort all the inputs, outputs.
// If a lock needs to be acquired on any partition, a read lock needs to be acquired on all
// its parents also
for (ReadEntity input : plan.getInputs()) {
LOG.debug("Adding " + input.getName() + " to list of lock inputs");
if (input.getType() == ReadEntity.Type.DATABASE) {
lockObjects.addAll(getLockObjects(plan, input.getDatabase(), null,
null, HiveLockMode.SHARED));
} else if (input.getType() == ReadEntity.Type.TABLE) {
lockObjects.addAll(getLockObjects(plan, null, input.getTable(), null,
HiveLockMode.SHARED));
} else {
lockObjects.addAll(getLockObjects(plan, null, null,
input.getPartition(),
HiveLockMode.SHARED));
}
}
for (WriteEntity output : plan.getOutputs()) {
LOG.debug("Adding " + output.getName() + " to list of lock outputs");
List<HiveLockObj> lockObj = null;
if (output.getType() == WriteEntity.Type.DATABASE) {
lockObjects.addAll(getLockObjects(plan, output.getDatabase(), null,
null,
output.isComplete() ? HiveLockMode.EXCLUSIVE : HiveLockMode.SHARED));
} else if (output.getTyp() == WriteEntity.Type.TABLE) {
lockObj = getLockObjects(plan, null, output.getTable(), null,
output.isComplete() ? HiveLockMode.EXCLUSIVE : HiveLockMode.SHARED);
} else if (output.getTyp() == WriteEntity.Type.PARTITION) {
lockObj = getLockObjects(plan, null, null, output.getPartition(),
HiveLockMode.EXCLUSIVE);
}
// In case of dynamic queries, it is possible to have incomplete dummy partitions
else if (output.getTyp() == WriteEntity.Type.DUMMYPARTITION) {
lockObj = getLockObjects(plan, null, null, output.getPartition(),
HiveLockMode.SHARED);
}
if(lockObj != null) {
lockObjects.addAll(lockObj);
ctx.getOutputLockObjects().put(output, lockObj);
}
}
if (lockObjects.isEmpty() && !ctx.isNeedLockMgr()) {
return;
}
HiveLockObject.HiveLockObjectData lockData =
new HiveLockObject.HiveLockObjectData(plan.getQueryId(),
String.valueOf(System.currentTimeMillis()),
"IMPLICIT",
plan.getQueryStr());
// Lock the database also
String currentDb = SessionState.get().getCurrentDatabase();
lockObjects.add(
new HiveLockObj(
new HiveLockObject(currentDb, lockData),
HiveLockMode.SHARED
)
);
dedupLockObjects(lockObjects);
List<HiveLock> hiveLocks = lockMgr.lock(lockObjects, false);
if (hiveLocks == null) {
throw new LockException(ErrorMsg.LOCK_CANNOT_BE_ACQUIRED.getMsg());
} else {
ctx.setHiveLocks(hiveLocks);
}
}
@Override
public void commitTxn() throws LockException {
// No-op
}
@Override
public void rollbackTxn() throws LockException {
// No-op
}
@Override
public void heartbeat() throws LockException {
// No-op
}
@Override
public ValidTxnList getValidTxns() throws LockException {
return new ValidTxnListImpl();
}
@Override
public boolean supportsExplicitLock() {
return true;
}
@Override
public boolean useNewShowLocksFormat() {
return false;
}
protected void destruct() {
if (lockMgr != null) {
try {
lockMgr.close();
} catch (LockException e) {
// Not much I can do about it.
LOG.warn("Got exception when closing lock manager " + e.getMessage());
}
}
}
/**
* Dedup the list of lock objects so that there is only one lock per table/partition.
* If there is both a shared and exclusive lock for the same object, this will deduped
* to just a single exclusive lock. Package level so that the unit tests
* can access it. Not intended for use outside this class.
* @param lockObjects
*/
static void dedupLockObjects(List<HiveLockObj> lockObjects) {
Map<String, HiveLockObj> lockMap = new HashMap<String, HiveLockObj>();
for (HiveLockObj lockObj : lockObjects) {
String lockName = lockObj.getName();
HiveLockObj foundLock = lockMap.get(lockName);
if (foundLock == null || lockObj.getMode() == HiveLockMode.EXCLUSIVE) {
lockMap.put(lockName, lockObj);
}
}
// copy set of deduped locks back to original list
lockObjects.clear();
for (HiveLockObj lockObj : lockMap.values()) {
lockObjects.add(lockObj);
}
}
private List<HiveLockObj> getLockObjects(QueryPlan plan, Database db,
Table t, Partition p,
HiveLockMode mode)
throws LockException {
List<HiveLockObj> locks = new LinkedList<HiveLockObj>();
HiveLockObject.HiveLockObjectData lockData =
new HiveLockObject.HiveLockObjectData(plan.getQueryId(),
String.valueOf(System.currentTimeMillis()),
"IMPLICIT",
plan.getQueryStr());
if (db != null) {
locks.add(new HiveLockObj(new HiveLockObject(db.getName(), lockData),
mode));
return locks;
}
if (t != null) {
locks.add(new HiveLockObj(new HiveLockObject(t, lockData), mode));
mode = HiveLockMode.SHARED;
locks.add(new HiveLockObj(new HiveLockObject(t.getDbName(), lockData), mode));
return locks;
}
if (p != null) {
if (!(p instanceof DummyPartition)) {
locks.add(new HiveLockObj(new HiveLockObject(p, lockData), mode));
}
// All the parents are locked in shared mode
mode = HiveLockMode.SHARED;
// For dummy partitions, only partition name is needed
String name = p.getName();
if (p instanceof DummyPartition) {
name = p.getName().split("@")[2];
}
String partialName = "";
String[] partns = name.split("/");
int len = p instanceof DummyPartition ? partns.length : partns.length - 1;
Map<String, String> partialSpec = new LinkedHashMap<String, String>();
for (int idx = 0; idx < len; idx++) {
String partn = partns[idx];
partialName += partn;
String[] nameValue = partn.split("=");
assert(nameValue.length == 2);
partialSpec.put(nameValue[0], nameValue[1]);
try {
locks.add(new HiveLockObj(
new HiveLockObject(new DummyPartition(p.getTable(), p.getTable().getDbName()
+ "/" + p.getTable().getTableName()
+ "/" + partialName,
partialSpec), lockData), mode));
partialName += "/";
} catch (HiveException e) {
throw new LockException(e.getMessage());
}
}
locks.add(new HiveLockObj(new HiveLockObject(p.getTable(), lockData), mode));
locks.add(new HiveLockObj(new HiveLockObject(p.getTable().getDbName(), lockData), mode));
}
return locks;
}
}