* @param outputInfo the table output info
* @throws IOException the exception in communicating with the metadata server
*/
@SuppressWarnings("unchecked")
public static void setOutput(Job job, HowlTableInfo outputInfo) throws IOException {
HiveMetaStoreClient client = null;
try {
Configuration conf = job.getConfiguration();
client = createHiveClient(outputInfo.getServerUri(), conf);
Table table = client.getTable(outputInfo.getDatabaseName(), outputInfo.getTableName());
if( outputInfo.getPartitionValues() == null ) {
outputInfo.setPartitionValues(new HashMap<String, String>());
} else {
//Convert user specified map to have lower case key names
Map<String, String> valueMap = new HashMap<String, String>();
for(Map.Entry<String, String> entry : outputInfo.getPartitionValues().entrySet()) {
valueMap.put(entry.getKey().toLowerCase(), entry.getValue());
}
outputInfo.setPartitionValues(valueMap);
}
//Handle duplicate publish
handleDuplicatePublish(job, outputInfo, client, table);
StorageDescriptor tblSD = table.getSd();
HowlSchema tableSchema = HowlUtil.extractSchemaFromStorageDescriptor(tblSD);
StorerInfo storerInfo = InitializeInput.extractStorerInfo(tblSD,table.getParameters());
List<String> partitionCols = new ArrayList<String>();
for(FieldSchema schema : table.getPartitionKeys()) {
partitionCols.add(schema.getName());
}
Class<? extends HowlOutputStorageDriver> driverClass =
(Class<? extends HowlOutputStorageDriver>) Class.forName(storerInfo.getOutputSDClass());
HowlOutputStorageDriver driver = driverClass.newInstance();
String tblLocation = tblSD.getLocation();
String location = driver.getOutputLocation(job,
tblLocation, partitionCols,
outputInfo.getPartitionValues());
//Serialize the output info into the configuration
OutputJobInfo jobInfo = new OutputJobInfo(outputInfo,
tableSchema, tableSchema, storerInfo, location, table);
conf.set(HOWL_KEY_OUTPUT_INFO, HowlUtil.serialize(jobInfo));
Path tblPath = new Path(tblLocation);
/* Set the umask in conf such that files/dirs get created with table-dir
* permissions. Following three assumptions are made:
* 1. Actual files/dirs creation is done by RecordWriter of underlying
* output format. It is assumed that they use default permissions while creation.
* 2. Default Permissions = FsPermission.getDefault() = 777.
* 3. UMask is honored by underlying filesystem.
*/
FsPermission.setUMask(conf, FsPermission.getDefault().applyUMask(
tblPath.getFileSystem(conf).getFileStatus(tblPath).getPermission()));
if(UserGroupInformation.isSecurityEnabled()){
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
// check if oozie has set up a howl deleg. token - if so use it
TokenSelector<? extends TokenIdentifier> tokenSelector = new DelegationTokenSelector();
// TODO: will oozie use a "service" called "oozie" - then instead of
// new Text() do new Text("oozie") below - if this change is made also
// remember to do:
// job.getConfiguration().set(HOWL_KEY_TOKEN_SIGNATURE, "oozie");
// Also change code in HowlOutputCommitter.cleanupJob() to cancel the
// token only if token.service is not "oozie" - remove the condition of
// HOWL_KEY_TOKEN_SIGNATURE != null in that code.
Token<? extends TokenIdentifier> token = tokenSelector.selectToken(
new Text(), ugi.getTokens());
if(token != null) {
job.getCredentials().addToken(new Text(ugi.getUserName()),token);
} else {
// we did not get token set up by oozie, let's get them ourselves here.
// we essentially get a token per unique Output HowlTableInfo - this is
// done because through Pig, setOutput() method is called multiple times
// We want to only get the token once per unique output HowlTableInfo -
// we cannot just get one token since in multi-query case (> 1 store in 1 job)
// or the case when a single pig script results in > 1 jobs, the single
// token will get cancelled by the output committer and the subsequent
// stores will fail - by tying the token with the concatenation of
// dbname, tablename and partition keyvalues of the output
// TableInfo, we can have as many tokens as there are stores and the TokenSelector
// will correctly pick the right tokens which the committer will use and
// cancel.
String tokenSignature = getTokenSignature(outputInfo);
if(tokenMap.get(tokenSignature) == null) {
// get delegation tokens from howl server and store them into the "job"
// These will be used in the HowlOutputCommitter to publish partitions to
// howl
String tokenStrForm = client.getDelegationTokenWithSignature(ugi.getUserName(),
tokenSignature);
Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>();
t.decodeFromUrlString(tokenStrForm);
tokenMap.put(tokenSignature, t);
}
job.getCredentials().addToken(new Text(ugi.getUserName() + tokenSignature),
tokenMap.get(tokenSignature));
// this will be used by the outputcommitter to pass on to the metastore client
// which in turn will pass on to the TokenSelector so that it can select
// the right token.
job.getConfiguration().set(HOWL_KEY_TOKEN_SIGNATURE, tokenSignature);
}
}
} catch(Exception e) {
if( e instanceof HowlException ) {
throw (HowlException) e;
} else {
throw new HowlException(ErrorType.ERROR_SET_OUTPUT, e);
}
} finally {
if( client != null ) {
client.close();
}
}
}