moveTaskOutputs(fs, src, src, tblPath,false);
fs.delete(src, true);
return;
}
HiveMetaStoreClient client = null;
List<String> values = null;
HCatTableInfo tableInfo = jobInfo.getTableInfo();
List<Partition> partitionsAdded = new ArrayList<Partition>();
try {
client = HCatOutputFormat.createHiveClient(tableInfo.getServerUri(), conf);
StorerInfo storer = InitializeInput.extractStorerInfo(table.getSd(),table.getParameters());
updateTableSchema(client, table, jobInfo.getOutputSchema());
FileStatus tblStat = fs.getFileStatus(tblPath);
String grpName = tblStat.getGroup();
FsPermission perms = tblStat.getPermission();
List<Partition> partitionsToAdd = new ArrayList<Partition>();
if (!dynamicPartitioningUsed){
partitionsToAdd.add(
constructPartition(
context,
tblPath.toString(), tableInfo.getPartitionValues()
,jobInfo.getOutputSchema(), getStorerParameterMap(storer)
,table, fs
,grpName,perms));
}else{
for (Entry<String,Map<String,String>> entry : partitionsDiscoveredByPath.entrySet()){
partitionsToAdd.add(
constructPartition(
context,
getPartitionRootLocation(entry.getKey(),entry.getValue().size()), entry.getValue()
,jobInfo.getOutputSchema(), getStorerParameterMap(storer)
,table, fs
,grpName,perms));
}
}
//Publish the new partition(s)
if (dynamicPartitioningUsed && harProcessor.isEnabled() && (!partitionsToAdd.isEmpty())){
Path src = new Path(ptnRootLocation);
// check here for each dir we're copying out, to see if it already exists, error out if so
moveTaskOutputs(fs, src, src, tblPath,true);
moveTaskOutputs(fs, src, src, tblPath,false);
fs.delete(src, true);
// for (Partition partition : partitionsToAdd){
// partitionsAdded.add(client.add_partition(partition));
// // currently following add_partition instead of add_partitions because latter isn't
// // all-or-nothing and we want to be able to roll back partitions we added if need be.
// }
try {
client.add_partitions(partitionsToAdd);
partitionsAdded = partitionsToAdd;
} catch (Exception e){
// There was an error adding partitions : rollback fs copy and rethrow
for (Partition p : partitionsToAdd){
Path ptnPath = new Path(harProcessor.getParentFSPath(new Path(p.getSd().getLocation())));
if (fs.exists(ptnPath)){
fs.delete(ptnPath,true);
}
}
throw e;
}
}else{
// no harProcessor, regular operation
// No duplicate partition publish case to worry about because we'll
// get a AlreadyExistsException here if so, and appropriately rollback
client.add_partitions(partitionsToAdd);
partitionsAdded = partitionsToAdd;
if (dynamicPartitioningUsed && (partitionsAdded.size()>0)){
Path src = new Path(ptnRootLocation);
moveTaskOutputs(fs, src, src, tblPath,false);
fs.delete(src, true);
}
}
if( baseCommitter != null ) {
baseCommitter.cleanupJob(context);
}
// cancel the deleg. tokens that were acquired for this job now that
// we are done - we should cancel if the tokens were acquired by
// HCatOutputFormat and not if they were supplied by Oozie. In the latter
// case the HCAT_KEY_TOKEN_SIGNATURE property in the conf will not be set
String tokenStrForm = client.getTokenStrForm();
if(tokenStrForm != null && context.getConfiguration().get
(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
client.cancelDelegationToken(tokenStrForm);
}
if (harProcessor.isEnabled()){
String jcTokenStrForm =
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
String jcTokenSignature =
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
if(jcTokenStrForm != null && jcTokenSignature != null) {
HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
}
}
} catch (Exception e) {
if( partitionsAdded.size() > 0 ) {
try {
//baseCommitter.cleanupJob failed, try to clean up the metastore
for (Partition p : partitionsAdded){
client.dropPartition(tableInfo.getDatabaseName(),
tableInfo.getTableName(), p.getValues());
}
} catch(Exception te) {
//Keep cause as the original exception
throw new HCatException(ErrorType.ERROR_PUBLISHING_PARTITION, e);
}
}
if( e instanceof HCatException ) {
throw (HCatException) e;
} else {
throw new HCatException(ErrorType.ERROR_PUBLISHING_PARTITION, e);
}
} finally {
if( client != null ) {
client.close();
}
}
}