innerPlan, sortKeyInfos, secondarySortKeyInfo);
try {
innerPlanDiscover.process();
} catch (FrontendException e) {
int errorCode = 2213;
throw new VisitorException("Error visiting inner plan for ForEach", errorCode, e);
}
secondarySortKeyInfo = innerPlanDiscover.getSecondarySortKeyInfo();
if (innerPlanDiscover.getSortsToRemove() != null) {
for (POSort sort : innerPlanDiscover.getSortsToRemove()) {
sortsToRemove.add(new POToChange(sort, innerPlan, foreach));
}
}
if (innerPlanDiscover.getDistinctsToChange() != null) {
for (PODistinct distinct : innerPlanDiscover
.getDistinctsToChange()) {
distinctsToChange.add(new POToChange(distinct, innerPlan,
foreach));
}
}
}
try {
// Change PODistinct to use POSortedDistinct, which assume the input
// data is sorted
for (POToChange distinctToChange : distinctsToChange) {
numDistinctChanged++;
PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
String scope = oldDistinct.getOperatorKey().scope;
POSortedDistinct newDistinct = new POSortedDistinct(
new OperatorKey(scope, NodeIdGenerator.getGenerator()
.getNextNodeId(scope)), oldDistinct
.getRequestedParallelism(), oldDistinct
.getInputs());
newDistinct.setInputs(oldDistinct.getInputs());
newDistinct.setResultType(oldDistinct.getResultType());
distinctToChange.plan.replace(oldDistinct, newDistinct);
distinctToChange.forEach.getLeaves();
}
// Removed POSort, if the successor require a databag, we need to
// add a PORelationToExprProject
// to convert tuples into databag
for (POToChange sortToRemove : sortsToRemove) {
numSortRemoved++;
POSort oldSort = (POSort) sortToRemove.oper;
String scope = oldSort.getOperatorKey().scope;
List<PhysicalOperator> preds = sortToRemove.plan
.getPredecessors(sortToRemove.oper);
List<PhysicalOperator> succs = sortToRemove.plan
.getSuccessors(sortToRemove.oper);
POProject project = null;
if ((preds == null
|| preds.get(0).getResultType() != DataType.BAG
&& oldSort.getResultType() == DataType.BAG) // sort to remove do change the result type
&& (succs == null || !(succs.get(0) instanceof PORelationToExprProject))) // successor is not PORelationToExprProject
{
project = new PORelationToExprProject(new OperatorKey(
scope, NodeIdGenerator.getGenerator()
.getNextNodeId(scope)), oldSort
.getRequestedParallelism());
project.setInputs(oldSort.getInputs());
project.setResultType(DataType.BAG);
project.setStar(true);
}
if (project == null)
sortToRemove.plan.removeAndReconnect(sortToRemove.oper);
else
sortToRemove.plan.replace(oldSort, project);
sortToRemove.forEach.getLeaves();
}
} catch (PlanException e) {
int errorCode = 2202;
throw new VisitorException(
"Error change distinct/sort to use secondary key optimizer",
errorCode, e);
}
if (secondarySortKeyInfo != null) {
// Adjust POLocalRearrange, POPackage, MapReduceOper to use the
// secondary key
numMRUseSecondaryKey++;
mr.setUseSecondaryKey(true);
mr.setSecondarySortOrder(secondarySortKeyInfo.getAscs());
int indexOfRearrangeToChange = -1;
for (ColumnChainInfo columnChainInfo : secondarySortKeyInfo
.getColumnChains()) {
ColumnInfo currentColumn = columnChainInfo.getColumnInfos()
.get(0);
int index = currentColumn.columns.get(0);
if (indexOfRearrangeToChange == -1)
indexOfRearrangeToChange = index;
else if (indexOfRearrangeToChange != index) {
int errorCode = 2203;
throw new VisitorException("Sort on columns from different inputs.", errorCode);
}
}
if (mapLeaf instanceof POLocalRearrange) {
((POLocalRearrange) mapLeaf).setUseSecondaryKey(true);
setSecondaryPlan(mr.mapPlan, (POLocalRearrange) mapLeaf,
secondarySortKeyInfo);
} else if (mapLeaf instanceof POUnion) {
List<PhysicalOperator> preds = mr.mapPlan
.getPredecessors(mapLeaf);
boolean found = false;
for (PhysicalOperator pred : preds) {
POLocalRearrange rearrange = (POLocalRearrange) pred;
rearrange.setUseSecondaryKey(true);
if (rearrange.getIndex() == indexOfRearrangeToChange) {
// Try to find the POLocalRearrange for the secondary key
found = true;
setSecondaryPlan(mr.mapPlan, rearrange, secondarySortKeyInfo);
}
}
if (!found)
{
int errorCode = 2214;
throw new VisitorException("Cannot find POLocalRearrange to set secondary plan", errorCode);
}
}
POPackage pack = (POPackage) root;
pack.setUseSecondaryKey(true);
}