Schema loadSchema = load.getSchema();
for (int i=0;i<loaderRequiredFields.size();i++)
{
Pair<Integer, Integer> pair = loaderRequiredFields.getField(i);
MapKeysInfo mapKeysInfo = loaderRequiredFields.getMapKeysInfo(i);
RequiredField requiredField = new RequiredField();
requiredField.setIndex(pair.second);
requiredField.setAlias(loadSchema.getField(pair.second).alias);
requiredField.setType(loadSchema.getField(pair.second).type);
if (mapKeysInfo!=null && !mapKeysInfo.needAllKeys())
{
List<RequiredField> subFieldList = new ArrayList<RequiredField>();
for (String key : mapKeysInfo.getKeys())
{
RequiredField mapKeyField = new RequiredField();
mapKeyField.setIndex(-1);
mapKeyField.setType(DataType.UNKNOWN);
mapKeyField.setAlias(key);
subFieldList.add(mapKeyField);
}
requiredField.setSubFields(subFieldList);
}
// Sort requiredFieldList, loader expect required field list sorted by index
int j=0;
while (requiredFieldList.getFields().size()>j && requiredFieldList.getFields().get(j).getIndex()<pair.second)
j++;
requiredFieldList.getFields().add(j, requiredField);
}
boolean[] columnRequired = new boolean[load.getSchema().size()];
RequiredFieldResponse response = null;
try {
response = load.pushProjection(requiredFieldList);
} catch (FrontendException e) {
log.warn("fieldsToRead on "+load+" throw an exception, skip it");
}
// If the request is not granted, probably the loader support position prune only,
// and do not prune map key pruning (such as PigStorage). Drop all map keys (means
// we do not prune map keys) and try again
if (response==null || !response.getRequiredFieldResponse())
{
for (RequiredField rf : requiredFieldList.getFields())
{
if (rf.getType() == DataType.MAP)
rf.setSubFields(null);
}
try {
response = load.pushProjection(requiredFieldList);
} catch (FrontendException e) {
log.warn("fieldsToRead on "+load+" throw an exception, skip it");
}
}
// Loader does not support column pruning, insert foreach
LogicalOperator forEach = null;
if (response==null || !response.getRequiredFieldResponse())
{
List<Integer> columnsToProject = new ArrayList<Integer>();
for (RequiredField rf : requiredFieldList.getFields())
columnsToProject.add(rf.getIndex());
forEach = load.insertPlainForEachAfter(columnsToProject);
}
// Begin to prune
for (Pair<Integer, Integer> pair: loaderRequiredFields.getFields())
columnRequired[pair.second] = true;
List<Pair<Integer, Integer>> pruneList = new ArrayList<Pair<Integer, Integer>>();
for (int i=0;i<columnRequired.length;i++)
{
if (!columnRequired[i])
pruneList.add(new Pair<Integer, Integer>(0, i));
}
StringBuffer message = new StringBuffer();
if (pruneList.size()!=0)
{
if (forEach == null)
pruner.addPruneMap(load, pruneList);
else
pruner.addPruneMap(forEach, pruneList);
message.append("Columns pruned for " + load.getAlias() + ": ");
for (int i=0;i<pruneList.size();i++)
{
message.append("$"+pruneList.get(i).second);
if (i!=pruneList.size()-1)
message.append(", ");
}
log.info(message);
}
else
log.info("No column pruned for " + load.getAlias());
message = new StringBuffer();;
for (RequiredField rf : requiredFieldList.getFields())
{
if (rf.getSubFields()!=null)
{
message.append("Map key required for " + load.getAlias()+": ");
if (rf.getIndex()!=-1)
message.append("$"+rf.getIndex());
else
message.append(rf.getAlias());
message.append("->[");
for (int i=0;i<rf.getSubFields().size();i++)
{
RequiredField keyrf = rf.getSubFields().get(i);
message.append(keyrf);
if (i!=rf.getSubFields().size()-1)
message.append(",");
}
message.append("] ");