* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

//Synthetic data generation is done by making constraint tuples for each operator as we traverse the plan

//and try to replace the constraints with values as far as possible. We only deal with simple conditions right now

    Map<LOLoad, DataBag> baseData = null;

    Map<LOLoad, DataBag> newBaseData = new HashMap<LOLoad, DataBag>();

    Map<Operator, DataBag> derivedData = null;

    private boolean limit = false;

    private final Map<Operator, PhysicalOperator> logToPhysMap;

    private Map<LOLimit, Long> oriLimitMap;

    Map<Operator, DataBag> outputConstraintsMap = new HashMap<Operator, DataBag>();

    Log log = LogFactory.getLog(getClass());

    // Augmentation moves from the leaves to root and hence needs a

    // depthfirstwalker

    public AugmentBaseDataVisitor(OperatorPlan plan,

            Map<Operator, PhysicalOperator> logToPhysMap,

            Map<LOLoad, DataBag> baseData,

            Map<Operator, DataBag> derivedData) throws FrontendException {

        super(plan, new PreOrderDepthFirstWalker(

                plan));

        this.baseData = baseData;

        this.derivedData = derivedData;

        this.logToPhysMap = logToPhysMap;

    }

    public void setLimit() {

        limit = true;

    }

    public Map<LOLoad, DataBag> getNewBaseData() throws ExecException {

        // consolidate base data from different LOADs on the same inputs

        MultiMap<FileSpec, DataBag> inputDataMap = new MultiMap<FileSpec, DataBag>();

        for (Map.Entry<LOLoad, DataBag> e : newBaseData.entrySet()) {

            inputDataMap.put(e.getKey().getFileSpec(), e.getValue());

        }

        int index = 0;

        for (FileSpec fs : inputDataMap.keySet()) {

            int maxSchemaSize = 0;

            Tuple tupleOfMaxSchemaSize = null;

            for (DataBag bag : inputDataMap.get(fs)) {

                if (bag.size() > 0) {

                    int size = 0;

                    Tuple t = null;

                    t = bag.iterator().next();

                    size = t.size();

                    if (size > maxSchemaSize) {

                        maxSchemaSize = size;

                        tupleOfMaxSchemaSize = t;

                    }

                }

            }

            for (DataBag bag : inputDataMap.get(fs)) {

                if (bag.size() > 0) {

                    for (Iterator<Tuple> it = bag.iterator(); it.hasNext();) {

                        Tuple t = it.next();

                        for (int i = t.size(); i < maxSchemaSize; ++i) {

                            t.append(tupleOfMaxSchemaSize.get(i));

                        }

                    }

                }

            }

            index++;

        }

        for (Map.Entry<LOLoad, DataBag> e : baseData.entrySet()) {

            DataBag bag = newBaseData.get(e.getKey());

            if (bag == null) {

                bag = BagFactory.getInstance().newDefaultBag();

                newBaseData.put(e.getKey(), bag);

            }

            bag.addAll(e.getValue());

        }

        return newBaseData;

    }

    public Map<LOLimit, Long> getOriLimitMap() {

        return oriLimitMap;

    }

    @Override

    public void visit(LOCogroup cg) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        // we first get the outputconstraints for the current cogroup

        DataBag outputConstraints = outputConstraintsMap.get(cg);

        outputConstraintsMap.remove(cg);

        boolean ableToHandle = true;

        // we then check if we can handle this cogroup and try to collect some

        // information about grouping

        List<List<Integer>> groupSpecs = new LinkedList<List<Integer>>();

        int numCols = -1;

        for (int index = 0; index < cg.getInputs((LogicalPlan)plan).size(); ++index) {

            Collection<LogicalExpressionPlan> groupByPlans =

                cg.getExpressionPlans().get(index);

            List<Integer> groupCols = new ArrayList<Integer>();

            for (LogicalExpressionPlan plan : groupByPlans) {

                Operator leaf = plan.getSinks().get(0);

                if (leaf instanceof ProjectExpression) {

                    groupCols.add(Integer.valueOf(((ProjectExpression) leaf).getColNum()));

                } else {

                    ableToHandle = false;

                    break;

                }

            }

            if (numCols == -1) {

                numCols = groupCols.size();

            }

            if (groupCols.size() != groupByPlans.size()

                    || groupCols.size() != numCols) {

                // we came across an unworkable cogroup plan

                break;

            } else {

                groupSpecs.add(groupCols);

            }

        }

        // we should now have some workable data at this point to synthesize

        // tuples

        try {

            if (ableToHandle) {

                // we need to go through the output constraints first

                int numInputs = cg.getInputs((LogicalPlan) plan).size();

                if (outputConstraints != null) {

                    for (Iterator<Tuple> it = outputConstraints.iterator(); it

                            .hasNext();) {

                        Tuple outputConstraint = it.next();

                        Object groupLabel = outputConstraint.get(0);

                        for (int input = 0; input < numInputs; input++) {

                            int numInputFields = ((LogicalRelationalOperator) cg.getInputs((LogicalPlan) plan).get(input))

                                    .getSchema().size();

                            List<Integer> groupCols = groupSpecs.get(input);

                            DataBag output = outputConstraintsMap.get(cg

                                    .getInputs((LogicalPlan) plan).get(input));

                            if (output == null) {

                                output = BagFactory.getInstance()

                                        .newDefaultBag();

                                outputConstraintsMap.put(cg.getInputs((LogicalPlan) plan).get(

                                        input), output);

                            }

                            for (int i = 0; i < 2; i++) {

                                Tuple inputConstraint = GetGroupByInput(

                                        groupLabel, groupCols, numInputFields);

                                if (inputConstraint != null)

                                    output.add(inputConstraint);

                            }

                        }

                    }

                }

                // then, go through all organic data groups and add input

                // constraints to make each group big enough

                DataBag outputData = derivedData.get(cg);

                for (Iterator<Tuple> it = outputData.iterator(); it.hasNext();) {

                    Tuple groupTup = it.next();

                    Object groupLabel = groupTup.get(0);

                    for (int input = 0; input < numInputs; input++) {

                        int numInputFields = ((LogicalRelationalOperator)cg.getInputs((LogicalPlan) plan).get(input))

                                .getSchema().size();

                        List<Integer> groupCols = groupSpecs.get(input);

                        DataBag output = outputConstraintsMap.get(cg

                                .getInputs((LogicalPlan) plan).get(input));

                        if (output == null) {

                            output = BagFactory.getInstance().newDefaultBag();

                            outputConstraintsMap.put(cg.getInputs((LogicalPlan) plan).get(input),

                                    output);

                        }

                        int numTupsToAdd = 2

                                - (int) ((DataBag) groupTup.get(input + 1))

                                        .size();

                        for (int i = 0; i < numTupsToAdd; i++) {

                            Tuple inputConstraint = GetGroupByInput(groupLabel,

                                    groupCols, numInputFields);

                            if (inputConstraint != null)

                                output.add(inputConstraint);

                        }

                    }

                }

            }

        } catch (Exception e) {

            log

                    .error("Error visiting Cogroup during Augmentation phase of Example Generator! "

                            + e.getMessage());

            throw new FrontendException(

                    "Error visiting Cogroup during Augmentation phase of Example Generator! "

                            + e.getMessage());

        }

    }

    @Override

    public void visit(LOJoin join) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        // we first get the outputconstraints for the current cogroup

        DataBag outputConstraints = outputConstraintsMap.get(join);

        outputConstraintsMap.remove(join);

        boolean ableToHandle = true;

        // we then check if we can handle this cogroup and try to collect some

        // information about grouping

        List<List<Integer>> groupSpecs = new LinkedList<List<Integer>>();

        int numCols = -1;

        for (int index = 0; index < join.getInputs((LogicalPlan)plan).size(); ++index) {

            Collection<LogicalExpressionPlan> groupByPlans =

                join.getExpressionPlans().get(index);

            List<Integer> groupCols = new ArrayList<Integer>();

            for (LogicalExpressionPlan plan : groupByPlans) {

                Operator leaf = plan.getSinks().get(0);

                if (leaf instanceof ProjectExpression) {

                    groupCols.add(Integer.valueOf(((ProjectExpression) leaf).getColNum()));

                } else {

                    ableToHandle = false;

                    break;

                }

            }

            if (numCols == -1) {

                numCols = groupCols.size();

            }

            if (groupCols.size() != groupByPlans.size()

                    || groupCols.size() != numCols) {

                // we came across an unworkable cogroup plan

                break;

            } else {

                groupSpecs.add(groupCols);

            }

        }

        // we should now have some workable data at this point to synthesize

        // tuples

        try {

            if (ableToHandle) {

                // we need to go through the output constraints first

                int numInputs = join.getInputs((LogicalPlan) plan).size();

                if (outputConstraints != null) {

                    for (Iterator<Tuple> it = outputConstraints.iterator(); it

                            .hasNext();) {

                        Tuple outputConstraint = it.next();

                        for (int input = 0; input < numInputs; input++) {

                            int numInputFields = ((LogicalRelationalOperator) join.getInputs((LogicalPlan) plan).get(input))

                                    .getSchema().size();

                            List<Integer> groupCols = groupSpecs.get(input);

                            DataBag output = outputConstraintsMap.get(join

                                    .getInputs((LogicalPlan) plan).get(input));

                            if (output == null) {

                                output = BagFactory.getInstance()

                                        .newDefaultBag();

                                outputConstraintsMap.put(join.getInputs((LogicalPlan) plan).get(

                                        input), output);

                            }

                            Tuple inputConstraint = GetJoinInput(

                                    outputConstraint, groupCols, numInputFields);

                            if (inputConstraint != null)

                                output.add(inputConstraint);

                        }

                    }

                }

                // then, go through all organic data groups and add input

                // constraints to make each group big enough

                DataBag outputData = derivedData.get(join);

                if (outputData.size() == 0) {

                    DataBag output0 = outputConstraintsMap.get(join.getInputs((LogicalPlan) plan).get(0));

                    if (output0 == null || output0.size() == 0) {

                        output0 = derivedData.get(join.getInputs((LogicalPlan) plan).get(0));

                    }

                    Tuple inputConstraint0 = output0.iterator().next();

                    for (int input = 1; input < numInputs; input++) {

                        DataBag output = outputConstraintsMap.get(join.getInputs((LogicalPlan) plan).get(input));

                        if (output == null)

                        {

                            output = BagFactory.getInstance().newDefaultBag();

                            outputConstraintsMap.put(join.getInputs((LogicalPlan) plan).get(input),

                                    output);

                        }

                        int numInputFields = ((LogicalRelationalOperator)join.getInputs((LogicalPlan) plan).get(input)).getSchema().size();

                        Tuple inputConstraint = GetJoinInput(inputConstraint0, groupSpecs.get(0), groupSpecs.get(input), numInputFields);

                        if (inputConstraint != null)

                            output.add(inputConstraint);

                    }

                }

            }

        } catch (Exception e) {

            log

                    .error("Error visiting Cogroup during Augmentation phase of Example Generator! "

                            + e.getMessage());

            throw new FrontendException(

                    "Error visiting Cogroup during Augmentation phase of Example Generator! "

                            + e.getMessage());

        }

    }

    @Override

    public void visit(LOCross cs) throws FrontendException {

    }

    @Override

    public void visit(LODistinct dt) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(dt);

        outputConstraintsMap.remove(dt);

        DataBag inputConstraints = outputConstraintsMap.get(dt.getInput((LogicalPlan) plan));

        if (inputConstraints == null) {

            inputConstraints = BagFactory.getInstance().newDefaultBag();

            outputConstraintsMap.put(dt.getInput((LogicalPlan) plan), inputConstraints);

        }

        if (outputConstraints != null && outputConstraints.size() > 0) {

            for (Iterator<Tuple> it = outputConstraints.iterator(); it.hasNext();)

            {

                inputConstraints.add(it.next());

            }

        }

        boolean emptyInputConstraints = inputConstraints.size() == 0;

        if (emptyInputConstraints) {

            DataBag inputData = derivedData.get(dt.getInput((LogicalPlan) plan));

            for (Iterator<Tuple> it = inputData.iterator(); it.hasNext();)

            {

                inputConstraints.add(it.next());

            }

        }

        Set<Tuple> distinctSet = new HashSet<Tuple>();

        Iterator<Tuple> it;

        for (it = inputConstraints.iterator(); it.hasNext();) {

            if (!distinctSet.add(it.next()))

                break;

        }

        if (!it.hasNext())

        {

            // no duplicates found: generate one

            if (inputConstraints.size()> 0) {

                Tuple src = ((ExampleTuple)inputConstraints.iterator().next()).toTuple(),

                      tgt = TupleFactory.getInstance().newTuple(src.getAll());

                ExampleTuple inputConstraint = new ExampleTuple(tgt);

                inputConstraint.synthetic = true;

                inputConstraints.add(inputConstraint);

            } else if (emptyInputConstraints)

                inputConstraints.clear();

        }

    }

    @Override

    public void visit(LOFilter filter) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(filter);

        outputConstraintsMap.remove(filter);

        LogicalExpressionPlan filterCond = filter.getFilterPlan();

        DataBag inputConstraints = outputConstraintsMap.get(filter.getInput((LogicalPlan) plan));

        if (inputConstraints == null) {

            inputConstraints = BagFactory.getInstance().newDefaultBag();

            outputConstraintsMap.put(filter.getInput((LogicalPlan) plan), inputConstraints);

        }

        DataBag outputData = derivedData.get(filter);

        DataBag inputData = derivedData.get(filter.getInput((LogicalPlan) plan));

        try {

            if (outputConstraints != null && outputConstraints.size() > 0) { // there

                // 's

                // one

                // or

                // more

                // output

                // constraints

                // ;

                // generate

                // corresponding

                // input

                // constraints

                for (Iterator<Tuple> it = outputConstraints.iterator(); it

                        .hasNext();) {

                    Tuple outputConstraint = it.next();

                    ExampleTuple inputConstraint = GenerateMatchingTuple(

                            outputConstraint, filterCond, false);

                    if (inputConstraint != null)

                        inputConstraints.add(inputConstraint);

                }

            } else if (outputData.size() == 0) { // no output constraints, but

                // output is empty; generate

                // one input that will pass the

                // filter

                ExampleTuple inputConstraint = GenerateMatchingTuple(filter

                        .getSchema(), filterCond, false);

                if (inputConstraint != null)

                    inputConstraints.add(inputConstraint);

            }

            // if necessary, insert a negative example (i.e. a tuple that does

            // not pass the filter)

            if (outputData.size() == inputData.size()) { // all tuples pass the

                // filter; generate one

                // input that will not

                // pass the filter

                ExampleTuple inputConstraint = GenerateMatchingTuple(filter

                        .getSchema(), filterCond, true);

                if (inputConstraint != null)

                    inputConstraints.add(inputConstraint);

            }

        } catch (Exception e) {

            log

                    .error("Error visiting Load during Augmentation phase of Example Generator! "

                            + e.getMessage(), e);

            throw new FrontendException(

                    "Error visiting Load during Augmentation phase of Example Generator! "

                            + e.getMessage(), e);

        }

    }

    @Override

    public void visit(LOForEach forEach) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(forEach);

        outputConstraintsMap.remove(forEach);

        LogicalPlan plan = forEach.getInnerPlan();

        boolean ableToHandle = true;

        List<Integer> cols = new ArrayList<Integer>();

        boolean cast = false;

        if (outputConstraints == null || outputConstraints.size() == 0)

            // we dont have to do anything in this case

            return;

        Operator op = plan.getSinks().get(0);

        if (op instanceof CastExpression) {

                cast = true;

                op = ((CastExpression) op).getExpression();

            }

            if (!(op instanceof ProjectExpression)) {

                ableToHandle = false;

            } else {

                cols.add(Integer.valueOf(((ProjectExpression) op).getColNum()));

            }

        if (ableToHandle) {

            // we can only handle simple projections

            DataBag output = BagFactory.getInstance().newDefaultBag();

            for (Iterator<Tuple> it = outputConstraints.iterator(); it

                    .hasNext();) {

                Tuple outputConstraint = it.next();

                try {

                    Tuple inputConstraint = BackPropConstraint(

                            outputConstraint, cols, ((LogicalRelationalOperator)plan

                                    .getPredecessors(forEach).get(0))

                                    .getSchema(), cast);

                    output.add(inputConstraint);

                } catch (Exception e) {

                    e.printStackTrace();

                    throw new FrontendException(

                            "Operator error during Augmenting Phase in Example Generator "

                                    + e.getMessage());

                }

            }

            outputConstraintsMap.put(plan.getPredecessors(forEach)

                    .get(0), output);

        }

    }

    @Override

    public void visit(LOLoad load) throws FrontendException {

        DataBag inputData = baseData.get(load);

       // check if the inputData exists

        if (inputData == null || inputData.size() == 0) {

            log.error("No (valid) input data found!");

            throw new RuntimeException("No (valid) input data found!");

        }

        DataBag newInputData = newBaseData.get(load);

        if (newInputData == null) {

            newInputData = BagFactory.getInstance().newDefaultBag();

            newBaseData.put(load, newInputData);

        }

        LogicalSchema schema;

        try {

            schema = load.getSchema();

            if (schema == null)

                throw new RuntimeException(

                        "Example Generator requires a schema. Please provide a schema while loading data");

        } catch (FrontendException e) {

            log

                    .error("Error visiting Load during Augmentation phase of Example Generator! "

                            + e.getMessage());

            throw new FrontendException(

                    "Error visiting Load during Augmentation phase of Example Generator! "

                            + e.getMessage());

        }

        Tuple exampleTuple = inputData.iterator().next();

        DataBag outputConstraints = outputConstraintsMap.get(load);

        outputConstraintsMap.remove(load);

        // first of all, we are required to guarantee that there is at least one

        // output tuple

        if (outputConstraints == null || outputConstraints.size() == 0) {

            outputConstraints = BagFactory.getInstance().newDefaultBag();

            outputConstraints.add(TupleFactory.getInstance().newTuple(

                    schema.getFields().size()));

        }

        // create example tuple to steal values from when we encounter

        // "don't care" fields (i.e. null fields)

        System.out.println(exampleTuple.toString());

        // run through output constraints; for each one synthesize a tuple and

        // add it to the base data

        // (while synthesizing individual fields, try to match fields that exist

        // in the real data)

        boolean newInput = false;

        for (Iterator<Tuple> it = outputConstraints.iterator(); it.hasNext();) {

            Tuple outputConstraint = it.next();

            // sanity check:

            if (outputConstraint.size() != schema.getFields().size())

                throw new RuntimeException(

                        "Internal error: incorrect number of fields in constraint tuple.");

            Tuple inputT = TupleFactory.getInstance().newTuple(

                    outputConstraint.size());

            ExampleTuple inputTuple = new ExampleTuple(inputT);

            try {

                for (int i = 0; i < inputTuple.size(); i++) {

                    Object d = outputConstraint.get(i);

                    if (d == null && i < exampleTuple.size())

                        d = exampleTuple.get(i);

                    inputTuple.set(i, d);

                }

                if (outputConstraint instanceof ExampleTuple)

                    inputTuple.synthetic = ((ExampleTuple) outputConstraint).synthetic;

                else

                    // raw tuple should have been synthesized

                    inputTuple.synthetic = true;

            } catch (ExecException e) {

                log

                        .error("Error visiting Load during Augmentation phase of Example Generator! "

                                + e.getMessage());

                throw new FrontendException(

                        "Error visiting Load during Augmentation phase of Example Generator! "

                                + e.getMessage());

            }

            try {

                if (inputTuple.synthetic || !inInput(inputTuple, inputData, schema))

                {

                    inputTuple.synthetic = true;

                    newInputData.add(inputTuple);

                    if (!newInput)

                        newInput = true;

                }

            } catch (ExecException e) {

                throw new FrontendException(

                  "Error visiting Load during Augmentation phase of Example Generator! "

                          + e.getMessage());

            }

        }

    }

    private boolean inInput(Tuple newTuple, DataBag input, LogicalSchema schema) throws ExecException {

        boolean result;

        for (Iterator<Tuple> iter = input.iterator(); iter.hasNext();) {

            result = true;

            Tuple tmp = iter.next();

            for (int i = 0; i < schema.size(); ++i)

                if (!newTuple.get(i).equals(tmp.get(i)))

                {

                    result = false;

                    break;

                }

            if (result)

                return true;

        }

        return false;

    }

    @Override

    public void visit(LOSort s) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(s);

        outputConstraintsMap.remove(s);

        if (outputConstraints == null)

            outputConstraintsMap.put(s.getInput((LogicalPlan) plan), BagFactory.getInstance()

                    .newDefaultBag());

        else

            outputConstraintsMap.put(s.getInput((LogicalPlan) plan), outputConstraints);

    }

    @Override

    public void visit(LOSplit split) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

          return;

    }

    @Override

    public void visit(LOStore store) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(store);

        if (outputConstraints == null) {

            outputConstraintsMap.put(plan.getPredecessors(store)

                    .get(0), BagFactory.getInstance().newDefaultBag());

        } else {

            outputConstraintsMap.remove(store);

            outputConstraintsMap.put(plan.getPredecessors(store)

                    .get(0), outputConstraints);

        }

    }

    @Override

    public void visit(LOUnion u) throws FrontendException {

        if (limit && !((PreOrderDepthFirstWalker) currentWalker).getBranchFlag())

            return;

        DataBag outputConstraints = outputConstraintsMap.get(u);

        outputConstraintsMap.remove(u);

        if (outputConstraints == null || outputConstraints.size() == 0) {

            // we dont need to do anything

            // we just find the inputs, create empty bags as their

            // outputConstraints and return

            for (Operator op : u.getInputs((LogicalPlan) plan)) {

                DataBag constraints = BagFactory.getInstance().newDefaultBag();

                outputConstraintsMap.put(op, constraints);

            }

            return;

        }

        // since we have some outputConstraints, we apply them to the inputs

        // round-robin

        int count = 0;

        List<Operator> inputs = u.getInputs(((LogicalPlan) plan));

        int noInputs = inputs.size();

        for (Operator op : inputs) {

            DataBag constraint = BagFactory.getInstance().newDefaultBag();

            outputConstraintsMap.put(op, constraint);

        }

        for (Iterator<Tuple> it = outputConstraints.iterator(); it.hasNext();) {

            DataBag constraint = outputConstraintsMap.get(inputs.get(count));

            constraint.add(it.next());

            count = (count + 1) % noInputs;

        }

    }

    @Override

    public void visit(LOLimit lm) throws FrontendException {

        if (!limit) // not augment for LIMIT in this traversal

            return;

        if (oriLimitMap == null)

            oriLimitMap = new HashMap<LOLimit, Long>();

        DataBag outputConstraints = outputConstraintsMap.get(lm);

        outputConstraintsMap.remove(lm);

        DataBag inputConstraints = outputConstraintsMap.get(lm.getInput((LogicalPlan) plan));

        if (inputConstraints == null) {

            inputConstraints = BagFactory.getInstance().newDefaultBag();

            outputConstraintsMap.put(lm.getInput((LogicalPlan) plan), inputConstraints);

        }

        DataBag inputData = derivedData.get(lm.getInput((LogicalPlan) plan));

        if (outputConstraints != null && outputConstraints.size() > 0) { // there

            // 's

            // one

            // or

            // more

            // output

            // constraints

            // ;

            // generate

            // corresponding

            // input

            // constraints

            for (Iterator<Tuple> it = outputConstraints.iterator(); it

                  .hasNext();) {

                inputConstraints.add(it.next());

             // ... plus one more if only one

             if (inputConstraints.size() == 1) {

                inputConstraints.add(inputData.iterator().next());

                ((PreOrderDepthFirstWalker) currentWalker).setBranchFlag();

             }

          }

        } else if (inputConstraints.size() == 0){

            // add all input to input constraints ...

            inputConstraints.addAll(inputData);

            // ... plus one more if only one

            if (inputConstraints.size() == 1) {

                inputConstraints.add(inputData.iterator().next());

                ((PreOrderDepthFirstWalker) currentWalker).setBranchFlag();

            }

        }

        POLimit poLimit = (POLimit) logToPhysMap.get(lm);

        oriLimitMap.put(lm, Long.valueOf(poLimit.getLimit()));

        poLimit.setLimit(inputConstraints.size()-1);

        lm.setLimit(poLimit.getLimit());

    }

    Tuple GetGroupByInput(Object groupLabel, List<Integer> groupCols,

            int numFields) throws ExecException {

        Tuple t = TupleFactory.getInstance().newTuple(numFields);

        if (groupCols.size() == 1) {

            // GroupLabel would be a data atom

            t.set(groupCols.get(0), groupLabel);

        } else {

            if (!(groupLabel instanceof Tuple))

                throw new RuntimeException("Unrecognized group label!");

            Tuple group = (Tuple) groupLabel;

            for (int i = 0; i < groupCols.size(); i++) {

                t.set(groupCols.get(i), group.get(i));

            }

        }

        return t;

    }

    Tuple GetJoinInput(Tuple group, List<Integer> groupCols0, List<Integer> groupCols,

        int numFields) throws ExecException {

        Tuple t = TupleFactory.getInstance().newTuple(numFields);

        if (groupCols.size() == 1) {

            // GroupLabel would be a data atom

            t.set(groupCols.get(0), group.get(groupCols0.get(0)));

        } else {

            if (!(group instanceof Tuple))

                throw new RuntimeException("Unrecognized group label!");

            for (int i = 0; i < groupCols.size(); i++) {

                t.set(groupCols.get(i), group.get(groupCols0.get(i)));

            }

        }

        return t;

    }

    Tuple GetJoinInput(Tuple group, List<Integer> groupCols,

        int numFields) throws ExecException {

        Tuple t = TupleFactory.getInstance().newTuple(numFields);

        if (groupCols.size() == 1) {

            // GroupLabel would be a data atom

            t.set(groupCols.get(0), group);

        } else {

            if (!(group instanceof Tuple))

                throw new RuntimeException("Unrecognized group label!");

            for (int i = 0; i < groupCols.size(); i++) {

                t.set(groupCols.get(i), group.get(i));

            }

        }

        return t;

    }

    Tuple BackPropConstraint(Tuple outputConstraint, List<Integer> cols,

            LogicalSchema inputSchema, boolean cast) throws ExecException {

        Tuple inputConst = TupleFactory.getInstance().newTuple(

                inputSchema.getFields().size());

        Tuple inputConstraint = new ExampleTuple(inputConst);

        for (int outCol = 0; outCol < outputConstraint.size(); outCol++) {

            int inCol = cols.get(outCol);

            Object outVal = outputConstraint.get(outCol);

            Object inVal = inputConstraint.get(inCol);

            if (inVal == null && outVal != null) {

                // inputConstraint.set(inCol, outVal);

                inputConstraint.set(inCol, (cast) ? new DataByteArray(outVal

                        .toString().getBytes()) : outVal);

            } else {

                if (outVal != null) {

                    // unable to back-propagate, due to conflicting column

                    // constraints, so give up

                    return null;

                }

            }

        }

        return inputConstraint;

    }

    // generate a constraint tuple that conforms to the schema and passes the

    // predicate

    // (or null if unable to find such a tuple)

    ExampleTuple GenerateMatchingTuple(LogicalSchema schema, LogicalExpressionPlan plan,

            boolean invert) throws FrontendException, ExecException {

        return GenerateMatchingTuple(TupleFactory.getInstance().newTuple(

                schema.getFields().size()), plan, invert);

    }

    // generate a constraint tuple that conforms to the constraint and passes

    // the predicate

    // (or null if unable to find such a tuple)

    //

    // for now, constraint tuples are tuples whose fields are a blend of actual

    // data values and nulls,

    // where a null stands for "don't care"

    //

    // in the future, may want to replace "don't care" with a more rich

    // constraint language; this would

    // help, e.g. in the case of two filters in a row (you want the downstream

    // filter to tell the upstream filter

    // what predicate it wants satisfied in a given field)

    //

    ExampleTuple GenerateMatchingTuple(Tuple constraint, LogicalExpressionPlan predicate,

            boolean invert) throws ExecException, FrontendException {

        Tuple t = TupleFactory.getInstance().newTuple(constraint.size());

        ExampleTuple tOut = new ExampleTuple(t);

        for (int i = 0; i < t.size(); i++)

            tOut.set(i, constraint.get(i));

        GenerateMatchingTupleHelper(tOut, predicate

                .getSources().get(0), invert);

        tOut.synthetic = true;

        return tOut;

    }

    void GenerateMatchingTupleHelper(Tuple t, Operator pred,

            boolean invert) throws FrontendException, ExecException {

        if (pred instanceof BinaryExpression)

            GenerateMatchingTupleHelper(t, (BinaryExpression) pred,

                    invert);

        else if (pred instanceof NotExpression)

            GenerateMatchingTupleHelper(t, (NotExpression) pred, invert);

        else if (pred instanceof IsNullExpression)

            GenerateMatchingTupleHelper(t, (IsNullExpression) pred, invert);

        else if (pred instanceof UserFuncExpression)

            // Don't know how to generate input tuple for UDF, return null

            // to suppress the generation

            t = null;

        else

            throw new FrontendException("Unknown operator in filter predicate");

    }

    void GenerateMatchingTupleHelper(Tuple t, BinaryExpression pred,

            boolean invert) throws FrontendException, ExecException {

        if (pred instanceof AndExpression) {

            GenerateMatchingTupleHelper(t, (AndExpression) pred, invert);

            return;

        } else if (pred instanceof OrExpression) {

            GenerateMatchingTupleHelper(t, (OrExpression) pred, invert);

            return;

        }

        // now we are sure that the expression operators are the roots of the

        // plan

        boolean leftIsConst = false, rightIsConst = false;

        Object leftConst = null, rightConst = null;

        byte leftDataType = 0, rightDataType = 0;

        int leftCol = -1, rightCol = -1;

        if (pred instanceof AddExpression || pred instanceof SubtractExpression

                || pred instanceof MultiplyExpression || pred instanceof DivideExpression

                || pred instanceof ModExpression || pred instanceof RegexExpression)

            return; // We don't try to work around these operators right now

        if (pred.getLhs() instanceof ConstantExpression) {

            leftIsConst = true;

            leftConst = ((ConstantExpression) (pred.getLhs())).getValue();

        } else {

            LogicalExpression lhs = pred.getLhs();

            if (lhs instanceof CastExpression)

                lhs = ((CastExpression) lhs).getExpression();

            // if (!(pred.getLhsOperand() instanceof ProjectExpression && ((ProjectExpression)

            // pred

            // .getLhsOperand()).getProjection().size() == 1))

            // return; // too hard

            if (!(lhs instanceof ProjectExpression))

                return;

            leftCol = ((ProjectExpression) lhs).getColNum();

            leftDataType = ((ProjectExpression) lhs).getType();

            Object d = t.get(leftCol);

            if (d != null) {

                leftIsConst = true;

                leftConst = d;

            }

        }

        if (pred.getRhs() instanceof ConstantExpression) {

            rightIsConst = true;

            rightConst = ((ConstantExpression) (pred.getRhs())).getValue();

        } else {

            Operator rhs = pred.getRhs();

            if (rhs instanceof CastExpression)

                rhs = ((CastExpression) rhs).getExpression();

            // if (!(pred.getRhsOperand() instanceof ProjectExpression && ((ProjectExpression)

            // pred

            // .getRhsOperand()).getProjection().size() == 1))

            // return; // too hard

            if (!(rhs instanceof ProjectExpression))

                return;

            rightCol = ((ProjectExpression) rhs).getColNum();

            rightDataType = ((ProjectExpression) rhs).getType();

            Object d = t.get(rightCol);

            if (d != null) {

                rightIsConst = true;

                rightConst = d;

            }

        }

        if (leftIsConst && rightIsConst)

            return; // can't really change the result if both are constants

        // now we try to change some nulls to constants

        // convert some nulls to constants

        if (!invert) {

            if (pred instanceof EqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType, leftConst

                            .toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, rightConst

                            .toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "0"));

                }

            } else if (pred instanceof NotEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType,

                            GetUnequalValue(leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetUnequalValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "1"));

                }

            } else if (pred instanceof GreaterThanExpression

                    || pred instanceof GreaterThanEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType,

                            GetSmallerValue(leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetLargerValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "1"));

                    t.set(rightCol, generateData(rightDataType, "0"));

                }

            } else if (pred instanceof LessThanExpression

                    || pred instanceof LessThanEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType, GetLargerValue(

                            leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetSmallerValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "1"));

                }

            }

        } else {

            if (pred instanceof EqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType,

                            GetUnequalValue(leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetUnequalValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "1"));

                }

            } else if (pred instanceof NotEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType, leftConst

                            .toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, rightConst

                            .toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "0"));

                }

            } else if (pred instanceof GreaterThanExpression

                    || pred instanceof GreaterThanEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType, GetLargerValue(

                            leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetSmallerValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "0"));

                    t.set(rightCol, generateData(rightDataType, "1"));

                }

            } else if (pred instanceof LessThanExpression

                    || pred instanceof LessThanEqualExpression) {

                if (leftIsConst) {

                    t.set(rightCol, generateData(rightDataType,

                            GetSmallerValue(leftConst).toString()));

                } else if (rightIsConst) {

                    t.set(leftCol, generateData(leftDataType, GetLargerValue(

                            rightConst).toString()));

                } else {

                    t.set(leftCol, generateData(leftDataType, "1"));

                    t.set(rightCol, generateData(rightDataType, "0"));

                }

            }

        }

    }

    void GenerateMatchingTupleHelper(Tuple t, AndExpression op, boolean invert)

            throws FrontendException, ExecException {

        Operator input = op.getLhs();

        GenerateMatchingTupleHelper(t, input, invert);

        input = op.getRhs();

        GenerateMatchingTupleHelper(t, input, invert);

    }

    void GenerateMatchingTupleHelper(Tuple t, OrExpression op, boolean invert)

            throws FrontendException, ExecException {

        Operator input = op.getLhs();

        GenerateMatchingTupleHelper(t, input, invert);

        input = op.getRhs();

        GenerateMatchingTupleHelper(t, input, invert);

    }

    void GenerateMatchingTupleHelper(Tuple t, NotExpression op, boolean invert)

            throws FrontendException, ExecException {

        LogicalExpression input = op.getExpression();

        GenerateMatchingTupleHelper(t, input, !invert);

    }

    void GenerateMatchingTupleHelper(Tuple t, IsNullExpression op, boolean invert)

            throws FrontendException, ExecException {

        byte type = op.getExpression().getType();

        if (!invert)

            t.set(0, null);

        else

            t.set(0, generateData(type, "0"));

    }

    Object GetUnequalValue(Object v) {

        byte type = DataType.findType(v);

        if (type == DataType.BAG || type == DataType.TUPLE

                || type == DataType.MAP)

            return null;

        Object zero = generateData(type, "0");

        if (v.equals(zero))

            return generateData(type, "1");

        return zero;

    }

    Object GetSmallerValue(Object v) {

        byte type = DataType.findType(v);

        if (type == DataType.BAG || type == DataType.TUPLE

                || type == DataType.MAP)

            return null;

        switch (type) {

        case DataType.CHARARRAY:

            String str = (String) v;

            if (str.length() > 0)

                return str.substring(0, str.length() - 1);

            else

                return null;

        case DataType.BYTEARRAY:

            DataByteArray data = (DataByteArray) v;

            if (data.size() > 0)

                return new DataByteArray(data.get(), 0, data.size() - 1);

            else

                return null;

        case DataType.INTEGER:

            return Integer.valueOf((Integer) v - 1);

        case DataType.LONG:

            return Long.valueOf((Long) v - 1);

        case DataType.FLOAT:

            return Float.valueOf((Float) v - 1);

        case DataType.DOUBLE:

            return Double.valueOf((Double) v - 1);