Package org.apache.pig.newplan

Examples of org.apache.pig.newplan.Operator


            LOLimit limit = (LOLimit) matched.getSources().get(0);

            // Find the next foreach operator.
            List<Operator> preds = currentPlan.getPredecessors(limit);
            Operator pred = preds.get(0);

            if (pred instanceof LOForEach) {
                // We can safely move LOLimit up
                // Get operator before LOForEach
                Operator prepredecessor = currentPlan.getPredecessors(pred)
                    .get(0);
                currentPlan.removeAndReconnect(limit);
                currentPlan.insertBetween(prepredecessor, limit, pred);
            } else if (pred instanceof LOCross || pred instanceof LOUnion) {
                // Limit can be duplicated, and the new instance pushed in front
View Full Code Here


            p = new OperatorSubPlan(currentPlan);
        }
        Iterator<Operator> iter = currentPlan.getOperators();
       
        while(iter.hasNext()) {
            Operator op = iter.next();
            if (op instanceof LOForEach) {
                addOperator(op, p);
            }
        }
       
View Full Code Here

    }
   
    private void clearAnnotation() {
        Iterator<Operator> iter = currentPlan.getOperators();
        while (iter.hasNext()) {
            Operator op = iter.next();
            op.removeAnnotation(INPUTUIDS);
            op.removeAnnotation(OUTPUTUIDS);
            op.removeAnnotation(REQUIREDCOLS);
        }
    }
View Full Code Here

        private OperatorSubPlan subPlan;

        @Override
        public boolean check(OperatorPlan matched) throws FrontendException {  
            // check if it is inner join
            Operator current = matched.getSources().get(0);
           
            Operator pred = findNonFilterPredecessor( current );
            if( pred == null )
                return false;
           
            // sort, distinct, or sort by is always okay.
            if( pred instanceof LOSort || pred instanceof LODistinct || pred instanceof LOUnion ) {
View Full Code Here

        Set<Long> collectUidFromExpPlan(LogicalExpressionPlan filterPlan) throws FrontendException {
            Set<Long> uids = new HashSet<Long>();
            Iterator<Operator> iter = filterPlan.getOperators();
            while(iter.hasNext()) {
                Operator op = iter.next();
                if (op instanceof ProjectExpression) {
                    long uid = ((ProjectExpression)op).getFieldSchema().uid;
                    uids.add(uid);
                }
            }
View Full Code Here

        /**
         * Starting from current operator (which is a filter), search its successors until
         * locating a non-filter operator. Null is returned if none is found.
         */
        private Operator findNonFilterPredecessor(Operator current) {
            Operator op = current;
            do {
                List<Operator> predecessors = currentPlan.getPredecessors( op );

                // if there are no predecessors return false
                if( predecessors == null || predecessors.size() == 0 ) {
                    return null;
                }

                Operator pred = predecessors.get( 0 );
                if( pred instanceof LOFilter ) {
                    op = pred;
                    continue;
                } else {
                    return pred;
View Full Code Here

            subPlan = new OperatorSubPlan(currentPlan);

            LOFilter filter = (LOFilter)matched.getSources().get(0);
           
            // This is the one that we will insert filter btwn it and it's input.
            Operator predecessor = this.findNonFilterPredecessor( filter );
            subPlan.add( predecessor) ;
           
            // Disconnect the filter in the plan without removing it from the plan.
            Operator predec = currentPlan.getPredecessors( filter ).get( 0 );
            Operator succed;
           
            if (currentPlan.getSuccessors(filter)!=null)
                succed = currentPlan.getSuccessors(filter).get(0);
            else
                succed = null;
           
            Pair<Integer, Integer> p1 = currentPlan.disconnect(predec, filter);
            if (succed!=null) {
                subPlan.add(succed);
                Pair<Integer, Integer> p2 = currentPlan.disconnect(filter, succed);
                currentPlan.connect(predec, p1.first, succed, p2.second);
            }
           
            if( predecessor instanceof LOSort || predecessor instanceof LODistinct ||
                ( predecessor instanceof LOCogroup && currentPlan.getPredecessors( predecessor ).size() == 1 ) ) {
                // For sort, put the filter in front of it.
                Operator prev = currentPlan.getPredecessors( predecessor ).get( 0 );
               
                insertFilter( prev, predecessor, filter );
                return;
            }

            // Find the predecessor of join that contains all required uids.
            LogicalExpressionPlan filterPlan = filter.getFilterPlan();
            List<Operator> preds = currentPlan.getPredecessors( predecessor );
            Map<Integer, Operator> inputs = findInputsToAddFilter( filterPlan, predecessor, preds );
           
            LOFilter newFilter = null;               
            for( Entry<Integer, Operator> entry : inputs.entrySet() ) {
                int inputIndex = entry.getKey();
                Operator pred = entry.getValue();
               
                // Find projection field offset
                int columnOffset = 0;
                if( predecessor instanceof LOJoin || predecessor instanceof LOCross ) {
                    for( int i = 0; i < inputIndex; i++ ) {
                        columnOffset += ( (LogicalRelationalOperator)preds.get( i ) ).getSchema().size();
                    }
                }
               
                // Reuse the filter for the first match. For others, need to make a copy of the filter
                // and add it between input and predecessor.
                newFilter = newFilter == null ? filter : new LOFilter( (LogicalPlan)currentPlan );
               
                currentPlan.add( newFilter );
                subPlan.add( newFilter );
                subPlan.add( pred );
                LogicalExpressionPlan fPlan = filterPlan.deepCopy();
                List<Operator> sinks = fPlan.getSinks();
                List<ProjectExpression> projExprs = new ArrayList<ProjectExpression>();
                for( Operator sink : sinks ) {
                    if( sink instanceof ProjectExpression )
                        projExprs.add( (ProjectExpression)sink );
                }
               
                if( predecessor instanceof LOCogroup ) {
                    for( ProjectExpression projExpr : projExprs ) {
                        // Need to merge filter condition and cogroup by expression;
                        LogicalExpressionPlan plan = ((LOCogroup) predecessor).getExpressionPlans().get( inputIndex ).iterator().next();
                        LogicalExpressionPlan copy = plan.deepCopy();
                        LogicalExpression root = (LogicalExpression)copy.getSinks().get( 0 );
                        List<Operator> predecessors = fPlan.getPredecessors( projExpr );
                        if( predecessors == null || predecessors.size() == 0 ) {
                            fPlan.remove( projExpr );
                            fPlan.add( root );
                        } else {
                            fPlan.add( root );
                            Operator pred1 = predecessors.get( 0 );
                            Pair<Integer, Integer> pair = fPlan.disconnect( pred1, projExpr );
                            fPlan.connect( pred1, pair.first, root, pair.second );
                            fPlan.remove( projExpr );
                        }
                    }
View Full Code Here

            // Find the predecessor of join that contains all required uids.
            for(int j=0; j<preds.size(); j++) {
                // Filter can push to LOJoin outer branch, but no inner branch
                if( hasAll((LogicalRelationalOperator)preds.get(j), uids) &&
                        (predecessor instanceof LOCross || predecessor instanceof LOJoin && (isInner || innerFlags[j]))) {
                    Operator input = preds.get(j);  
                    subPlan.add(input);
                    inputs.put( j, input );
                }
            }
            return  inputs;
View Full Code Here

        }

        @Override
        public void visit(LOStore store) throws FrontendException {
           
            Operator storePred = store.getPlan().getPredecessors(store).get(0);
            if(storePred == null){
                int errCode = 2051;
                String msg = "Did not find a predecessor for Store." ;
                throw new FrontendException(msg, errCode, PigException.BUG);   
            }
           
            SortInfo sortInfo = null;
            if(storePred instanceof LOLimit) {
                storePred = store.getPlan().getPredecessors(storePred).get(0);
            } else if (storePred instanceof LOSplitOutput) {
                LOSplitOutput splitOutput = (LOSplitOutput)storePred;
                // We assume this is the LOSplitOutput we injected for this case:
                // b = order a by $0; store b into '1'; store b into '2';
                // In this case, we should mark both '1' and '2' as sorted
                LogicalExpressionPlan conditionPlan = splitOutput.getFilterPlan();
                if (conditionPlan.getSinks().size()==1) {
                    Operator root = conditionPlan.getSinks().get(0);
                    if (root instanceof ConstantExpression) {
                        Object value = ((ConstantExpression)root).getValue();
                        if (value instanceof Boolean && (Boolean)value==true) {
                            Operator split = splitOutput.getPlan().getPredecessors(splitOutput).get(0);
                            if (split instanceof LOSplit)
                                storePred = store.getPlan().getPredecessors(split).get(0);
                        }
                    }
                }
View Full Code Here

        @Override
        public void transform(OperatorPlan plan) throws FrontendException {
            Iterator<Operator> iter = plan.getOperators();
            while (iter.hasNext()) {
                Operator op = iter.next();
                if (op instanceof LOCogroup) {
                    LOCogroup group = (LOCogroup)op;
                    if(group.getRequestedParallelisam() > 1){
                        log.warn("Resetting parallism to 1 for the group/cogroup " +
                                group.getAlias() +
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.Operator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.