Examples of OperatorKey

Examples of org.apache.pig.impl.plan.OperatorKey

    
    @Override
    public void visit(LONative loNative) throws FrontendException{     
        String scope = DEFAULT_SCOPE;
        
        PONative poNative = new PONative(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)));
        poNative.setAlias(loNative.getAlias());
        poNative.setNativeMRjar(loNative.getNativeMRJar());
        poNative.setParams(loNative.getParams());
        poNative.setResultType(DataType.BAG);

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

    
    @Override
    public void visit(LOFilter filter) throws FrontendException {
        String scope = DEFAULT_SCOPE;
//        System.err.println("Entering Filter");
        POFilter poFilter = new POFilter(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), filter.getRequestedParallelisam());
        poFilter.setAlias(filter.getAlias());
        poFilter.setResultType(DataType.BAG);
        currentPlan.add(poFilter);
        logToPhyMap.put(filter, poFilter);

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

        currentPlan = currentPlans.pop();


        // get the physical operator for sort
        POSort poSort;
        if (sort.getUserFunc() == null) {
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelisam(), null,
                    sortPlans, sort.getAscendingCols(), null);
        } else {
            POUserComparisonFunc comparator = new POUserComparisonFunc(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), sort
                    .getRequestedParallelisam(), null, sort.getUserFunc());
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelisam(), null,
                    sortPlans, sort.getAscendingCols(), comparator);
        }
        poSort.setAlias(sort.getAlias());
        poSort.setLimit(sort.getLimit());

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

    @Override
    public void visit(LOCross cross) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        List<Operator> inputs = cross.getPlan().getPredecessors(cross);
        
        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                scope, nodeGen.getNextNodeId(scope)), cross
                .getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), cross.getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
        
        int count = 0;
        
        try {
            currentPlan.connect(poGlobal, poPackage);
            List<Boolean> flattenLst = Arrays.asList(true, true);
            
            for (Operator op : inputs) {
                PhysicalPlan fep1 = new PhysicalPlan();
                ConstantExpression ce1 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelisam());
                ce1.setValue(inputs.size());
                ce1.setResultType(DataType.INTEGER);
                fep1.add(ce1);
                
                ConstantExpression ce2 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelisam());
                ce2.setValue(count);
                ce2.setResultType(DataType.INTEGER);
                fep1.add(ce2);
                /*Tuple ce1val = TupleFactory.getInstance().newTuple(2);
                ce1val.set(0,inputs.size());
                ce1val.set(1,count);
                ce1.setValue(ce1val);
                ce1.setResultType(DataType.TUPLE);*/
                
                


                POUserFunc gfc = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelisam(), Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2), new FuncSpec(GFCross.class.getName()));
                gfc.setAlias(cross.getAlias());
                gfc.setResultType(DataType.BAG);
                fep1.addAsLeaf(gfc);
                gfc.setInputs(Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2));
                /*fep1.add(gfc);
                fep1.connect(ce1, gfc);
                fep1.connect(ce2, gfc);*/
                
                PhysicalPlan fep2 = new PhysicalPlan();
                POProject feproj = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam());
                feproj.setAlias(cross.getAlias());
                feproj.setResultType(DataType.TUPLE);
                feproj.setStar(true);
                feproj.setOverloaded(false);
                fep2.add(feproj);
                List<PhysicalPlan> fePlans = Arrays.asList(fep1, fep2);
                
                POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), fePlans, flattenLst );
                fe.setAlias(cross.getAlias());
                currentPlan.add(fe);
                currentPlan.connect(logToPhyMap.get(op), fe);
                
                POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                        scope, nodeGen.getNextNodeId(scope)), cross
                        .getRequestedParallelisam());
                physOp.setAlias(cross.getAlias());
                List<PhysicalPlan> lrPlans = new ArrayList<PhysicalPlan>();
                for(int i=0;i<inputs.size();i++){
                    PhysicalPlan lrp1 = new PhysicalPlan();
                    POProject lrproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), i);
                    lrproj1.setAlias(cross.getAlias());
                    lrproj1.setOverloaded(false);
                    lrproj1.setResultType(DataType.INTEGER);
                    lrp1.add(lrproj1);
                    lrPlans.add(lrp1);
                }
                
                physOp.setCross(true);
                physOp.setIndex(count++);
                physOp.setKeyType(DataType.TUPLE);
                physOp.setPlans(lrPlans);
                physOp.setResultType(DataType.TUPLE);
                
                currentPlan.add(physOp);
                currentPlan.connect(fe, physOp);
                currentPlan.connect(physOp, poGlobal);
            }
        } catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
        } catch (ExecException e) {
            int errCode = 2058;
            String msg = "Unable to set index on newly create POLocalRearrange.";
            throw new VisitorException(msg, errCode, PigException.BUG, e);
        }
        
        poPackage.setKeyType(DataType.TUPLE);
        poPackage.setResultType(DataType.TUPLE);
        poPackage.setNumInps(count);
        boolean inner[] = new boolean[count];
        for (int i=0;i<count;i++) {
            inner[i] = true;
        }
        poPackage.setInner(inner);
        
        List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
        List<Boolean> flattenLst = new ArrayList<Boolean>();
        for(int i=1;i<=count;i++){
            PhysicalPlan fep1 = new PhysicalPlan();
            POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), i);
            feproj1.setAlias(cross.getAlias());
            feproj1.setResultType(DataType.BAG);
            feproj1.setOverloaded(false);
            fep1.add(feproj1);
            fePlans.add(fep1);
            flattenLst.add(true);
        }
        
        POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), fePlans, flattenLst );
        fe.setAlias(cross.getAlias());
        currentPlan.add(fe);
        try{
            currentPlan.connect(poPackage, fe);
        }catch (PlanException e1) {

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

    }
    
    @Override
    public void visit(LOStream stream) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POStream poStream = new POStream(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), stream.getExecutableManager(), 
                stream.getStreamingCommand(), this.pc.getProperties());
        poStream.setAlias(stream.getAlias());
        currentPlan.add(poStream);
        logToPhyMap.put(stream, poStream);

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey


    @Override
    public void visit(LOInnerLoad load) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        
        POProject exprOp = new POProject(new OperatorKey(scope, nodeGen
              .getNextNodeId(scope)));
        
        LogicalSchema s = load.getSchema();


        if (load.sourceIsBag()) {

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

        boolean[] flatten = gen.getFlattenFlags();
        List<Boolean> flattenList = new ArrayList<Boolean>();
        for(boolean fl: flatten) {
            flattenList.add(fl);
        }
        POForEach poFE = new POForEach(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), foreach.getRequestedParallelisam(), innerPlans, flattenList);
        poFE.setAlias(foreach.getAlias());
        poFE.setResultType(DataType.BAG);
        logToPhyMap.put(foreach, poFE);
        currentPlan.add(poFE);

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

    
    @Override
    public void visit(LOStore loStore) throws FrontendException {
        String scope = DEFAULT_SCOPE;
//        System.err.println("Entering Store");
        POStore store = new POStore(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)));
        store.setAlias(((LogicalRelationalOperator)loStore.getPlan().
                getPredecessors(loStore).get(0)).getAlias());
        store.setSFile(loStore.getOutputSpec());
        store.setInputSpec(loStore.getInputSpec());

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

    
    private void translateCollectedCogroup(LOCogroup cg) throws FrontendException {
        // can have only one input
        LogicalRelationalOperator pred = (LogicalRelationalOperator) plan.getPredecessors(cg).get(0);
        List<LogicalExpressionPlan> exprPlans = (List<LogicalExpressionPlan>) cg.getExpressionPlans().get(0);
        POCollectedGroup physOp = new POCollectedGroup(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));
        physOp.setAlias(cg.getAlias());
        List<PhysicalPlan> pExprPlans = translateExpressionPlans(cg, exprPlans);
        
        try {

View Full Code Here

Examples of org.apache.pig.impl.plan.OperatorKey

            Operator op = inputs.get(i);
            PhysicalOperator physOp = logToPhyMap.get(op);
            inpPOs.add(physOp);
            
            List<LogicalExpressionPlan> plans = (List<LogicalExpressionPlan>)innerPlans.get(i);
            POLocalRearrange poInnerLR = new POLocalRearrange(new OperatorKey(DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));
            poInnerLR.setAlias(relationalOp.getAlias());
            // LR will contain list of physical plans, because there could be
            // multiple keys and each key can be an expression.
            List<PhysicalPlan> exprPlans = translateExpressionPlans(relationalOp, plans);
            try {
                poInnerLR.setPlans(exprPlans);
            } catch (PlanException pe) {
                int errCode = 2071;
                String msg = "Problem with setting up local rearrange's plans.";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, pe);
            }
            innerLRs[count] = poInnerLR;
            try {
                poInnerLR.setIndex(count++);
            } catch (ExecException e1) {
                int errCode = 2058;
                String msg = "Unable to set index on newly create POLocalRearrange.";
                throw new VisitorException(msg, errCode, PigException.BUG, e1);
            }
            poInnerLR.setKeyType(plans.size() > 1 ? DataType.TUPLE : 
                        exprPlans.get(0).getLeaves().get(0).getResultType());
            poInnerLR.setResultType(DataType.TUPLE);
        }
        
        POMergeCogroup poCogrp = new POMergeCogroup(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)),inpPOs,innerLRs,relationalOp.getRequestedParallelisam());
        return poCogrp;
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.