Examples of PhysicalOperator

org.apache.drill.exec.physical.base.PhysicalOperator
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
This is the base class for all operators. This supports a generic way of processing inputs which can be overridden by operators extending this class. The input model assumes that it can either be taken from an operator or can be attached directly to this operator. Also it is assumed that inputs to an operator are always in the form of a tuple. For this pipeline rework, we assume a pull based model, i.e, the root operator is going to call getNext with the appropriate type which initiates a cascade of getNext calls that unroll to create input for the root operator to work on. Any operator that extends the PhysicalOperator, supports a getNext with all the different types of parameter types. The concrete implementation should use the result type of its input operator to decide the type of getNext's parameter. This is done to avoid switch/case based on the type as much as possible. The default is assumed to return an erroneus Result corresponding to an unsupported operation on that type. So the operators need to implement only those types that are supported.
org.apache.pig.impl.physicalLayer.PhysicalOperator

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator


        if (preds == null)
            return;


        for (Operator pred : preds) {
            PhysicalOperator from = logToPhyMap.get(pred);
            currentPlan.createSoftLink(from, logToPhyMap.get(op));
        }
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        // connections in this plan and create equivalent connections in the
        // clone.
        Map<PhysicalOperator, PhysicalOperator> matches = 
            new HashMap<PhysicalOperator, PhysicalOperator>(mOps.size());
        for (PhysicalOperator op : mOps.keySet()) {
            PhysicalOperator c = op.clone();
            clone.add(c);
            matches.put(op, c);
        }


        // Build the edges
        for (PhysicalOperator op : mFromEdges.keySet()) {
            PhysicalOperator cloneFrom = matches.get(op);
            if (cloneFrom == null) {
                String msg = "Unable to find clone for op " + op.name();
                throw new CloneNotSupportedException(msg);
            }
            Collection<PhysicalOperator> toOps = mFromEdges.get(op);
            for (PhysicalOperator toOp : toOps) {
                PhysicalOperator cloneTo = matches.get(toOp);
                if (cloneTo == null) {
                    String msg = "Unable to find clone for op " + toOp.name();
                    throw new CloneNotSupportedException(msg);
                }
                try {
                    clone.connect(cloneFrom, cloneTo);
                } catch (PlanException pe) {
                    CloneNotSupportedException cnse = new CloneNotSupportedException();
                    cnse.initCause(pe);
                    throw cnse;
                }
            }
        }


        // Fix up all the inputs in the operators themselves.
        for (PhysicalOperator op : mOps.keySet()) {
            List<PhysicalOperator> inputs = op.getInputs();
            if (inputs == null || inputs.size() == 0) continue;
            List<PhysicalOperator> newInputs = 
                new ArrayList<PhysicalOperator>(inputs.size());
            PhysicalOperator cloneOp = matches.get(op);
            if (cloneOp == null) {
                String msg = "Unable to find clone for op " + op.name();
                throw new CloneNotSupportedException(msg);
            }
            for (PhysicalOperator iOp : inputs) {
                PhysicalOperator cloneIOp = matches.get(iOp);
                if (cloneIOp == null) {
                    String msg = "Unable to find clone for op " + iOp.name();
                    throw new CloneNotSupportedException(msg);
                }
                newInputs.add(cloneIOp);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        
        assertEquals( 3, phyPlan.size() );
        assertEquals( 1, phyPlan.getRoots().size() );
        assertEquals( 1, phyPlan.getLeaves().size() );
        
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt") );
        
        // Check for Filter
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        assertEquals( 2, filPlan.getRoots().size() );
        assertEquals( 1, filPlan.getLeaves().size() );
        
        PhysicalOperator eq = filPlan.getLeaves().get(0);
        assertEquals( EqualToExpr.class, eq.getClass() );
        
        PhysicalOperator prj1 = filPlan.getRoots().get(0);
        assertEquals( POProject.class, prj1.getClass() );
        assertEquals( 0, ((POProject)prj1).getColumn() );
        PhysicalOperator constExp = filPlan.getRoots().get(1);
        assertEquals( ConstantExpression.class, constExp.getClass() );
        assertEquals( null, ((ConstantExpression)constExp).getValue() );
        
        // Check for Store
        PhysicalOperator stor = phyPlan.getSuccessors(fil).get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty"));
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        PhysicalPlan physicalPlan = translatePlan(newPlan);
        assertEquals(9, physicalPlan.size());
        assertEquals(physicalPlan.getRoots().size(), 2);
        
        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj3 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d1.txt") );
                
        PhysicalOperator LoR1 = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj4 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals(1, ((POProject)prj4).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
        
        PhysicalOperator GoR = (PhysicalOperator)physicalPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );
        
        PhysicalOperator Pack = (PhysicalOperator)physicalPlan.getSuccessors(GoR).get(0);
        assertEquals( POPackage.class, Pack.getClass() );


        // Check for ForEach
        PhysicalOperator ForE = (PhysicalOperator)physicalPlan.getSuccessors(Pack).get(0);
        assertEquals( POForEach.class, ForE.getClass() );
        PhysicalOperator prj5 = ((POForEach)ForE).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj5.getClass() );
        assertEquals( 1, ((POProject)prj5).getColumn() ); 
        PhysicalOperator prj6 = ((POForEach)ForE).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj6.getClass() );
        assertEquals( 2, ((POProject)prj6).getColumn() );
        
        // Filter Operator
        PhysicalOperator fil = (PhysicalOperator)physicalPlan.getSuccessors(ForE).get(0);
        assertEquals( POFilter.class, fil.getClass() );        
        
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        List<PhysicalOperator> filRoots = filPlan.getRoots();
        
        assertEquals( ConstantExpression.class, filRoots.get(1).getClass() );
        ConstantExpression ce1 = (ConstantExpression) filRoots.get(1);
        assertEquals( null, ce1.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots.get(3).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(3);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(0).getClass() );
        POProject prj1 = (POProject) filRoots.get(0);
        assertEquals( 3, prj1.getColumn() );
        assertEquals( POProject.class, filRoots.get(2).getClass() );
        POProject prj2 = (POProject) filRoots.get(2);
        assertEquals( 0, prj2.getColumn() );




        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)physicalPlan.getSuccessors(fil).get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty") );
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        assertEquals(16, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 3);
        assertEquals(phyPlan.getLeaves().size(), 1 );


        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj1 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj1.getClass() );
        assertEquals(0, ((POProject)prj1).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d3.txt") );


        PhysicalOperator LoR1 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj2 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj2.getClass() );
        assertEquals(1, ((POProject)prj2).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
        
        PhysicalOperator GoR = (PhysicalOperator)phyPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );
        
        PhysicalOperator Pack = (PhysicalOperator)phyPlan.getSuccessors(GoR).get(0);
        assertEquals( POPackage.class, Pack.getClass() );
        
        PhysicalOperator LoR2 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(2)).get(0);
        assertEquals( POLocalRearrange.class, LoR2.getClass() );
        POLocalRearrange Lor2 = (POLocalRearrange) LoR2;
        PhysicalOperator prj3 = Lor2.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp3 = Lor2.getInputs().get(0);
        assertEquals( POLoad.class, inp3.getClass() );
        assertTrue(  ((POLoad)inp3).getLFile().getFileName().contains("d1.txt") );
        
        PhysicalOperator GoR2 = (PhysicalOperator)phyPlan.getSuccessors(LoR2).get(0);
        assertEquals( POGlobalRearrange.class, GoR2.getClass() );
        
        PhysicalOperator Pack2 = (PhysicalOperator)phyPlan.getSuccessors(GoR2).get(0);
        assertEquals( POPackage.class, Pack2.getClass() );
        
        // Check for ForEach
        PhysicalOperator ForE = (PhysicalOperator)phyPlan.getSuccessors(Pack).get(0);
        assertEquals( POForEach.class, ForE.getClass() );
        PhysicalOperator prj4 = ((POForEach)ForE).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals( 1, ((POProject)prj4).getColumn() ); 
        PhysicalOperator prj5 = ((POForEach)ForE).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj5.getClass() );
        assertEquals( 2, ((POProject)prj5).getColumn() );
        
        PhysicalOperator ForE2 = (PhysicalOperator)phyPlan.getSuccessors(Pack2).get(0);
        assertEquals( POForEach.class, ForE2.getClass() );
        PhysicalOperator prj6 = ((POForEach)ForE2).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj6.getClass() );
        assertEquals( 1, ((POProject)prj6).getColumn() ); 
        PhysicalOperator prj7 = ((POForEach)ForE2).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj7.getClass() );
        assertEquals( 2, ((POProject)prj7).getColumn() );
        
        // Check Filter Operator
        PhysicalOperator fil = (PhysicalOperator)phyPlan.getSuccessors(ForE).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        List<PhysicalOperator> filRoots = filPlan.getRoots();
        
        assertEquals( ConstantExpression.class, filRoots.get(0).getClass() );
        ConstantExpression ce1 = (ConstantExpression) filRoots.get(0);
        assertEquals( null, ce1.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(2);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(1).getClass() );
        POProject prj8 = (POProject) filRoots.get(1);
        assertEquals( 5, prj8.getColumn() );
        assertEquals( POProject.class, filRoots.get(3).getClass() );
        POProject prj9 = (POProject) filRoots.get(3);
        assertEquals( 2, prj9.getColumn() );
        
        
        PhysicalOperator fil2 = (PhysicalOperator)phyPlan.getSuccessors(ForE2).get(0);
        assertEquals( POFilter.class, fil2.getClass() );
        
        PhysicalOperator LoR3 = (PhysicalOperator)phyPlan.getSuccessors(fil2).get(0);
        assertEquals( POLocalRearrange.class, LoR3.getClass() );
        POLocalRearrange Lor3 = (POLocalRearrange) LoR3;
        PhysicalOperator prj12 = Lor3.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj12.getClass() );
        assertEquals(3, ((POProject)prj12).getColumn() );
        
        PhysicalPlan filPlan2 = ((POFilter)fil2).getPlan();
        List<PhysicalOperator> filRoots2 = filPlan2.getRoots();
        
        assertEquals( ConstantExpression.class, filRoots2.get(0).getClass() );
        ConstantExpression ce3 = (ConstantExpression) filRoots2.get(0);
        assertEquals( null, ce3.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots2.get(2).getClass() );
        ConstantExpression ce4 = (ConstantExpression) filRoots2.get(2);
        assertEquals( null, ce4.getValue() );
        assertEquals( POProject.class, filRoots2.get(1).getClass() );
        POProject prj10 = (POProject) filRoots2.get(1);
        assertEquals( 3, prj10.getColumn() );
        assertEquals( POProject.class, filRoots2.get(3).getClass() );
        POProject prj11 = (POProject) filRoots2.get(3);
        assertEquals( 0, prj11.getColumn() );
        
        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)phyPlan.getLeaves().get(0);
        assertEquals( stor, phyPlan.getSuccessors(fil).get(0));
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty") );
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
        
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
        
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        
        PhysicalOperator equal = filPlan.getLeaves().get(0);
        assertEquals( EqualToExpr.class, equal.getClass() );
        assertEquals( DataType.BOOLEAN, ((EqualToExpr)equal).getResultType() );
        
        PhysicalOperator constExpr = ((EqualToExpr)equal).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
        
        PhysicalOperator castExpr = ((EqualToExpr)equal).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
        
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
        
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );        
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
        
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
        
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        
        PhysicalOperator greaterThan = filPlan.getLeaves().get(0);
        assertEquals( GreaterThanExpr.class, greaterThan.getClass() );
        assertEquals( DataType.BOOLEAN, ((GreaterThanExpr)greaterThan).getResultType() );
        
        PhysicalOperator constExpr = ((GreaterThanExpr)greaterThan).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
        
        PhysicalOperator castExpr = ((GreaterThanExpr)greaterThan).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
        
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
        
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );  
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
        
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
        
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        
        PhysicalOperator lessThan = filPlan.getLeaves().get(0);
        assertEquals( LessThanExpr.class, lessThan.getClass() );
        assertEquals( DataType.BOOLEAN, ((LessThanExpr)lessThan).getResultType() );
        
        PhysicalOperator constExpr = ((LessThanExpr)lessThan).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
        
        PhysicalOperator castExpr = ((LessThanExpr)lessThan).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
        
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
        
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );  
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
        
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
        
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
        
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
        
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.