Package org.apache.pig.backend.hadoop.executionengine.physicalLayer

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator


        // connections in this plan and create equivalent connections in the
        // clone.
        Map<PhysicalOperator, PhysicalOperator> matches =
            new HashMap<PhysicalOperator, PhysicalOperator>(mOps.size());
        for (PhysicalOperator op : mOps.keySet()) {
            PhysicalOperator c = op.clone();
            clone.add(c);
            matches.put(op, c);
        }

        // Build the edges
        for (PhysicalOperator op : mFromEdges.keySet()) {
            PhysicalOperator cloneFrom = matches.get(op);
            if (cloneFrom == null) {
                String msg = "Unable to find clone for op " + op.name();
                throw new CloneNotSupportedException(msg);
            }
            Collection<PhysicalOperator> toOps = mFromEdges.get(op);
            for (PhysicalOperator toOp : toOps) {
                PhysicalOperator cloneTo = matches.get(toOp);
                if (cloneTo == null) {
                    String msg = "Unable to find clone for op " + toOp.name();
                    throw new CloneNotSupportedException(msg);
                }
                try {
                    clone.connect(cloneFrom, cloneTo);
                } catch (PlanException pe) {
                    CloneNotSupportedException cnse = new CloneNotSupportedException();
                    cnse.initCause(pe);
                    throw cnse;
                }
            }
        }

        // Fix up all the inputs in the operators themselves.
        for (PhysicalOperator op : mOps.keySet()) {
            List<PhysicalOperator> inputs = op.getInputs();
            if (inputs == null || inputs.size() == 0) continue;
            List<PhysicalOperator> newInputs =
                new ArrayList<PhysicalOperator>(inputs.size());
            PhysicalOperator cloneOp = matches.get(op);
            if (cloneOp == null) {
                String msg = "Unable to find clone for op " + op.name();
                throw new CloneNotSupportedException(msg);
            }
            for (PhysicalOperator iOp : inputs) {
                PhysicalOperator cloneIOp = matches.get(iOp);
                if (cloneIOp == null) {
                    String msg = "Unable to find clone for op " + iOp.name();
                    throw new CloneNotSupportedException(msg);
                }
                newInputs.add(cloneIOp);
View Full Code Here


       
        assertEquals( 3, phyPlan.size() );
        assertEquals( 1, phyPlan.getRoots().size() );
        assertEquals( 1, phyPlan.getLeaves().size() );
       
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt") );
       
        // Check for Filter
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        assertEquals( 2, filPlan.getRoots().size() );
        assertEquals( 1, filPlan.getLeaves().size() );
       
        PhysicalOperator eq = filPlan.getLeaves().get(0);
        assertEquals( EqualToExpr.class, eq.getClass() );
       
        PhysicalOperator prj1 = filPlan.getRoots().get(0);
        assertEquals( POProject.class, prj1.getClass() );
        assertEquals( 0, ((POProject)prj1).getColumn() );
        PhysicalOperator constExp = filPlan.getRoots().get(1);
        assertEquals( ConstantExpression.class, constExp.getClass() );
        assertEquals( null, ((ConstantExpression)constExp).getValue() );
       
        // Check for Store
        PhysicalOperator stor = phyPlan.getSuccessors(fil).get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty"));
    }
View Full Code Here

        PhysicalPlan physicalPlan = translatePlan(newPlan);
        assertEquals(9, physicalPlan.size());
        assertEquals(physicalPlan.getRoots().size(), 2);
       
        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj3 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d1.txt") );
               
        PhysicalOperator LoR1 = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj4 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals(1, ((POProject)prj4).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
       
        PhysicalOperator GoR = (PhysicalOperator)physicalPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );
       
        PhysicalOperator Pack = (PhysicalOperator)physicalPlan.getSuccessors(GoR).get(0);
        assertEquals( POPackage.class, Pack.getClass() );

        // Check for ForEach
        PhysicalOperator ForE = (PhysicalOperator)physicalPlan.getSuccessors(Pack).get(0);
        assertEquals( POForEach.class, ForE.getClass() );
        PhysicalOperator prj5 = ((POForEach)ForE).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj5.getClass() );
        assertEquals( 1, ((POProject)prj5).getColumn() );
        PhysicalOperator prj6 = ((POForEach)ForE).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj6.getClass() );
        assertEquals( 2, ((POProject)prj6).getColumn() );
       
        // Filter Operator
        PhysicalOperator fil = (PhysicalOperator)physicalPlan.getSuccessors(ForE).get(0);
        assertEquals( POFilter.class, fil.getClass() );       
       
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        List<PhysicalOperator> filRoots = filPlan.getRoots();
       
        assertEquals( ConstantExpression.class, filRoots.get(1).getClass() );
        ConstantExpression ce1 = (ConstantExpression) filRoots.get(1);
        assertEquals( null, ce1.getValue() );
        assertEquals( ConstantExpression.class, filRoots.get(3).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(3);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(0).getClass() );
        POProject prj1 = (POProject) filRoots.get(0);
        assertEquals( 3, prj1.getColumn() );
        assertEquals( POProject.class, filRoots.get(2).getClass() );
        POProject prj2 = (POProject) filRoots.get(2);
        assertEquals( 0, prj2.getColumn() );


        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)physicalPlan.getSuccessors(fil).get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty") );
    }
View Full Code Here

        assertEquals(16, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 3);
        assertEquals(phyPlan.getLeaves().size(), 1 );

        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj1 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj1.getClass() );
        assertEquals(0, ((POProject)prj1).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d3.txt") );

        PhysicalOperator LoR1 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj2 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj2.getClass() );
        assertEquals(1, ((POProject)prj2).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
       
        PhysicalOperator GoR = (PhysicalOperator)phyPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );
       
        PhysicalOperator Pack = (PhysicalOperator)phyPlan.getSuccessors(GoR).get(0);
        assertEquals( POPackage.class, Pack.getClass() );
       
        PhysicalOperator LoR2 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(2)).get(0);
        assertEquals( POLocalRearrange.class, LoR2.getClass() );
        POLocalRearrange Lor2 = (POLocalRearrange) LoR2;
        PhysicalOperator prj3 = Lor2.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp3 = Lor2.getInputs().get(0);
        assertEquals( POLoad.class, inp3.getClass() );
        assertTrue(  ((POLoad)inp3).getLFile().getFileName().contains("d1.txt") );
       
        PhysicalOperator GoR2 = (PhysicalOperator)phyPlan.getSuccessors(LoR2).get(0);
        assertEquals( POGlobalRearrange.class, GoR2.getClass() );
       
        PhysicalOperator Pack2 = (PhysicalOperator)phyPlan.getSuccessors(GoR2).get(0);
        assertEquals( POPackage.class, Pack2.getClass() );
       
        // Check for ForEach
        PhysicalOperator ForE = (PhysicalOperator)phyPlan.getSuccessors(Pack).get(0);
        assertEquals( POForEach.class, ForE.getClass() );
        PhysicalOperator prj4 = ((POForEach)ForE).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals( 1, ((POProject)prj4).getColumn() );
        PhysicalOperator prj5 = ((POForEach)ForE).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj5.getClass() );
        assertEquals( 2, ((POProject)prj5).getColumn() );
       
        PhysicalOperator ForE2 = (PhysicalOperator)phyPlan.getSuccessors(Pack2).get(0);
        assertEquals( POForEach.class, ForE2.getClass() );
        PhysicalOperator prj6 = ((POForEach)ForE2).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj6.getClass() );
        assertEquals( 1, ((POProject)prj6).getColumn() );
        PhysicalOperator prj7 = ((POForEach)ForE2).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj7.getClass() );
        assertEquals( 2, ((POProject)prj7).getColumn() );
       
        // Check Filter Operator
        PhysicalOperator fil = (PhysicalOperator)phyPlan.getSuccessors(ForE).get(0);
        assertEquals( POFilter.class, fil.getClass() );
       
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        List<PhysicalOperator> filRoots = filPlan.getRoots();
       
        assertEquals( ConstantExpression.class, filRoots.get(0).getClass() );
        ConstantExpression ce1 = (ConstantExpression) filRoots.get(0);
        assertEquals( null, ce1.getValue() );
        assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(2);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(1).getClass() );
        POProject prj8 = (POProject) filRoots.get(1);
        assertEquals( 5, prj8.getColumn() );
        assertEquals( POProject.class, filRoots.get(3).getClass() );
        POProject prj9 = (POProject) filRoots.get(3);
        assertEquals( 2, prj9.getColumn() );
       
       
        PhysicalOperator fil2 = (PhysicalOperator)phyPlan.getSuccessors(ForE2).get(0);
        assertEquals( POFilter.class, fil2.getClass() );
       
        PhysicalOperator LoR3 = (PhysicalOperator)phyPlan.getSuccessors(fil2).get(0);
        assertEquals( POLocalRearrange.class, LoR3.getClass() );
        POLocalRearrange Lor3 = (POLocalRearrange) LoR3;
        PhysicalOperator prj12 = Lor3.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj12.getClass() );
        assertEquals(3, ((POProject)prj12).getColumn() );
       
        PhysicalPlan filPlan2 = ((POFilter)fil2).getPlan();
        List<PhysicalOperator> filRoots2 = filPlan2.getRoots();
       
        assertEquals( ConstantExpression.class, filRoots2.get(0).getClass() );
        ConstantExpression ce3 = (ConstantExpression) filRoots2.get(0);
        assertEquals( null, ce3.getValue() );
        assertEquals( ConstantExpression.class, filRoots2.get(2).getClass() );
        ConstantExpression ce4 = (ConstantExpression) filRoots2.get(2);
        assertEquals( null, ce4.getValue() );
        assertEquals( POProject.class, filRoots2.get(1).getClass() );
        POProject prj10 = (POProject) filRoots2.get(1);
        assertEquals( 3, prj10.getColumn() );
        assertEquals( POProject.class, filRoots2.get(3).getClass() );
        POProject prj11 = (POProject) filRoots2.get(3);
        assertEquals( 0, prj11.getColumn() );
       
        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)phyPlan.getLeaves().get(0);
        assertEquals( stor, phyPlan.getSuccessors(fil).get(0));
        assertEquals( POStore.class, stor.getClass() );
        assertTrue(  ((POStore)stor).getSFile().getFileName().contains("empty") );
    }
View Full Code Here

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
       
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
       
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
       
        PhysicalOperator equal = filPlan.getLeaves().get(0);
        assertEquals( EqualToExpr.class, equal.getClass() );
        assertEquals( DataType.BOOLEAN, ((EqualToExpr)equal).getResultType() );
       
        PhysicalOperator constExpr = ((EqualToExpr)equal).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
       
        PhysicalOperator castExpr = ((EqualToExpr)equal).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
       
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
       
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );       
    }
View Full Code Here

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
       
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
       
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
       
        PhysicalOperator greaterThan = filPlan.getLeaves().get(0);
        assertEquals( GreaterThanExpr.class, greaterThan.getClass() );
        assertEquals( DataType.BOOLEAN, ((GreaterThanExpr)greaterThan).getResultType() );
       
        PhysicalOperator constExpr = ((GreaterThanExpr)greaterThan).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
       
        PhysicalOperator castExpr = ((GreaterThanExpr)greaterThan).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
       
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
       
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) )
    }
View Full Code Here

        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
        assertEquals(3, phyPlan.size());
        assertEquals(phyPlan.getRoots().size(), 1);
        assertEquals(phyPlan.getLeaves().size(), 1 );
       
        PhysicalOperator load = phyPlan.getRoots().get(0);
        assertEquals( POLoad.class, load.getClass() );
        assertTrue(  ((POLoad)load).getLFile().getFileName().contains("d.txt"));
       
        PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
       
        PhysicalOperator lessThan = filPlan.getLeaves().get(0);
        assertEquals( LessThanExpr.class, lessThan.getClass() );
        assertEquals( DataType.BOOLEAN, ((LessThanExpr)lessThan).getResultType() );
       
        PhysicalOperator constExpr = ((LessThanExpr)lessThan).getRhs();
        assertEquals( ConstantExpression.class, constExpr.getClass() );
        assertEquals( 10, ((ConstantExpression)constExpr).getValue() );
       
        PhysicalOperator castExpr = ((LessThanExpr)lessThan).getLhs();
        assertEquals( POCast.class, castExpr.getClass() );
        assertEquals( DataType.INTEGER, ((POCast)castExpr).getResultType() );
       
        PhysicalOperator prj = ((POCast)castExpr).getInputs().get(0);
        assertEquals( POProject.class, prj.getClass() );
        assertEquals( 0, ((POProject)prj).getColumn() );
       
        PhysicalOperator stor = phyPlan.getLeaves().get(0);
        assertEquals( POStore.class, stor.getClass() );
        assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) )
    }
View Full Code Here

        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
View Full Code Here

        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
View Full Code Here

        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.