Examples of PhysicalPlan


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        planTester.buildPlan("b = load '/tmp/input2' using " +
            TestMergeJoin.DummyIndexableLoader.class.getName() + ";");
        planTester.buildPlan("c = join a by $0, b by $0 using \"merge\";");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        assertEquals("Checking number of MR Jobs for merge join with " +
            "IndexableLoadFunc:", 1, mp.size());
       
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        planTester.buildPlan("d = order c by a0;");
        planTester.buildPlan("e = foreach d generate a1+a2;");
        LogicalPlan lp = planTester.buildPlan("store e into '/tmp';");
        planTester.typeCheckPlan(lp);
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

     * Test that POSortedDistinct gets printed as POSortedDistinct
     * @throws Exception
     */
    @Test
    public void testSortedDistinctInForeach() throws Exception {
        PhysicalPlan php = new PhysicalPlan();
        PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
        php.merge(grpChain1);
       
        List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
        PhysicalPlan inplan = new PhysicalPlan();
        PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()),
                -1, null);
        inplan.addAsLeaf(op1);
        inputs.add(inplan);
        List<Boolean> toFlattens = new ArrayList<Boolean>();
        toFlattens.add(false);
        POForEach pofe = new POForEach(new OperatorKey("", r.nextLong()), 1,
                inputs, toFlattens);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        // 2) Multi store case - in this case, there is no single storefunc
        // in the conf - instead we would need to look at the
        // map and reduce plans and get the POStores out of it and then get hold
        // of the respective StoreFuncs
        String sFuncString = conf.get("pig.storeFunc");
        PhysicalPlan mp = (PhysicalPlan) ObjectSerializer.deserialize(
                    conf.get("pig.mapPlan"));
        List<POStore> mapStores = PlanHelper.getStores(mp);
        PhysicalPlan rp = (PhysicalPlan) ObjectSerializer.deserialize(
                    conf.get("pig.reducePlan"));
        List<POStore> reduceStores = new ArrayList<POStore>();
        if(rp != null) {
            reduceStores = PlanHelper.getStores(rp);   
        }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        Random r = new Random();
        // get the list of original inputs

        // List<PhysicalOperator> inputs = physOp.getInputs();
        List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        // for(PhysicalOperator input : physOp.getInputs()) {
        for (PhysicalOperator input : physPlan.getPredecessors(physOp)) {
            inputs.add(input.getInputs().get(0));
            // input.setInputs(null);
            phy.add(input);
            try {
                phy.connect(input, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + input.name() + " to "
                        + physOp.name());
            }
        }

        physOp.setLineageTracer(lineage);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(cg.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        // get the list of original inputs

        List<PhysicalOperator> inputs = physOp.getInputs();
        physOp.setInputs(null);
        physOp.setLineageTracer(lineage);
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        // replace the original inputs by POReads
        for (LogicalOperator l : op.getPlan().getPredecessors(op)) {
            DataBag bag = derivedData.get(l);
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
        List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        for (PhysicalOperator input : physOp.getInputs()) {
            inputs.add(input.getInputs().get(0));
            input.setInputs(null);
            phy.add(input);
            try {
                phy.connect(input, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + input.name() + " to "
                        + physOp.name());
            }
        }
        physOp.setLineageTracer(lineage);

        physOp.setLineageTracer(null);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(op.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

public class LogToPhyTranslationVisitor extends LogicalRelationalNodesVisitor {
   
    public LogToPhyTranslationVisitor(OperatorPlan plan) throws FrontendException {
        super(plan, new DependencyOrderWalker(plan));
        currentPlan = new PhysicalPlan();
        logToPhyMap = new HashMap<Operator, PhysicalOperator>();
        currentPlans = new Stack<PhysicalPlan>();
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        poFilter.setResultType(DataType.BAG);
        currentPlan.add(poFilter);
        logToPhyMap.put(filter, poFilter);
        currentPlans.push(currentPlan);

        currentPlan = new PhysicalPlan();

//        PlanWalker childWalker = currentWalker
//                .spawnChildWalker(filter.getFilterPlan());
        PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(filter.getFilterPlan());
        pushWalker(childWalker);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

        List<PhysicalPlan> sortPlans = new ArrayList<PhysicalPlan>(logPlans.size());

        // convert all the logical expression plans to physical expression plans
        currentPlans.push(currentPlan);
        for (LogicalExpressionPlan plan : logPlans) {
            currentPlan = new PhysicalPlan();
            PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(plan);
            pushWalker(childWalker);
            childWalker.walk(new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
                    childWalker, sort, currentPlan, logToPhyMap));
            sortPlans.add(currentPlan);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.