Package org.apache.pig.backend.hadoop.executionengine.physicalLayer

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator


      pp.addAsLeaf(store);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
     
      for(MapReduceOper mro: mrPlan.getLeaves()) {
        if(mro.reducePlan != null) {
          PhysicalOperator po = mro.reducePlan.getRoots().get(0);
          if(po instanceof POPackage) {
            ((POPackage)po).setKeyType(DataType.BAG);
            mro.setGlobalSort(true);
          }
        }
View Full Code Here


    }
   
    @Override
    public void visit(LOSplit split) throws VisitorException {
  String scope = split.getOperatorKey().scope;
        PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), split.getRequestedParallelism());
       
        logToPhyMap.put(split, physOp);

        currentPlan.add(physOp);
        PhysicalOperator from = logToPhyMap.get(split.getPlan()
                .getPredecessors(split).get(0));
        try {
            currentPlan.connect(from, physOp);
        } catch (PlanException e) {
            log.error("Invalid physical operator in the plan" + e.getMessage());
View Full Code Here

    }
   
    @Override
    public void visit(LOSplitOutput split) throws VisitorException {
  String scope = split.getOperatorKey().scope;
        PhysicalOperator physOp = new POSplitOutput(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), split.getRequestedParallelism());
        logToPhyMap.put(split, physOp);

        currentPlan.add(physOp);
        currentPlans.push(currentPlan);
        currentPlan = new PhysicalPlan();
        PlanWalker<LogicalOperator, LogicalPlan> childWalker = mCurrentWalker
                .spawnChildWalker(split.getConditionPlan());
        pushWalker(childWalker);
        mCurrentWalker.walk(this);
        popWalker();

        ((POSplitOutput) physOp).setPlan(currentPlan);
        currentPlan = currentPlans.pop();
        currentPlan.add(physOp);
        PhysicalOperator from = logToPhyMap.get(split.getPlan()
                .getPredecessors(split).get(0));
        try {
            currentPlan.connect(from, physOp);
        } catch (PlanException e) {
            log.error("Invalid physical operator in the plan" + e.getMessage());
View Full Code Here

        currentPlan.add(poStream);
        logToPhyMap.put(stream, poStream);
       
        List<LogicalOperator> op = stream.getPlan().getPredecessors(stream);

        PhysicalOperator from = logToPhyMap.get(op.get(0));
        try {
            currentPlan.connect(from, poStream);
        } catch (PlanException e) {
            log.error("Invalid physical operators in the physical plan"
                    + e.getMessage());
View Full Code Here

        logToPhyMap.put(cross, pocross);
        currentPlan.add(pocross);
       
       
        for(LogicalOperator in : cross.getInputs()) {
            PhysicalOperator from = logToPhyMap.get(in);
            try {
                currentPlan.connect(from, pocross);
            } catch (PlanException e) {
                log.error("Invalid physical operators in the physical plan"
                        + e.getMessage());
View Full Code Here

            String message = "Cannot resolve Store output schema"
            throw new VisitorException(message, errorCode, PigException.BUG, e1);   
        }
        //store.setPc(pc);
        currentPlan.add(store);
        PhysicalOperator from = logToPhyMap.get(loStore
                .getPlan().getPredecessors(loStore).get(0));
       
        POCounter counter = new POCounter(new OperatorKey(scope, nodeGen.getNextNodeId(scope)));
        currentPlan.add(counter);
        try {
View Full Code Here

        PhysicalPlan grpChain1 = GenPhyOp.grpChain();
        php.merge(grpChain1);

        php.connect(ld, grpChain1.getRoots().get(0));

        PhysicalOperator leaf = php.getLeaves().get(0);

        PhysicalPlan grpChain2 = GenPhyOp.grpChain();
        php.merge(grpChain2);

        php.connect(leaf, grpChain2.getRoots().get(0));
View Full Code Here

    @Override
    protected void visit(LOCogroup cg) throws VisitorException {
        // evaluateOperator(cg);
        // there is a slightly different code path for cogroup because of the
        // local rearranges
        PhysicalOperator physOp = LogToPhyMap.get(cg);
        Random r = new Random();
        // get the list of original inputs

        // List<PhysicalOperator> inputs = physOp.getInputs();
        List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        // for(PhysicalOperator input : physOp.getInputs()) {
        for (PhysicalOperator input : physPlan.getPredecessors(physOp)) {
            inputs.add(input.getInputs().get(0));
            // input.setInputs(null);
            phy.add(input);
            try {
                phy.connect(input, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + input.name() + " to "
                        + physOp.name());
            }
        }

        physOp.setLineageTracer(lineage);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(cg.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
            }
        }

        DataBag output = BagFactory.getInstance().newDefaultBag();
        Tuple t = null;
        try {
            for (Result res = physOp.getNext(t); res.returnStatus != POStatus.STATUS_EOP; res = physOp
                    .getNext(t)) {
                output.add((Tuple) res.result);
            }
        } catch (ExecException e) {
            log.error("Error evaluating operator : " + physOp.name());
        }
        derivedData.put(cg, output);

        try {
            Collection<IdentityHashSet<Tuple>> eq = EquivalenceClasses
                    .getEquivalenceClasses(cg, derivedData);
            EqClasses.addAll(eq);
            OpToEqClasses.put(cg, eq);
        } catch (ExecException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            log
                    .error("Error updating equivalence classes while evaluating operators. \n"
                            + e.getMessage());
        }

        // re-attach the original operators
        // for(int i = 0; i < inputs.size(); i++) {
        // try {
        // physPlan.connect(inputs.get(i), physOp.getInputs().get(i));
        //   
        // } catch (PlanException e) {
        // // TODO Auto-generated catch block
        // e.printStackTrace();
        // log.error("Error connecting " + inputs.get(i).name() + " to " +
        // physOp.getInputs().get(i).name());
        // }
        // }
        physOp.setLineageTracer(null);
    }
View Full Code Here

    // return plan;
    //
    // }

    private void evaluateOperator(LogicalOperator op) {
        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs

        List<PhysicalOperator> inputs = physOp.getInputs();
        physOp.setInputs(null);
        physOp.setLineageTracer(lineage);
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        // replace the original inputs by POReads
        for (LogicalOperator l : op.getPlan().getPredecessors(op)) {
            DataBag bag = derivedData.get(l);
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
            }
        }

        DataBag output = BagFactory.getInstance().newDefaultBag();
        Tuple t = null;
        try {
            for (Result res = physOp.getNext(t); res.returnStatus != POStatus.STATUS_EOP; res = physOp
                    .getNext(t)) {
                output.add((Tuple) res.result);
            }
        } catch (ExecException e) {
            log.error("Error evaluating operator : " + physOp.name());
        }
        derivedData.put(op, output);

        try {
            Collection<IdentityHashSet<Tuple>> eq = EquivalenceClasses
                    .getEquivalenceClasses(op, derivedData);
            EqClasses.addAll(eq);
            OpToEqClasses.put(op, eq);
        } catch (ExecException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            log
                    .error("Error updating equivalence classes while evaluating operators. \n"
                            + e.getMessage());
        }

        // re-attach the original operators
        physOp.setInputs(inputs);
        physOp.setLineageTracer(null);
    }
View Full Code Here

        return false;
    }

    @Override
    public Result getNext(Integer i) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte resultType = in.getResultType();
        switch (resultType) {
        case DataType.BAG: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.TUPLE: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                try {
                    dba = (DataByteArray) res.result;
                } catch (ClassCastException e) {
                    // res.result is not of type ByteArray. But it can be one of the types from which cast is still possible.
                    if (realType == null)
                        // Find the type and cache it.
                        realType = DataType.findType(res.result);
                    try {
                        res.result = DataType.toInteger(res.result, realType);
                    } catch (ClassCastException cce) {
                        // Type has changed. Need to find type again and try casting it again.
                        realType = DataType.findType(res.result);
                        res.result = DataType.toInteger(res.result, realType);
                    }
                    return res;
                }
                try {
                    if (null != caster) {
                        res.result = caster.bytesToInteger(dba.get());
                    } else {
                        int errCode = 1075;
                        String msg = "Received a bytearray from the UDF. Cannot determine how to convert the bytearray to int.";
                        throw new ExecException(msg, errCode, PigException.INPUT);
                    }
                } catch (ExecException ee) {
                    throw ee;
                } catch (IOException e) {
                    log.error("Error while casting from ByteArray to Integer");
                }
            }
            return res;
        }

        case DataType.MAP: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BOOLEAN: {
            Boolean b = null;
            Result res = in.getNext(b);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                if (((Boolean) res.result) == true)
                    res.result = Integer.valueOf(1);
                else
                    res.result = Integer.valueOf(0);
            }
            return res;
        }
        case DataType.INTEGER: {

            Result res = in.getNext(i);
            return res;
        }

        case DataType.DOUBLE: {
            Double d = null;
            Result res = in.getNext(d);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = DataType.toInteger(res.result);
                res.result = Integer.valueOf(((Double) res.result).intValue());
            }
            return res;
        }

        case DataType.LONG: {
            Long l = null;
            Result res = in.getNext(l);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = Integer.valueOf(((Long) res.result).intValue());
            }
            return res;
        }

        case DataType.FLOAT: {
            Float f = null;
            Result res = in.getNext(f);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = Integer.valueOf(((Float) res.result).intValue());
            }
            return res;
        }

        case DataType.CHARARRAY: {
            String str = null;
            Result res = in.getNext(str);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = CastUtils.stringToInteger((String)res.result);
            }
            return res;
        }
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.