Package org.apache.pig.backend.hadoop.executionengine.physicalLayer

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator


        return res;
    }

    @Override
    public Result getNext(Long l) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte resultType = in.getResultType();
        switch (resultType) {
        case DataType.BAG: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.TUPLE: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.MAP: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                try {
                    dba = (DataByteArray) res.result;
                } catch (ClassCastException e) {
                    // res.result is not of type ByteArray. But it can be one of the types from which cast is still possible.
                    if (realType == null)
                        // Find the type in first call and cache it.
                        realType = DataType.findType(res.result);
                    try {
                        res.result = DataType.toLong(res.result, realType);
                    } catch (ClassCastException cce) {
                        // Type has changed. Need to find type again and try casting it again.
                        realType = DataType.findType(res.result);
                        res.result = DataType.toLong(res.result, realType);
                    }
                    return res;
                }
                try {
                    if (null != caster) {
                        res.result = caster.bytesToLong(dba.get());
                    } else {
                        int errCode = 1075;
                        String msg = "Received a bytearray from the UDF. Cannot determine how to convert the bytearray to long.";
                        throw new ExecException(msg, errCode, PigException.INPUT);
                    }
                } catch (ExecException ee) {
                    throw ee;
                } catch (IOException e) {
                    log.error("Error while casting from ByteArray to Long");
                }
            }
            return res;
        }

        case DataType.BOOLEAN: {
            Boolean b = null;
            Result res = in.getNext(b);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                if (((Boolean) res.result) == true)
                    res.result = Long.valueOf(1);
                else
                    res.result = Long.valueOf(0);
            }
            return res;
        }
        case DataType.INTEGER: {
            Integer dummyI = null;
            Result res = in.getNext(dummyI);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = Long.valueOf(((Integer) res.result).longValue());
            }
            return res;
        }

        case DataType.DOUBLE: {
            Double d = null;
            Result res = in.getNext(d);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = DataType.toInteger(res.result);
                res.result = Long.valueOf(((Double) res.result).longValue());
            }
            return res;
        }

        case DataType.LONG: {

            Result res = in.getNext(l);

            return res;
        }

        case DataType.FLOAT: {
            Float f = null;
            Result res = in.getNext(f);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = Long.valueOf(((Float) res.result).longValue());
            }
            return res;
        }

        case DataType.CHARARRAY: {
            String str = null;
            Result res = in.getNext(str);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = CastUtils.stringToLong((String)res.result);
            }
            return res;
        }
View Full Code Here


        return res;
    }

    @Override
    public Result getNext(Double d) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte resultType = in.getResultType();
        switch (resultType) {
        case DataType.BAG: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.TUPLE: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.MAP: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                try {
                    dba = (DataByteArray) res.result;
                } catch (ClassCastException e) {
                    // res.result is not of type ByteArray. But it can be one of the types from which cast is still possible.
                    if (realType == null)
                        // Find the type in first call and cache it.
                        realType = DataType.findType(res.result);
                    try {
                        res.result = DataType.toDouble(res.result, realType);
                    } catch (ClassCastException cce) {
                        // Type has changed. Need to find type again and try casting it again.
                        realType = DataType.findType(res.result);
                        res.result = DataType.toDouble(res.result, realType);
                    }
                    return res;
                }
                try {
                    if (null != caster) {
                        res.result = caster.bytesToDouble(dba.get());
                    } else {
                        int errCode = 1075;
                        String msg = "Received a bytearray from the UDF. Cannot determine how to convert the bytearray to double.";
                        throw new ExecException(msg, errCode, PigException.INPUT);
                    }
                } catch (ExecException ee) {
                    throw ee;
                } catch (IOException e) {
                    log.error("Error while casting from ByteArray to Double");
                }
            }
            return res;
        }

        case DataType.BOOLEAN: {
            Boolean b = null;
            Result res = in.getNext(b);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                if (((Boolean) res.result) == true)
                    res.result = new Double(1);
                else
                    res.result = new Double(0);
            }
            return res;
        }
        case DataType.INTEGER: {
            Integer dummyI = null;
            Result res = in.getNext(dummyI);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = new Double(((Integer) res.result).doubleValue());
            }
            return res;
        }

        case DataType.DOUBLE: {

            Result res = in.getNext(d);

            return res;
        }

        case DataType.LONG: {
            Long l = null;
            Result res = in.getNext(l);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = new Double(((Long) res.result).doubleValue());
            }
            return res;
        }

        case DataType.FLOAT: {
            Float f = null;
            Result res = in.getNext(f);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = new Double(((Float) res.result).doubleValue());
            }
            return res;
        }

        case DataType.CHARARRAY: {
            String str = null;
            Result res = in.getNext(str);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = CastUtils.stringToDouble((String)res.result);
            }
            return res;
        }
View Full Code Here

        return res;
    }

    @Override
    public Result getNext(Float f) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte resultType = in.getResultType();
        switch (resultType) {
        case DataType.BAG: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.TUPLE: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.MAP: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                try {
                    dba = (DataByteArray) res.result;
                } catch (ClassCastException e) {
                    // res.result is not of type ByteArray. But it can be one of the types from which cast is still possible.
                    if (realType == null)
                        // Find the type in first call and cache it.
                        realType = DataType.findType(res.result);
                    try {
                        res.result = DataType.toFloat(res.result, realType);
                    } catch (ClassCastException cce) {
                        // Type has changed. Need to find type again and try casting it again.
                        realType = DataType.findType(res.result);
                        res.result = DataType.toFloat(res.result, realType);
                    }
                    return res;
                }
                try {
                    if (null != caster) {
                        res.result = caster.bytesToFloat(dba.get());
                    } else {
                        int errCode = 1075;
                        String msg = "Received a bytearray from the UDF. Cannot determine how to convert the bytearray to float.";
                        throw new ExecException(msg, errCode, PigException.INPUT);
                    }
                } catch (ExecException ee) {
                    throw ee;
                } catch (IOException e) {
                    log.error("Error while casting from ByteArray to Float");
                }
            }
            return res;
        }

        case DataType.BOOLEAN: {
            Boolean b = null;
            Result res = in.getNext(b);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                if (((Boolean) res.result) == true)
                    res.result = new Float(1);
                else
                    res.result = new Float(0);
            }
            return res;
        }
        case DataType.INTEGER: {
            Integer dummyI = null;
            Result res = in.getNext(dummyI);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = new Float(((Integer) res.result).floatValue());
            }
            return res;
        }

        case DataType.DOUBLE: {
            Double d = null;
            Result res = in.getNext(d);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = DataType.toInteger(res.result);
                res.result = new Float(((Double) res.result).floatValue());
            }
            return res;
        }

        case DataType.LONG: {

            Long l = null;
            Result res = in.getNext(l);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = new Float(((Long) res.result).floatValue());
            }
            return res;
        }

        case DataType.FLOAT: {

            Result res = in.getNext(f);

            return res;
        }

        case DataType.CHARARRAY: {
            String str = null;
            Result res = in.getNext(str);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = CastUtils.stringToFloat((String)res.result);
            }
            return res;
        }
View Full Code Here

        return res;
    }

    @Override
    public Result getNext(String str) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte resultType = in.getResultType();
        switch (resultType) {
        case DataType.BAG: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.TUPLE: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.MAP: {
            Result res = new Result();
            res.returnStatus = POStatus.STATUS_ERR;
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                try {
                    dba = (DataByteArray) res.result;
                } catch (ClassCastException e) {
                    // res.result is not of type ByteArray. But it can be one of the types from which cast is still possible.
                    if (realType == null)
                        // Find the type in first call and cache it.
                        realType = DataType.findType(res.result);
                    try {
                        res.result = DataType.toString(res.result, realType);
                    } catch (ClassCastException cce) {
                        // Type has changed. Need to find type again and try casting it again.
                        realType = DataType.findType(res.result);
                        res.result = DataType.toString(res.result, realType);
                    }
                    return res;
                }
                try {
                    if (null != caster) {
                        res.result = caster.bytesToCharArray(dba.get());
                    } else {
                        int errCode = 1075;
                        String msg = "Received a bytearray from the UDF. Cannot determine how to convert the bytearray to string.";
                        throw new ExecException(msg, errCode, PigException.INPUT);
                    }
                } catch (ExecException ee) {
                    throw ee;
                } catch (IOException e) {
                    log
                            .error("Error while casting from ByteArray to CharArray");
                }
            }
            return res;
        }

        case DataType.BOOLEAN: {
            Boolean b = null;
            Result res = in.getNext(b);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                if (((Boolean) res.result) == true)
                    res.result = "1";
                else
                    res.result = "0";
            }
            return res;
        }
        case DataType.INTEGER: {
            Integer dummyI = null;
            Result res = in.getNext(dummyI);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = ((Integer) res.result).toString();
            }
            return res;
        }

        case DataType.DOUBLE: {
            Double d = null;
            Result res = in.getNext(d);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = DataType.toInteger(res.result);
                res.result = ((Double) res.result).toString();
            }
            return res;
        }

        case DataType.LONG: {

            Long l = null;
            Result res = in.getNext(l);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = ((Long) res.result).toString();
            }
            return res;
        }

        case DataType.FLOAT: {
            Float f = null;
            Result res = in.getNext(f);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                res.result = ((Float) res.result).toString();
            }
            return res;
        }

        case DataType.CHARARRAY: {
            Result res = in.getNext(str);

            return res;
        }

        }
View Full Code Here

        return res;
    }

    @Override
    public Result getNext(Tuple t) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte castToType = DataType.TUPLE;
        Byte resultType = in.getResultType();
        switch (resultType) {

        case DataType.TUPLE: {
            Result res = in.getNext(t);
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = new
                // String(((DataByteArray)res.result).toString());
                if (castNotNeeded) {
                    // we examined the data once before and
View Full Code Here

        return res;
    }

    @Override
    public Result getNext(DataBag bag) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte castToType = DataType.BAG;
        Byte resultType = in.getResultType();
        switch (resultType) {

        case DataType.BAG: {
            Result res = in.getNext(bag);
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = new
                // String(((DataByteArray)res.result).toString());
                if (castNotNeeded) {
                    // we examined the data once before and
View Full Code Here

        return res;
    }

    @Override
    public Result getNext(Map m) throws ExecException {
        PhysicalOperator in = inputs.get(0);
        Byte castToType = DataType.MAP;
        Byte resultType = in.getResultType();
        switch (resultType) {

        case DataType.MAP: {
            Result res = in.getNext(m);
            return res;
        }

        case DataType.BYTEARRAY: {
            DataByteArray dba = null;
            Result res = in.getNext(dba);
            if (res.returnStatus == POStatus.STATUS_OK && res.result != null) {
                // res.result = new
                // String(((DataByteArray)res.result).toString());
                if (castNotNeeded) {
                    // we examined the data once before and
View Full Code Here

        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
        List<PhysicalOperator> inputs = new ArrayList<PhysicalOperator>();
        PhysicalPlan phy = new PhysicalPlan();
        phy.add(physOp);

        for (PhysicalOperator input : physOp.getInputs()) {
            inputs.add(input.getInputs().get(0));
            input.setInputs(null);
            phy.add(input);
            try {
                phy.connect(input, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + input.name() + " to "
                        + physOp.name());
            }
        }
        physOp.setLineageTracer(lineage);

        physOp.setLineageTracer(null);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(op.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                log.error("Error connecting " + por.name() + " to "
                        + physOp.name());
            }
        }

        // replace the original inputs by POReads
        // for(LogicalOperator l : op.getPlan().getPredecessors(op)) {
        // DataBag bag = derivedData.get(l);
        // PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
        // phy.add(por);
        // try {
        // phy.connect(por, physOp);
        // } catch (PlanException e) {
        // // TODO Auto-generated catch block
        // e.printStackTrace();
        // log.error("Error connecting " + por.name() + " to " + physOp.name());
        // }
        // }

        DataBag output = BagFactory.getInstance().newDefaultBag();
        Tuple t = null;
        try {
            for (Result res = physOp.getNext(t); res.returnStatus != POStatus.STATUS_EOP; res = physOp
                    .getNext(t)) {
                output.add((Tuple) res.result);
            }
        } catch (ExecException e) {
            log.error("Error evaluating operator : " + physOp.name());
        }

        this.lineage = oldLineage;

        physOp.setInputs(inputs);
        physOp.setLineageTracer(null);

        return output;
    }
View Full Code Here

    }
   
    @SuppressWarnings("unchecked")
    public Result getNext(Tuple t) throws ExecException {
  if(it == null) {
      PhysicalOperator op = getInputs().get(0);
      Result res = getInputs().get(0).getNext(t);
      if(res.returnStatus == POStatus.STATUS_OK)
    it = (Iterator<Tuple>) res.result;
  }
  Result res = null;
View Full Code Here

            // Check map plan
            List<PhysicalOperator> mpLeaves = mr.mapPlan.getLeaves();
            if (mpLeaves.size()!=1) {
                return;
            }
            PhysicalOperator op = mpLeaves.get(0);
           
            if (!(op instanceof POUnion)) {
                return;
            }
           
            // Check reduce plan
            List<PhysicalOperator> mrRoots = mr.reducePlan.getRoots();
            if (mrRoots.size()!=1) {
                return;
            }
           
            op = mrRoots.get(0);
            if (!(op instanceof POPackage)) {
                return;
            }
            POPackage pack = (POPackage)op;
           
            List<PhysicalOperator> sucs = mr.reducePlan.getSuccessors(pack);
            if (sucs.size()!=1) {
                return;
            }
           
            op = sucs.get(0);
            boolean lastInputFlattened = true;
            boolean allSimple = true;
            if (op instanceof POForEach)
            {
                POForEach forEach = (POForEach)op;
                List<PhysicalPlan> planList = forEach.getInputPlans();
                List<Boolean> flatten = forEach.getToBeFlattened();
                POProject projOfLastInput = null;
                int i = 0;
                // check all nested foreach plans
                // 1. If it is simple projection
                // 2. If last input is all flattened
                for (PhysicalPlan p:planList)
                {
                    PhysicalOperator opProj = p.getRoots().get(0);
                    if (!(opProj instanceof POProject))
                    {
                        allSimple = false;
                        break;
                    }
                    POProject proj = (POProject)opProj;
                    // the project should just be for one column
                    // from the input
                    if(proj.getColumns().size() != 1) {
                        allSimple = false;
                        break;
                    }
                   
                    try {
                        // if input to project is the last input
                        if (proj.getColumn() == pack.getNumInps())
                        {
                            // if we had already seen another project
                            // which was also for the last input, then
                            // we might be trying to flatten twice on the
                            // last input in which case we can't optimize by
                            // just streaming the tuple to those projects
                            // IMPORTANT NOTE: THIS WILL NEED TO CHANGE WHEN WE
                            // OPTIMIZE BUILTINS LIKE SUM() AND COUNT() TO
                            // TAKE IN STREAMING INPUT
                            if(projOfLastInput != null) {
                                allSimple = false;
                                break;
                            }
                            projOfLastInput = proj;
                            // make sure the project is on a bag which needs to be
                            // flattened
                            if (!flatten.get(i) || proj.getResultType() != DataType.BAG)
                            {
                                lastInputFlattened = false;
                                break;
                            }
                        }
                    } catch (ExecException e) {
                        int errCode = 2069;
                        String msg = "Error during map reduce compilation. Problem in accessing column from project operator.";
                        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
                    }
                   
                    // if all deeper operators are all project
                    PhysicalOperator succ = p.getSuccessors(proj)!=null?p.getSuccessors(proj).get(0):null;
                    while (succ!=null)
                    {
                        if (!(succ instanceof POProject))
                        {
                            allSimple = false;
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.