Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema


        LogicalPlan innerPlan = (LogicalPlan)gen.getPlan();
        ArrayList<Operator> inputs = new ArrayList<Operator>();
        int idx = 0;
        for( LogicalExpressionPlan exprPlan : exprPlans ) {
            LogicalExpression expr = (LogicalExpression)exprPlan.getSources().get(0);
            LogicalSchema userSchema = schemas.get(idx);
            if (userSchema == null && expr.hasFieldSchema()) {
                LogicalSchema ls = new LogicalSchema();
                try {
                    ls.addField(expr.getFieldSchema());
                    schemas.set(idx, ls);
                } catch (FrontendException e) {
                    // if we get an exception, then we have no schema to set
                }
            }
View Full Code Here


    @Override
    public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
        if (fieldSchema!=null)
            return fieldSchema;

        LogicalSchema inputSchema = new LogicalSchema();
        List<Operator> succs = plan.getSuccessors(this);

        if (succs!=null) {
            for(Operator lo : succs){
                if (((LogicalExpression)lo).getFieldSchema()==null) {
                    inputSchema = null;
                    break;
                }
                inputSchema.addField(((LogicalExpression)lo).getFieldSchema());
            }
        }

        if (lazilyInitializeInvokerFunction) {
            initializeInvokerFunction();
View Full Code Here

            }
            return mappedKeys;
        }

        protected void setupColNameMaps() throws FrontendException {
            LogicalSchema loLoadSchema = loLoad.getSchema();
            LogicalSchema loadFuncSchema = loLoad.getDeterminedSchema();
             for(int i = 0; i < loadFuncSchema.size(); i++) {
                colNameMap.put(loadFuncSchema.getField(i).alias,
                        (i < loLoadSchema.size() ? loLoadSchema.getField(i).alias :
                            loadFuncSchema.getField(i).alias));

                reverseColNameMap.put((i < loLoadSchema.size() ? loLoadSchema.getField(i).alias :
                            loadFuncSchema.getField(i).alias),
                            loadFuncSchema.getField(i).alias);
            }
        }
View Full Code Here

        return uid2LoadFuncMap;
    }
   
    @Override
    public void visit(LOLoad load) throws FrontendException{
        LogicalSchema schema = load.getSchema();
        setLoadFuncForUids(schema, load.getFileSpec().getFuncSpec());
        rel2InputFuncMap.put(load, load.getFileSpec().getFuncSpec());
    }
View Full Code Here

    public void visit(LOInnerLoad innerLoad) throws FrontendException{
        LOForEach foreach = innerLoad.getLOForEach();
        LogicalRelationalOperator pred =
            ((LogicalRelationalOperator)foreach.getPlan().getPredecessors(foreach).get(0));

        LogicalSchema predSchema = pred.getSchema();
       
        //if this has a schema, the lineage can be tracked using the uid in input schema
        if(innerLoad.getSchema() != null){
            if(innerLoad.getSchema().size() == 1
                    && innerLoad.getSchema().getField(0).type == DataType.BYTEARRAY
                    && uid2LoadFuncMap.get(innerLoad.getSchema().getField(0).uid) == null
                    && predSchema != null
            ){
                long inpUid = predSchema.getField(innerLoad.getProjection().getColNum()).uid;
                if(uid2LoadFuncMap.get(inpUid) != null){
                    addUidLoadFuncToMap(innerLoad.getSchema().getField(0).uid, uid2LoadFuncMap.get(inpUid));
                }
                return;
            }
View Full Code Here

     * @param relOp
     * @return
     * @throws FrontendException
     */
    private FuncSpec getAssociatedLoadFunc(LogicalRelationalOperator relOp) throws FrontendException {
        LogicalSchema schema = relOp.getSchema();
        FuncSpec funcSpec = null;
        if(schema != null){
            if(schema.size() == 0)
                return null;
            funcSpec = uid2LoadFuncMap.get(schema.getField(0).uid);
            if(funcSpec != null) {
                for(int i=1; i<schema.size(); i++){
                    LogicalFieldSchema fs = schema.getField(i);
                    if(! haveIdenticalCasters(funcSpec,
                            uid2LoadFuncMap.get(fs.uid))){
                        //all uid are not associated with same func spec, there is no
                        // single func spec that represents all the fields
                        funcSpec = null;
View Full Code Here

                throw new AssertionError("Group plans should have only one output");
            }
            groupPlanSchemas.add(((LogicalExpression)expPlan.getSources().get(0)).getFieldSchema());
        }
       
        LogicalSchema sch = group.getSchema();

        //if the group plans are associated with same load function , associate
        //same load fucntion with group column schema
        if (getAssociatedLoadFunc(group)!=null) {
            addUidLoadFuncToMap(sch.getField(0).uid, rel2InputFuncMap.get(group));
            if (sch.getField(0).schema!=null)
                setLoadFuncForUids(sch.getField(0).schema, rel2InputFuncMap.get(group));
        }
        else
            mapMatchLoadFuncToUid(sch.getField(0), groupPlanSchemas);
       
       
       
        //set the load func spec for the bags in the schema, this helps if
        // the input schemas are not set
        //group schema has a group column followed by bags corresponding to each
        // input
        if(sch.size() != inputs.size()+1 ){
            throw new AssertionError("cogroup schema size not same as number of inputs");
        }
       
       
       
        for(int i=1; i < sch.size(); i++){
            long uid = sch.getField(i).uid;
            LogicalRelationalOperator input = (LogicalRelationalOperator) inputs.get(i-1);
            if(getAssociatedLoadFunc(input) != null){
                addUidLoadFuncToMap(uid, rel2InputFuncMap.get(input));
            }
        }
View Full Code Here

                //get the output schema corresponding to this exp plan
                gen.getSchema();
                if(gen.getOutputPlanSchemas() == null || gen.getOutputPlanSchemas().size() <= i){
                    return;
                }
                LogicalSchema sch = gen.getOutputPlanSchemas().get(i);
                if(sch == null){
                    continue;
                }
               
                //get the only output exp of the ith plan
                LogicalExpression exp =
                    (LogicalExpression) gen.getOutputPlans().get(i).getSources().get(0);

                //get its funcspec and associate it with uid of all fields in the schema
                FuncSpec funcSpec = uid2LoadFuncMap.get(exp.getFieldSchema().uid);
                for(LogicalFieldSchema fs : sch.getFields()){
                    addUidLoadFuncToMap(fs.uid, funcSpec);
                }
            }
        }
       
View Full Code Here

    @Override
    public void visit(LOUnion relOp) throws FrontendException{
        mapToPredLoadFunc(relOp);
       
        // Since the uid changes for Union, add mappings for new uids to funcspec
        LogicalSchema schema = relOp.getSchema();
        if(schema != null){
            // For each output field, checking all the fields being
            // union-ed(bundled) together and only set the funcspec when ALL
            // of them come from the same caster
            //
            // A = (i,j)
            // B = (i,j)
            // C = UNION A, B;
            // Checking if A.i and B.i have the same caster.
            // Same for A.j and B.j
            // A.i and A.j may come from the different casters
            for (LogicalFieldSchema logicalFieldSchema : schema.getFields()) {
                Set<Long> inputs = relOp.getInputUids(logicalFieldSchema.uid);
                if( inputs.size() == 0 ) {
                    // uid was not changed.
                    // funcspec should be already set. skipping
                    continue;
View Full Code Here

        //In this visitor the uids for the fields in output are mapped to ones
        // in input based on position. uid2LoadFuncMap is updated with output
        // schema uid and load func mapped to corresponding uid in inputschema
       
       
        LogicalSchema inputSch =
            ((LogicalRelationalOperator)split.getPlan().getPredecessors(split).get(0)).getSchema();

        LogicalSchema outSchema = split.getSchema();
        if(inputSch == null && outSchema == null){
            return;
        }

        if(inputSch == null || outSchema == null){
            String msg = "Bug: in split only one of input/output schema is null "
                + split;
            throw new VisitorException(split, msg,2260, PigException.BUG) ;
        }
       
        if(inputSch.size() != outSchema.size()){
            String msg = "Bug: input and output schema size of split differ "
                + split;
            throw new VisitorException(split, msg,2261, PigException.BUG) ;
        }

        for(int i=0; i<inputSch.size(); i++){
            LogicalFieldSchema inField = inputSch.getField(i);
            LogicalFieldSchema outField = outSchema.getField(i);
            if(uid2LoadFuncMap.get(inField.uid) != null){
                addUidLoadFuncToMap(outField.uid, uid2LoadFuncMap.get(inField.uid));
            }
        }
       
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.