Package org.apache.pig.impl.logicalLayer.schema

Examples of org.apache.pig.impl.logicalLayer.schema.Schema


   
    @Test
    public void testFRJoinSch6() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1) using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1);");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertTrue(shjSch == null);
View Full Code Here


        List<ExpressionOperator> list = func.getArguments() ;

        // If the dependency graph is right, all the inputs
        // must already know the types
        Schema s = new Schema();
        for(ExpressionOperator op: list) {
            if (!DataType.isUsableType(op.getType())) {
                int errCode = 1014;
                String msg = "Problem with input " + op + " of User-defined function: " + func;
                msgCollector.collect(msg, MessageType.Error);
                throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
            }
            try {
                s.add(op.getFieldSchema());   
            } catch (FrontendException e) {
                int errCode = 1043;
                String msg = "Unable to retrieve field schema.";
                throw new TypeCheckerException(msg, errCode, PigException.INPUT, e);
            }
View Full Code Here

            // make sure there is only one type to "cast to" for the byte array
            // positions among the candidate funcSpecs
            Map<Integer, Pair<FuncSpec, Byte>> castToMap = new HashMap<Integer, Pair<FuncSpec, Byte>>();
            for (Iterator<Pair<Long, FuncSpec>> it = scoreFuncSpecList.iterator(); it.hasNext();) {
                FuncSpec funcSpec = it.next().second;
                Schema sch = funcSpec.getInputArgsSchema();
                for (Iterator<Integer> iter = byteArrayPositions.iterator(); iter
                        .hasNext();) {
                    Integer i = iter.next();
                    try {
                        if (!castToMap.containsKey(i)) {
                            // first candidate
                            castToMap.put(i, new Pair<FuncSpec, Byte>(funcSpec, sch
                                    .getField(i).type));
                        } else {
                            // make sure the existing type from an earlier candidate
                            // matches
                            Pair<FuncSpec, Byte> existingPair = castToMap.get(i);
                            if (sch.getField(i).type != existingPair.second) {
                                int errCode = 1046;
                                String msg = "Multiple matching functions for "
                                        + func.getFuncSpec() + " with input schema: "
                                        + "(" + existingPair.first.getInputArgsSchema()
                                        + ", " + funcSpec.getInputArgsSchema()
View Full Code Here

        // it should be a problem in the parser
        if (inputs.size() < 2) {
            throw new AssertionError("Union with Count(Operand) < 2") ;
        }

        Schema schema = null ;
        try {

            if (strictMode) {
                // Keep merging one by one just to check if there is
                // any problem with types in strict mode
                Schema tmpSchema = inputs.get(0).getSchema() ;
                for (int i=1; i< inputs.size() ;i++) {
                    // Assume the first input's aliases take precedance
                    tmpSchema = tmpSchema.merge(inputs.get(i).getSchema(), false) ;

                    // if they cannot be merged, we just give up
                    if (tmpSchema == null) {
                        int errCode = 1054;
                        String msg = "Cannot merge schemas from inputs of UNION" ;
View Full Code Here

            }
            else {

                // TODO: Don't recompute schema here
                //Schema groupBySchema = schema.getField(0).schema ;
                Schema groupBySchema = join.getTupleJoinSchema() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;
                    for(int j=0;j < innerPlans.size(); j++) {
                        LogicalPlan innerPlan = innerPlans.get(j) ;
                        byte innerType = innerPlan.getSingleLeafPlanOutputType() ;
                        byte expectedType = DataType.BYTEARRAY ;

                        if (!DataType.isAtomic(innerType) && (DataType.TUPLE != innerType)) {
                            int errCode = 1057;
                            String msg = "Join's inner plans can only"
                                         + "have one output (leaf)" ;
                            msgCollector.collect(msg, MessageType.Error) ;
                            throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                        }

                        try {
                            expectedType = groupBySchema.getField(j).type ;
                        }
                        catch(FrontendException fee) {
                            int errCode = 1060;
                            String msg = "Cannot resolve Join output schema" ;
                            msgCollector.collect(msg, MessageType.Error) ;
                            throw new TypeCheckerException(msg, errCode, PigException.INPUT, fee) ;
                        }

                        if (innerType != expectedType) {
                            insertAtomicCastForJoinInnerPlan(innerPlan,
                                                                join,
                                                                expectedType) ;
                        }
                    }
                }
            }
        }
        catch (FrontendException fe) {
            int errCode = 1060;
            String msg = "Cannot resolve Join output schema" ;
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(msg, errCode, PigException.INPUT, fe) ;
        }

        try {
            Schema outputSchema = join.regenerateSchema() ;
        }
        catch (FrontendException fe) {
            int errCode = 1060;
            String msg = "Cannot resolve Join output schema" ;
            msgCollector.collect(msg, MessageType.Error) ;
View Full Code Here

            }
            else {

                // TODO: Don't recompute schema here
                //Schema groupBySchema = schema.getField(0).schema ;
                Schema groupBySchema = cg.getTupleGroupBySchema() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(groupByPlans.get(input)) ;
                    for(int j=0;j < innerPlans.size(); j++) {
                        LogicalPlan innerPlan = innerPlans.get(j) ;
                        byte innerType = innerPlan.getSingleLeafPlanOutputType() ;
                        byte expectedType = DataType.BYTEARRAY ;

                        if (!DataType.isAtomic(innerType) && (DataType.TUPLE != innerType)) {
                            int errCode = 1061;
                            String msg = "Sorry, group by complex types"
                                       + " will be supported soon" ;
                            msgCollector.collect(msg, MessageType.Error) ;
                            throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                        }

                        try {
                            expectedType = groupBySchema.getField(j).type ;
                        }
                        catch(FrontendException fee) {
                            int errCode = 1060;
                            String msg = "Cannot resolve COGroup output schema" ;
                            msgCollector.collect(msg, MessageType.Error) ;
                            throw new TypeCheckerException(msg, errCode, PigException.INPUT, fee) ;
                        }

                        if (innerType != expectedType) {
                            insertAtomicCastForCOGroupInnerPlan(innerPlan,
                                                                cg,
                                                                expectedType) ;
                        }
                    }
                }
            }
        }
        catch (FrontendException fe) {
            int errCode = 1060;
            String msg = "Cannot resolve COGroup output schema" ;
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(msg, errCode, PigException.INPUT, fe) ;
        }

        // TODO: Don't recompute schema here. Remove all from here!
        // Generate output schema based on the schema generated from
        // COGroup itself

        try {
            Schema outputSchema = cg.regenerateSchema() ;
        }
        catch (FrontendException fe) {
            int errCode = 1060;
            String msg = "Cannot resolve COGroup output schema" ;
            msgCollector.collect(msg, MessageType.Error) ;
View Full Code Here

                    }
                }
            }
        }

        return new Schema(fsList) ;
    }
View Full Code Here

                    Schema.FieldSchema fs = Schema.FieldSchema.copyAndLink(((ExpressionOperator)op).getFieldSchema(), op);
                    if(DataType.isSchemaType(fs.type)) {
                        mSchema = fs.schema;
                    } else {
                        fss.add(fs);
                        mSchema = new Schema(fss);
                    }
                } else {
                    if (op.getSchema()!=null) {
                        mSchema = Schema.copyAndLink(op.getSchema(), op);
                    }
View Full Code Here

    public ProjectionMap getProjectionMap() {
       
        if(mIsProjectionMapComputed) return mProjectionMap;
        mIsProjectionMapComputed = true;
       
        Schema outputSchema;
        try {
            outputSchema = getSchema();
        } catch (FrontendException fee) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        Schema inputSchema = null;       
       
        List<LogicalOperator> predecessors = (ArrayList<LogicalOperator>)mPlan.getPredecessors(this);
        if(predecessors != null) {
            try {
                inputSchema = predecessors.get(0).getSchema();
View Full Code Here

            throw new TypeCheckerException(msg, errCode, PigException.INPUT);
        }

        // retrieve input schema to be casted
        // this will be used later
        Schema fromSchema = null ;
        try {
            fromSchema = fromOp.getSchema() ;
        }
        catch(FrontendException fe) {
            int errCode = 1055;
            String msg = "Problem while reading schema from input of " + fromOp.getClass().getSimpleName();
            throw new TypeCheckerException(msg, errCode, PigException.BUG, fe);
        }

        // make sure the supplied targetSchema has the same number of members
        // as number of output fields from "fromOp"
        if (fromSchema.size() != targetSchema.size()) {
            int errCode = 1078;
            String msg = "Schema size mismatch for casting. Input schema size: " + fromSchema.size() + ". Target schema size: " + targetSchema.size();
            throw new TypeCheckerException(msg, errCode, PigException.INPUT);
        }

        // Plans inside Generate. Fields that do not need casting will only
        // have Project. Fields that need casting will have Project + Cast
        ArrayList<LogicalPlan> generatePlans = new ArrayList<LogicalPlan>() ;

        int castNeededCounter = 0 ;
        for(int i=0;i < fromSchema.size(); i++) {

            LogicalPlan genPlan = new LogicalPlan() ;
            LOProject project = new LOProject(genPlan,
                                              genNewOperatorKey(fromOp),
                                              fromOp,
                                              i) ;
            project.setSentinel(true);
            genPlan.add(project);

            // add casting if necessary by comparing target types
            // to the input schema
            FieldSchema fs = null ;
            try {
                fs = fromSchema.getField(i) ;
            }
            catch(FrontendException fee) {
                int errCode = 1063;
                String msg = "Problem while reading"
                                + " field schema from input while"
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.schema.Schema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.