Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOStore


                Operator refOp = expr.getImplicitReferencedOperator();
                Operator attachedOp = expr.getAttachedLogicalOperator();
                LogicalPlan lp = (LogicalPlan) attachedOp.getPlan();
                List<Operator> succs = lp.getSuccessors( refOp );
                LOStore store = null;
                if( succs != null ) {
                    for( Operator succ : succs ) {
                        if( succ instanceof LOStore ) {
                            store = (LOStore)succ;
                            break;
                        }
                    }
                }

                if( store == null ) {
                    FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
                    FileSpec fileSpec;
                    try {
                        fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), funcSpec );                    // TODO: need to hookup the pigcontext.
                    } catch (IOException e) {
                        throw new PlanValidationException( expr, "Failed to process scalar" + e);
                    }
                    store = new LOStore( lp, fileSpec );
                    store.setTmpStore(true);
                    lp.add( store );
                    lp.connect( refOp, store );
                    expr.setImplicitReferencedOperator(store);
                }

                filenameConst.setValue( store.getOutputSpec().getFileName() );
               
                if( lp.getSoftLinkSuccessors( store ) == null ||
                    !lp.getSoftLinkSuccessors( store ).contains( attachedOp ) ) {
                    lp.createSoftLink( store, attachedOp );
                }
View Full Code Here


            long outputUid = scalarExp.getFieldSchema().uid;
            boolean foundInput = false; // a variable to do sanity check on num of input relations

            //find the input relation, and use it to get lineage
            for(Operator softPred : softPreds){
                LOStore inputStore = (LOStore) softPred;
                if(inputStore.getFileSpec().getFileName().equals(inputFile)){
                   
                    if(foundInput == true){
                        throw new FrontendException(
                                "More than one input found for scalar expression",
                                2268,
                                PigException.BUG
                        );
                    }
                    foundInput = true;
                   
                    //found the store corresponding to this scalar expression
                    LogicalSchema sch = inputStore.getSchema();
                    if(sch == null){
                        //see if there is a load function associated with the store
                        FuncSpec funcSpec = rel2InputFuncMap.get(inputStore);
                        addUidLoadFuncToMap(outputUid, funcSpec);
                    }else{
View Full Code Here

            String query = "A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);" +
            "B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
            "C = join A by id left, B by id using 'merge';" +
            "store C into 'out';";
            LogicalPlan lp = Util.buildLp(pigServer, query);
            LOStore store = (LOStore)lp.getSinks().get(0);
            LOJoin join = (LOJoin)lp.getPredecessors(store).get(0);
            assertEquals(LOJoin.JOINTYPE.MERGE, join.getJoinType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
View Full Code Here

        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }
       
        FileSpec fileSpec = new FileSpec( absPath, funcSpec );
        LOStore op = new LOStore( plan, fileSpec );
        return buildOp( loc, op, alias, inputAlias, null );
    }
View Full Code Here

        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
        stoFunc.setStoreFuncUDFContextSignature( LOStore.constructSignature( alias, fileName, funcSpec ) );

        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        LOStore store = new LOStore( lp, fileSpec );
        store.setAlias( alias );

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendExceptionioe.getMessage(), ioe );
View Full Code Here

            "b = foreach a generate *;" +
            "store b into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        assert(loForEach.getSchema()==null);
    }
View Full Code Here

            "B = foreach A generate u.$100;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
View Full Code Here

            "B = foreach A generate u.$100;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        Assert.assertTrue(schema.getField(0).type==DataType.BYTEARRAY);
    }
View Full Code Here

            "B = foreach A { B1 = filter u by $1==0; generate B1;};" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
View Full Code Here

            "B = order A by *;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOSort loSort = (LOSort)lp.getPredecessors(loStore).get(0);
        Operator sortPlanLeaf = loSort.getSortColPlans().get(0).getSources().get(0);
        LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
        Assert.assertTrue(sortPlanFS==null);
       
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOStore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.