Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOStore


        assertTrue(lp.size()>1);
        Operator op = lp.getSinks().get(0);

        assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getFileSpec().getFileName();
        p = p.replaceAll("hdfs://[\\-\\w:\\.]*/","/");

        if (isTmp) {
            assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here


            String query = "A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);" +
            "B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
            "C = join A by id left, B by id using 'merge';" +
            "store C into 'out';";
            LogicalPlan lp = Util.buildLp(pigServer, query);
            LOStore store = (LOStore)lp.getSinks().get(0);
            LOJoin join = (LOJoin)lp.getPredecessors(store).get(0);
            assertEquals(LOJoin.JOINTYPE.MERGE, join.getJoinType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
View Full Code Here

            String outputFile, DataStorage dfs) throws Exception {
        LogicalPlan plan = new LogicalPlan();
        FileSpec filespec1 = new FileSpec(generateTmpFile(inputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        FileSpec filespec2 = new FileSpec(generateTmpFile(outputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad(filespec1, null, plan, ConfigurationUtil.toConfiguration(dfs.getConfiguration()));
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface) PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null);

        LOFilter filter = new LOFilter(plan);

        plan.add(load);
        plan.add(store);
View Full Code Here

            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad( filespec1, null, plan,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;

        plan.add(load) ;
        plan.add(store) ;

        plan.connect(load, store) ;
View Full Code Here

            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad( filespec1, null, plan,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;

        plan.add(load) ;
        plan.add(store) ;

        plan.connect(load, store) ;
View Full Code Here

       
        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
        stoFunc.setStoreFuncUDFContextSignature(sig);
        LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
        store.setAlias(alias);

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendExceptionioe.getMessage(), ioe );
View Full Code Here

                }
                fileNameMap.put(fileNameKey, absolutePath);
            }
            FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);

            LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
            return buildOp(loc, op, alias, inputAlias, null);
        } catch(Exception ex) {
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
View Full Code Here

                Operator refOp = expr.getImplicitReferencedOperator();
                Operator attachedOp = expr.getAttachedLogicalOperator();
                LogicalPlan lp = (LogicalPlan) attachedOp.getPlan();
                List<Operator> succs = lp.getSuccessors( refOp );
                LOStore store = null;
                FuncSpec interStorageFuncSpec = new FuncSpec(InterStorage.class.getName());
                if( succs != null ) {
                    for( Operator succ : succs ) {
                        if( succ instanceof LOStore
                                && ((LOStore)succ).isTmpStore()
                                && interStorageFuncSpec.equals(
                                    ((LOStore)succ).getOutputSpec().getFuncSpec() ) ) {
                            store = (LOStore)succ;
                            break;
                        }
                    }
                }

                if( store == null ) {
                    FileSpec fileSpec;
                    try {
                        fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), interStorageFuncSpec );                    // TODO: need to hookup the pigcontext.
                    } catch (IOException e) {
                        throw new PlanValidationException( expr, "Failed to process scalar" + e);
                    }
                    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(interStorageFuncSpec);
                    String sig = LogicalPlanBuilder.newOperatorKey(scope);
                    stoFunc.setStoreFuncUDFContextSignature(sig);
                    store = new LOStore(lp, fileSpec, stoFunc, sig);
                    store.setTmpStore(true);
                    lp.add( store );
                    lp.connect( refOp, store );
                    expr.setImplicitReferencedOperator(store);
                }

                filenameConst.setValue( store.getOutputSpec().getFileName() );
               
                if( lp.getSoftLinkSuccessors( store ) == null ||
                    !lp.getSoftLinkSuccessors( store ).contains( attachedOp ) ) {
                    lp.createSoftLink( store, attachedOp );
                }
View Full Code Here

                "b = foreach a generate *;" +
                "store b into '111';";

        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        assertNull(loForEach.getSchema());
    }
View Full Code Here

                "B = foreach A generate u.$100;" +
                "store B into '111';";

        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        assertEquals(1, schema.size());
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        assertEquals(DataType.BAG, bagFieldSchema.type);
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOStore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.