Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOStore

410
411
412
413
414
415
416
417
        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }
       
        FileSpec fileSpec = new FileSpec( absPath, funcSpec );
        LOStore op = new LOStore( plan, fileSpec );
        return buildOp( loc, op, alias, inputAlias, null );
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
        stoFunc.setStoreFuncUDFContextSignature( LOStore.constructSignature( alias, fileName, funcSpec ) );

        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        LOStore store = new LOStore( lp, fileSpec );
        store.setAlias( alias );

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
       
       
        f1.setAlias("D");
        plan.add(f1);
       
        LogicalRelationalOperator s1 = new LOStore(plan);
        plan.add(s1);      
       
        // load --|-join - filter - store
        // load --|  
        plan.connect(l1, j1);
View Full Code Here
928
929
930
931
932
933
934
935
936
937
938
                }
                fileNameMap.put(fileNameKey, absolutePath);
            }
            FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);

            LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
            return buildOp(loc, op, alias, inputAlias, null);
        } catch(Exception ex) {
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
View Full Code Here
221
222
223
224
225
226
227
228
229
230
231
            + "D = foreach A generate owner, age/(double)C.total AS percentAge; "
            + "F = LIMIT D C.total/8;"
            + "store F into 'output';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);;
        optimizePlan(newLogicalPlan);
        LOStore store = (LOStore)newLogicalPlan.getSinks().get(0);
        LOForEach foreach1 = (LOForEach)newLogicalPlan.getPredecessors(store).get(0);
        LOForEach foreach2 = (LOForEach)newLogicalPlan.getPredecessors(foreach1).get(0);
        LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach2).get(0);
        Assert.assertTrue(newLogicalPlan.getSoftLinkPredecessors(limit).get(0) instanceof LOStore);
    }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
       
       
        f1.setAlias("D");
        plan.add(f1);
       
        LogicalRelationalOperator s1 = new LOStore(plan, null, null, null);
        plan.add(s1);      
       
        // load --|-join - filter - store
        // load --|  
        plan.connect(l1, j1);
View Full Code Here
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
                }
                fileNameMap.put(fileNameKey, absolutePath);
            }
            FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);

            LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
            return buildOp(loc, op, alias, inputAlias, null);
        } catch(Exception ex) {
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
        stoFunc.setStoreFuncUDFContextSignature(sig);
        LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
        store.setAlias(alias);

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
View Full Code Here
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
            long outputUid = scalarExp.getFieldSchema().uid;
            boolean foundInput = false; // a variable to do sanity check on num of input relations

            //find the input relation, and use it to get lineage
            for(Operator softPred : softPreds){
                LOStore inputStore = (LOStore) softPred;
                if(inputStore.getFileSpec().getFileName().equals(inputFile)){
                   
                    if(foundInput == true){
                        throw new FrontendException(
                                "More than one input found for scalar expression",
                                2268,
                                PigException.BUG
                        );
                    }
                    foundInput = true;
                   
                    //found the store corresponding to this scalar expression
                    LogicalSchema sch = inputStore.getSchema();
                    if(sch == null){
                        //see if there is a load function associated with the store
                        FuncSpec funcSpec = rel2InputFuncMap.get(inputStore);
                        addUidLoadFuncToMap(outputUid, funcSpec);
                    }else{
View Full Code Here
985
986
987
988
989
990
991
992
993
994
995
                }
                fileNameMap.put(fileNameKey, absolutePath);
            }
            FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);

            LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
            return buildOp(loc, op, alias, inputAlias, null);
        } catch(Exception ex) {
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOStore

  • org.apache.pig.newplan.logical.visitor.LineageFindRelVisitor$LineageFindExpVisitor
  • org.apache.pig.newplan.logical.visitor.ScalarVisitor
  • org.apache.pig.parser.LogicalPlanBuilder
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.penny.ParsedPigScript
  • org.apache.pig.test.TestBlackAndWhitelistValidator
  • org.apache.pig.test.TestCollectedGroup
  • org.apache.pig.test.TestInputOutputFileValidator
  • org.apache.pig.test.TestInputOutputMiniClusterFileValidator
  • org.apache.pig.test.TestLogicalPlanMigrationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.