Package org.apache.pig

Examples of org.apache.pig.StoreFuncInterface

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
        if( func == null ) {
            func = PigStorage.class.getName();
        }

        FuncSpec funcSpec = new FuncSpec( func );
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
       
        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
        stoFunc.setStoreFuncUDFContextSignature(sig);
        LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
        store.setAlias(alias);

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
            throw e;
        }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers =
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
           
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
        return contextCopy;  
    }

    static public void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
View Full Code Here
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
            LinkedList<POStore> mapStores = PlanHelper.getPhysicalOperators(mro.mapPlan, POStore.class);
            LinkedList<POStore> reduceStores = PlanHelper.getPhysicalOperators(mro.reducePlan, POStore.class);

            for (POStore st : mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
                if (sFunc instanceof OverwritableStoreFunc) {
                    OverwritableStoreFunc osf = (OverwritableStoreFunc) sFunc;
                    if (osf.shouldOverwrite()) {
                        osf.cleanupOutput(st, nwJob);
                    }
                }
            }

            for (POStore st : reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
                if (sFunc instanceof OverwritableStoreFunc) {
                    OverwritableStoreFunc osf = (OverwritableStoreFunc) sFunc;
                    if (osf.shouldOverwrite()) {
                        osf.cleanupOutput(st, nwJob);
                    }
View Full Code Here
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
    String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)
    throws ParserValidationException {
        try {
            // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
            StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
            String fileNameKey = inputAlias + "_" + (storeIndex++) ;

            String signature = inputAlias + "_" + newOperatorKey();
            stoFunc.setStoreFuncUDFContextSignature(signature);

            String absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = stoFunc.relToAbsPathForStoreLocation(
                        filename,
                        QueryParserUtils.getCurrentDir(pigContext));
                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers(absolutePath, pigContext);
                }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
            Operator input, String alias, PigContext pigContext) throws FrontendException {
        func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;

        FuncSpec funcSpec = new FuncSpec( func );
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
       
        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
        stoFunc.setStoreFuncUDFContextSignature(sig);
        LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
        store.setAlias(alias);

        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
            throw e;
        }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
                    try {
                        fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), interStorageFuncSpec );                    // TODO: need to hookup the pigcontext.
                    } catch (IOException e) {
                        throw new PlanValidationException( expr, "Failed to process scalar" + e);
                    }
                    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(interStorageFuncSpec);
                    String sig = LogicalPlanBuilder.newOperatorKey(scope);
                    stoFunc.setStoreFuncUDFContextSignature(sig);
                    store = new LOStore(lp, fileSpec, stoFunc, sig);
                    store.setTmpStore(true);
                    lp.add( store );
                    lp.connect( refOp, store );
                    expr.setImplicitReferencedOperator(store);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
            super(plan, new DepthFirstWalker(plan));
        }

        @Override
        public void visit(LOStore store) throws FrontendException {
            StoreFuncInterface sf = store.getStoreFunc();
            String outLoc = store.getOutputSpec().getFileName();
            int errCode = 2116;
            String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
            Job dummyJob;
           
            try {
                if(store.getSchema() != null){
                    sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));               
                }
                dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
                sf.setStoreLocation(outLoc, dummyJob);
            } catch (Exception ioe) {
                if(ioe instanceof PigException){
                    errCode = ((PigException)ioe).getErrorCode();
                }
                String exceptionMsg = ioe.getMessage();
                validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
                throw new VisitorException(store, validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
            }
           
            validationErrStr += " More info to follow:\n";
            try {
                sf.getOutputFormat().checkOutputSpecs(dummyJob);
            } catch (IOException ioe) {
                byte errSrc = pigCtx.getErrorSource();
                switch(errSrc) {
                case PigException.BUG:
                    errCode = 2002;
View Full Code Here
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
            // PIG-3741 This must be done before setStoreLocation on POStores
            Utils.setTmpFileCompressionOnConf(pigContext, conf);

            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers =
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
           
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.StoreFuncInterface

  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchPOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.impl.io.PigFile
  • org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor
  • org.apache.pig.newplan.logical.rules.InputOutputFileValidator$InputOutputFileVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.