Package org.apache.pig

Examples of org.apache.pig.StoreFuncInterface

167
168
169
170
171
172
173
174
175
176
177
178
179
     * @throws IOException on failure
     */
    public static void setLocation(JobContext jobContext, POStore store) throws
    IOException {
        Job storeJob = new Job(jobContext.getConfiguration());
        StoreFuncInterface storeFunc = store.getStoreFunc();
        String outputLocation = store.getSFile().getFileName();
        storeFunc.setStoreLocation(outputLocation, storeJob);
       
        // the setStoreLocation() method would indicate to the StoreFunc
        // to set the output location for its underlying OutputFormat.
        // Typically OutputFormat's store the output location in the
        // Configuration - so we need to get the modified Configuration
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
                    jobcontext.getConfiguration(), jobcontext.getJobID());
           
            // set output location
            PigOutputFormat.setLocation(jobContextCopy, store);
           
            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
           
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            try {
                of.checkOutputSpecs(jobContextCopy);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
                    try {
                        fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), interStorageFuncSpec );                    // TODO: need to hookup the pigcontext.
                    } catch (IOException e) {
                        throw new PlanValidationException( expr, "Failed to process scalar" + e);
                    }
                    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(interStorageFuncSpec);
                    String sig = LogicalPlanBuilder.newOperatorKey(scope);
                    stoFunc.setStoreFuncUDFContextSignature(sig);
                    store = new LOStore(lp, fileSpec, stoFunc, sig);
                    store.setTmpStore(true);
                    lp.add( store );
                    lp.connect( refOp, store );
                }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
        pigCtx = pigContext;
    }

    @Override
    public void visit(LOStore store) throws FrontendException {
        StoreFuncInterface sf = store.getStoreFunc();
        String outLoc = store.getOutputSpec().getFileName();
        int errCode = 2116;
        String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
        Job dummyJob;
       
        try {
            if(store.getSchema() != null){
                sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));               
            }
            dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
            sf.setStoreLocation(outLoc, dummyJob);
        } catch (Exception ioe) {
            if(ioe instanceof PigException){
                errCode = ((PigException)ioe).getErrorCode();
            }
            String exceptionMsg = ioe.getMessage();
            validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
            throw new VisitorException(store, validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
        }
       
        validationErrStr += " More info to follow:\n";
        try {
            sf.getOutputFormat().checkOutputSpecs(dummyJob);
        } catch (IOException ioe) {
            byte errSrc = pigCtx.getErrorSource();
            switch(errSrc) {
            case PigException.BUG:
                errCode = 2002;
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
   
    public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
        Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
        // create a simulated JobContext
        JobContext jc = HadoopShims.createJobContext(conf, new JobID());
        StoreFuncInterface sfunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(
                storeFuncSpec);
        OutputFormat<?,?> of = sfunc.getOutputFormat();
       
        POStore store = new POStore(new OperatorKey());
        store.setSFile(new FileSpec(file, storeFuncSpec));
        PigOutputFormat.setLocation(jc, store);
        OutputCommitter oc;
        // create a simulated TaskAttemptContext
        TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, new TaskAttemptID());
        PigOutputFormat.setLocation(tac, store);
        RecordWriter<?,?> rw ;
        try {
            of.checkOutputSpecs(jc);
            oc = of.getOutputCommitter(tac);
            oc.setupJob(jc);
            oc.setupTask(tac);
            rw = of.getRecordWriter(tac);
            sfunc.prepareToWrite(rw);
       
            for (Iterator<Tuple> it = data.iterator(); it.hasNext();) {
                Tuple row = it.next();
                sfunc.putNext(row);
            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
View Full Code Here
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
           
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);
           
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers =
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
           
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
        return contextCopy;  
    }

    static void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
View Full Code Here
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
            FuncSpec instantiatedFuncSpec =
                    funcSpec == null ?
                            new FuncSpec(PigStorage.class.getName()):
                            funcSpec;

            StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(instantiatedFuncSpec);
            String fileNameKey = inputAlias + "_" + (storeIndex++) ;

            String signature = inputAlias + "_" + newOperatorKey();
            stoFunc.setStoreFuncUDFContextSignature(signature);

            String absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = stoFunc.relToAbsPathForStoreLocation(
                        filename,
                        QueryParserUtils.getCurrentDir(pigContext));
                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers(absolutePath, pigContext);
                }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
                    try {
                        fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), funcSpec );                    // TODO: need to hookup the pigcontext.
                    } catch (IOException e) {
                        throw new PlanValidationException( expr, "Failed to process scalar" + e);
                    }
                    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
                    String sig = LogicalPlanBuilder.newOperatorKey(scope);
                    stoFunc.setStoreFuncUDFContextSignature(sig);
                    store = new LOStore(lp, fileSpec, stoFunc, sig);
                    store.setTmpStore(true);
                    lp.add( store );
                    lp.connect( refOp, store );
                    expr.setImplicitReferencedOperator(store);
View Full Code Here
TOP

Related Classes of org.apache.pig.StoreFuncInterface

  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchPOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.impl.io.PigFile
  • org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor
  • org.apache.pig.newplan.logical.rules.InputOutputFileValidator$InputOutputFileVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.