Package org.apache.pig

Examples of org.apache.pig.StoreFuncInterface

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
            super(plan, new DepthFirstWalker(plan));
        }

        @Override
        public void visit(LOStore store) throws FrontendException {
            StoreFuncInterface sf = store.getStoreFunc();
            String outLoc = store.getOutputSpec().getFileName();
            int errCode = 2116;
            String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
            Job dummyJob;
           
            try {
                if(store.getSchema() != null){
                    sf.checkSchema(new ResourceSchema(Util.translateSchema(store.getSchema()), store.getSortInfo()));               
                }
                dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
                sf.setStoreLocation(outLoc, dummyJob);
            } catch (IOException ioe) {
                if(ioe instanceof PigException){
                    errCode = ((PigException)ioe).getErrorCode();
                }
                String exceptionMsg = ioe.getMessage();
                validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
                throw new FrontendException(validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
            }
           
            validationErrStr += " More info to follow:\n";
            try {
                sf.getOutputFormat().checkOutputSpecs(dummyJob);
            } catch (IOException ioe) {
                byte errSrc = pigCtx.getErrorSource();
                switch(errSrc) {
                case PigException.BUG:
                    errCode = 2002;
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
     * to output format implementation.
     */
    @Override
    protected void visit(LOStore store) throws PlanValidationException{

        StoreFuncInterface sf = store.getStoreFunc();
        String outLoc = store.getOutputFile().getFileName();
        int errCode = 2116;
        String validationErrStr ="Output Location Validation Failed for: " + outLoc ;
        Job dummyJob;
       
        try {
            if(store.getSchema() != null){
                sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));               
            }
            dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
            sf.setStoreLocation(outLoc, dummyJob);
        } catch (IOException ioe) {
             if(ioe instanceof PigException){
            errCode = ((PigException)ioe).getErrorCode();
          }
          String exceptionMsg = ioe.getMessage();
          validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
          msgCollector.collect(validationErrStr, MessageType.Error) ;
            throw new PlanValidationException(validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
        }
       
        validationErrStr += " More info to follow:\n";
        try {
            sf.getOutputFormat().checkOutputSpecs(dummyJob);
        } catch (IOException ioe) {
            byte errSrc = pigCtx.getErrorSource();
            switch(errSrc) {
            case PigException.BUG:
                errCode = 2002;
View Full Code Here
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
           
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);
           
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
   
    public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
        Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
        // create a simulated JobContext
        JobContext jc = new JobContext(conf, new JobID());
        StoreFuncInterface sfunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(
                storeFuncSpec);
        OutputFormat<?,?> of = sfunc.getOutputFormat();
       
        POStore store = new POStore(new OperatorKey());
        store.setSFile(new FileSpec(file, storeFuncSpec));
        PigOutputFormat.setLocation(jc, store);
        OutputCommitter oc;
        // create a simulated TaskAttemptContext
        TaskAttemptContext tac = new TaskAttemptContext(conf, new TaskAttemptID());
        PigOutputFormat.setLocation(tac, store);
        RecordWriter<?,?> rw ;
        try {
            of.checkOutputSpecs(jc);
            oc = of.getOutputCommitter(tac);
            oc.setupJob(jc);
            oc.setupTask(tac);
            rw = of.getRecordWriter(tac);
            sfunc.prepareToWrite(rw);
       
            for (Iterator<Tuple> it = data.iterator(); it.hasNext();) {
                Tuple row = it.next();
                sfunc.putNext(row);
            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
            if(mapStores.size() == 1) {
                store = mapStores.get(0);
            } else {
                store = reduceStores.get(0);
            }
            StoreFuncInterface sFunc = store.getStoreFunc();
            // set output location
            PigOutputFormat.setLocation(taskattemptcontext, store);
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            RecordWriter writer = sFunc.getOutputFormat().getRecordWriter(
                    taskattemptcontext);
            return new PigRecordWriter(writer, sFunc, Mode.SINGLE_STORE);
        } else {
           // multi store case - in this case, all writing is done through
           // MapReducePOStoreImpl - set up a dummy RecordWriter
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
     * @throws IOException on failure
     */
    public static void setLocation(JobContext jobContext, POStore store) throws
    IOException {
        Job storeJob = new Job(jobContext.getConfiguration());
        StoreFuncInterface storeFunc = store.getStoreFunc();
        String outputLocation = store.getSFile().getFileName();
        storeFunc.setStoreLocation(outputLocation, storeJob);
       
        // the setStoreLocation() method would indicate to the StoreFunc
        // to set the output location for its underlying OutputFormat.
        // Typically OutputFormat's store the output location in the
        // Configuration - so we need to get the modified Configuration
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
                    jobcontext.getConfiguration(), jobcontext.getJobID());
           
            // set output location
            PigOutputFormat.setLocation(jobContextCopy, store);
           
            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
           
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
   
    @Override
    public StoreFuncInterface createStoreFunc(POStore store)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();

        // call the setStoreLocation on the storeFunc giving it the
        // Job. Typically this will result in the OutputFormat of the
        // storeFunc storing the output location in the Configuration
        // in the Job. The PigOutFormat.setLocation() method will merge
        // this modified Configuration into the configuration of the
        // Context we have
        PigOutputFormat.setLocation(context, store);
        OutputFormat outputFormat = storeFunc.getOutputFormat();

        // create a new record writer
        try {
            writer = outputFormat.getRecordWriter(context);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        storeFunc.prepareToWrite(writer);
       
        return storeFunc;
    }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers =
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
           
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
        return contextCopy;  
    }

    static void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
View Full Code Here
TOP

Related Classes of org.apache.pig.StoreFuncInterface

  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchPOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.impl.io.PigFile
  • org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor
  • org.apache.pig.newplan.logical.rules.InputOutputFileValidator$InputOutputFileVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.