Examples of org.apache.pig.StoreFuncInterface

org.apache.pig.StoreFuncInterface
StoreFuncs take records from Pig's processing and store them into a data store. Most frequently this is an HDFS file, but it could also be an HBase instance, RDBMS, etc.

        LinkedList<POStore> stores = (LinkedList<POStore>) ObjectSerializer.
        deserialize(conf.get(storeLookupKey));
        List<Pair<OutputCommitter, POStore>> committers = 
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
            
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }

View Full Code Here

        return contextCopy;   
    }


    static void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()

View Full Code Here

            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
            
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);

View Full Code Here

            if(mapStores.size() == 1) {
                store = mapStores.get(0);
            } else {
                store = reduceStores.get(0);
            }
            StoreFuncInterface sFunc = store.getStoreFunc();
            // set output location
            PigOutputFormat.setLocation(taskattemptcontext, store);
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            RecordWriter writer = sFunc.getOutputFormat().getRecordWriter(
                    taskattemptcontext);
            return new PigRecordWriter(writer, sFunc, Mode.SINGLE_STORE);
        } else {
           // multi store case - in this case, all writing is done through
           // MapReducePOStoreImpl - set up a dummy RecordWriter

View Full Code Here

     * @throws IOException on failure
     */
    public static void setLocation(JobContext jobContext, POStore store) throws 
    IOException {
        Job storeJob = new Job(jobContext.getConfiguration());
        StoreFuncInterface storeFunc = store.getStoreFunc();
        String outputLocation = store.getSFile().getFileName();
        storeFunc.setStoreLocation(outputLocation, storeJob);
        
        // the setStoreLocation() method would indicate to the StoreFunc
        // to set the output location for its underlying OutputFormat.
        // Typically OutputFormat's store the output location in the
        // Configuration - so we need to get the modified Configuration

View Full Code Here

                    jobcontext.getConfiguration(), jobcontext.getJobID());
            
            // set output location
            PigOutputFormat.setLocation(jobContextCopy, store);
            
            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }

View Full Code Here

    public Schema getSchema() {
        return schema;
    }
    
    public StoreFuncInterface getStoreFunc() {
        StoreFuncInterface sFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(sFile.getFuncSpec());
        sFunc.setStoreFuncUDFContextSignature(signature);
        return sFunc;
    }

View Full Code Here

     * to output format implementation.
     */
    @Override
    protected void visit(LOStore store) throws PlanValidationException{


        StoreFuncInterface sf = store.getStoreFunc();
        String outLoc = store.getOutputFile().getFileName();
        String errMsg = "Unexpected error. Could not validate the output " +
        "specification for: "+outLoc;
        int errCode = 2116;


        try {
            if(store.getSchema() != null){
                sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));                
            }
        } catch (FrontendException e) {
            msgCollector.collect(errMsg, MessageType.Error) ;
            throw new PlanValidationException(errMsg, errCode, pigCtx.getErrorSource(), e);
        } catch (IOException e) {
            msgCollector.collect(errMsg, MessageType.Error) ;
            throw new PlanValidationException(errMsg, errCode, pigCtx.getErrorSource(), e);
        }


        Job dummyJob;
        
        try {
            dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
            sf.setStoreLocation(outLoc, dummyJob);
        } catch (IOException ioe) {
            msgCollector.collect(errMsg, MessageType.Error) ;
            throw new PlanValidationException(errMsg, errCode, pigCtx.getErrorSource(), ioe);
        }
        try {
            sf.getOutputFormat().checkOutputSpecs(dummyJob);
        } catch (IOException ioe) {
            byte errSrc = pigCtx.getErrorSource();
            errCode = 0;
            switch(errSrc) {
            case PigException.BUG:

View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public StoreFuncInterface createStoreFunc(POStore store) 
        throws IOException {


        StoreFuncInterface storeFunc = store.getStoreFunc();


        // call the setStoreLocation on the storeFunc giving it the
        // Job. Typically this will result in the OutputFormat of the
        // storeFunc storing the output location in the Configuration
        // in the Job. The PigOutFormat.setLocation() method will merge 
        // this modified Configuration into the configuration of the
        // Context we have
        PigOutputFormat.setLocation(context, store);
        OutputFormat outputFormat = null;
        try {
            outputFormat = storeFunc.getOutputFormat();


            // create a new record writer
            writer = outputFormat.getRecordWriter(context);
            storeFunc.prepareToWrite(writer);
            return storeFunc;
            
        }catch(Exception e) {
            throw new IOException(e);
        }

View Full Code Here

            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
            
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.pig.StoreFuncInterface

org.apache.pig.backend.hadoop.executionengine.fetch.FetchPOStoreImpl

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.pig.impl.io.PigFile

org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor

org.apache.pig.newplan.logical.rules.InputOutputFileValidator$InputOutputFileVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.