Package org.apache.pig

Examples of org.apache.pig.StoreFunc


        }
        // In single store case, we would have removed the store from the
        // plan in JobControlCompiler
        if(sFuncString != null && (mapStores.size() + reduceStores.size() == 0)) {
            // single store case
            StoreFunc sFunc = MapRedUtil.getStoreFunc(new JobConf(conf));
            commit(sFunc, conf, conf.get(JobControlCompiler.PIG_STORE_CONFIG),
                    sFuncString);
        } else {
            // multi store case
            commitStores(mapStores, conf);
View Full Code Here


    }
   
    private void commitStores(List<POStore> stores, Configuration conf)
    throws IOException {
        for (POStore store : stores) {
            StoreFunc sFunc = (StoreFunc)PigContext.instantiateFuncFromSpec(
                    store.getSFile().getFuncSpec());
            StoreConfig storeConfig = new StoreConfig(store.getSFile().
                    getFileName(), store.getSchema(), store.getSortInfo());
            commit(sFunc, conf, storeConfig,
                    store.getSFile().getFuncSpec().toString());
View Full Code Here

    @Test
    public void testSFPig() throws Exception {
        byte[] buf = new byte[1024];
        FakeFSOutputStream os = new FakeFSOutputStream(buf);
        StoreFunc sfunc = new PigStorage("\t");
        sfunc.bindTo(os);

        int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
        Tuple f1 = Util.loadFlatTuple(new Tuple(input.length), input);

        sfunc.putNext(f1);
        sfunc.finish();
       
        FakeFSInputStream is = new FakeFSInputStream(buf);
        LoadFunc lfunc = new PigStorage();
        lfunc.bindTo(null, new BufferedPositionedInputStream(is), 0, buf.length);
        Tuple f2 = lfunc.getNext();
View Full Code Here

        Tuple t;
        while ((t = (Tuple) ((PhysicalOperator)opTable.get(inputs[0])).getNext()) != null) {
            b.add(t);
        }
        try {
            StoreFunc func = (StoreFunc) PigContext.instantiateFuncFromSpec(funcSpec);
            f.store(b, func, pigContext);
           
            // a result has materialized, track it!
            LocalResult materializedResult = new LocalResult(this.outFileSpec);
View Full Code Here

                // Instantiate both to compare them for equality
                LoadFunc streamLoader =
                    (LoadFunc)PigContext.instantiateFuncFromSpec(
                            streamOutputSpec.getSpec());
               
                StoreFunc outputStorer = (StoreFunc)PigContext.instantiateFuncFromSpec(
                                                storeFileSpec.getFuncSpec());
               

                // Check if the streaming command's outputSpec also implements
                // StoreFunc and if it does, are they of the same _reversible_
View Full Code Here

                HandleSpec streamInputSpec = command.getInputSpec();
               
                FileSpec loadFileSpec = load.getInputFileSpec();
               
                // Instantiate both to compare them for equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
               
                LoadFunc inputLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(
                                                loadFileSpec.getFuncSpec());

                // Check if the streaming command's inputSpec also implements
                // LoadFunc and if it does, are they of the same _reversible_
                // type?
                boolean sameType = false;
                try {
                    // Check if the streamStorer is _reversible_ as
                    // the inputLoader ...
                    if (streamStorer instanceof LoadFunc) {
                        // Cast to check if they are of the same type...
                        streamStorer.getClass().cast(inputLoader);
                       
                        // Now check if they both are reversible...
                        if (streamStorer instanceof ReversibleLoadStoreFunc &&
                            inputLoader instanceof ReversibleLoadStoreFunc) {
                            sameType = true;
                        }
                    }
                } catch (ClassCastException cce) {
                    sameType = false;
                }
               
                // Check if both LoadFunc objects belong to the same type and
                // are equivalent
                if (sameType && streamStorer.equals(inputLoader)) {
                    // Since they both are the same, we can flip them
                    // for BinaryStorage
                    load.setInputFileSpec(new FileSpec(loadFileSpec.getFileName(), BinaryStorage.class.getName()));
                    streamSpec.setOptimizedSpec(Handle.INPUT,
                                                   BinaryStorage.class.getName());
View Full Code Here

        this.outputFileSpec = fileSpec;
        this.append = append;

        //See if the store function spec is valid
        try {
            StoreFunc StoreFunc =
                (StoreFunc) PigContext.instantiateFuncFromSpec(
                    fileSpec.getFuncSpec());
        } catch(Exception e) {
            IOException ioe = new IOException(e.getMessage());
            ioe.setStackTrace(e.getStackTrace());
View Full Code Here

        return getRecordWriter(fs, job, outputDir, name, progress);
    }

    public PigRecordWriter getRecordWriter(FileSystem fs, JobConf job, Path outputDir, String name, Progressable progress)
            throws IOException {
        StoreFunc store;
        String storeFunc = (String) ObjectSerializer.deserialize(job.get("pig.storeFunc", "")) ;
        if (storeFunc.length() == 0) {
            store = new PigStorage();
        } else {
            store = (StoreFunc) PigContext.instantiateFuncFromSpec(storeFunc);
View Full Code Here

                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
                LoadFunc inputLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(
                                             loadFileSpec.getFuncSpec());
                LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                        "inputLoader:" + inputLoader);
                // Check if the streaming command's inputSpec also implements
                // LoadFunc and if it does, are they of the same _reversible_
                // type?
                boolean sameType = false;
                try {
                    // Check if the streamStorer is _reversible_ as
                    // the inputLoader ...
                    if (streamStorer instanceof LoadFunc) {
                        // Cast to check if they are of the same type...
                        streamStorer.getClass().cast(inputLoader);
                        LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                                "inputLoader:" + inputLoader);
                        // Now check if they both are reversible...
                        if (streamStorer instanceof ReversibleLoadStoreFunc &&
                            inputLoader instanceof ReversibleLoadStoreFunc) {
                            sameType = true;
                        }
                    }
                } catch (ClassCastException cce) {
                    sameType = false;
                }
                // Check if both LoadFunc objects belong to the same type and
                // are equivalent
                if (sameType && streamStorer.equals(inputLoader)) {
                    // Since they both are the same, we can flip them
                    // for BinaryStorage
                    mOptimizeLoad = true;                   
                }
            }
View Full Code Here

            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
           
            StoreFunc outputStorer = (StoreFunc)PigContext.instantiateFuncFromSpec(
                                         storeFileSpec.getFuncSpec());
           
            // Check if the streaming command's outputSpec also implements
            // StoreFunc and if it does, are they of the same _reversible_
            // type?
View Full Code Here

TOP

Related Classes of org.apache.pig.StoreFunc

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.