Examples of org.apache.pig.impl.logicalLayer.LOStream

Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOStream

org.apache.pig.impl.logicalLayer.LOStream
{@link LOStream} represents the specification of an externalcommand to be executed in a Pig Query. LOStream encapsulates all relevant details of the command specified by the user either directly via the STREAM operator or indirectly via a DEFINE operator. It includes details such as input/output/error specifications and also files to be shipped to the cluster and files to be cached.

    @Test
    public void testQueryStreamingNoSchema() throws FrontendException {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "`;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingNoSchema1() throws FrontendException {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` ;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingWithSchema() throws FrontendException {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingWithSchema1() throws FrontendException {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

            String msg = "Expected " + LOStream.class.getSimpleName()
                    + ", got "
                    + (lo == null ? lo : lo.getClass().getSimpleName());
            throw new OptimizerException(msg, errCode, PigException.BUG);            
        }
        LOStream stream = (LOStream)lo;
        
        // check if either the predecessor of stream is load with
        // the same loader function as the serializer of stream
        // OR if the successor of stream is store with the same
        // storage function as the deserializer of stream.

View Full Code Here

            LogicalOperator lo = nodes.get(0);
            if (lo == null || !(lo instanceof LOStream)) {
                throw new RuntimeException("Expected stream, got " +
                    lo.getClass().getName());
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization"; 
            throw new OptimizerException(msg, errCode, PigException.BUG, e);

View Full Code Here

    @Test
    public void testQueryStreamingNoSchema() {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "`;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingNoSchema1() {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` ;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingWithSchema() {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

    @Test
    public void testQueryStreamingWithSchema1() {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOStream

org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer

org.apache.pig.impl.logicalLayer.schema.Schema

org.apache.pig.impl.plan.ProjectionMap

org.apache.pig.impl.plan.RequiredFields

org.apache.pig.impl.streaming.StreamingCommand

org.apache.pig.impl.streaming.StreamingCommand.HandleSpec

org.apache.pig.test.TestRelevantFields

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.