Examples of org.apache.pig.impl.streaming.StreamingCommand

org.apache.pig.impl.streaming.StreamingCommand
{@link StreamingCommand} represents the specification of an externalcommand to be executed in a Pig Query. StreamingCommand encapsulates all relevant details of the command specified by the user either directly via the STREAM operator or indirectly via a DEFINE operator. It includes details such as input/output/error specifications and also files to be shipped to the cluster and files to be cached.

        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec(); 
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for 
                // equality
                StoreFunc streamStorer = 
                    (StoreFunc)PigContext.instantiateFuncFromSpec(

View Full Code Here

        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec(); 
            
            FileSpec storeFileSpec = store.getOutputFile();
            
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =

View Full Code Here

            super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
        }


        @Override
        public void visitStream(POStream stream) throws VisitorException {
            StreamingCommand command = stream.getCommand();
            cacheFiles.addAll(command.getCacheSpecs());
            shipFiles.addAll(command.getShipSpecs());
        }

View Full Code Here

    }


    StreamingCommand buildCommand(SourceLocation loc, String cmd, List<String> shipPaths, List<String> cachePaths,
            List<HandleSpec> inputHandleSpecs, List<HandleSpec> outputHandleSpecs,
            String logDir, Integer limit) throws RecognitionException {
        StreamingCommand command = null;
        try {
            command = buildCommand( loc, cmd );


            // Process ship paths
            if( shipPaths != null ) {
                if( shipPaths.size() == 0 ) {
                    command.setShipFiles( false );
                } else {
                    for( String path : shipPaths )
                        command.addPathToShip( path );
                }
            }


            // Process cache paths
            if( cachePaths != null ) {
                for( String path : cachePaths )
                    command.addPathToCache( path );
            }


            // Process input handle specs
            if( inputHandleSpecs != null ) {
                for( HandleSpec spec : inputHandleSpecs )
                    command.addHandleSpec( Handle.INPUT, spec );
            }


            // Process output handle specs
            if( outputHandleSpecs != null ) {
                for( HandleSpec spec : outputHandleSpecs )
                    command.addHandleSpec( Handle.OUTPUT, spec );
            }


            // error handling
            if( logDir != null )
                command.setLogDir( logDir );
            if( limit != null )
                command.setLogFilesLimit( limit );
        } catch(IOException e) {
            throw new PlanGenerationFailureException( intStream, loc, e );
        }


        return command;

View Full Code Here

    }


    StreamingCommand buildCommand(SourceLocation loc, String cmd) throws RecognitionException {
        try {
            String[] args = StreamingCommandUtils.splitArgs( cmd );
            StreamingCommand command = new StreamingCommand( pigContext, args );
            StreamingCommandUtils validator = new StreamingCommandUtils( pigContext );
            validator.checkAutoShipSpecs( command, args );
            return command;
        } catch (ParserException e) {
            throw new InvalidCommandException( intStream, loc, cmd );

View Full Code Here

        setStreams();
        startThreads();
    }


    private StreamingCommand startUdfController() throws IOException {
        StreamingCommand sc = new StreamingCommand(null, constructCommand());
        ProcessBuilder processBuilder = StreamingUtil.createProcess(sc);
        process = processBuilder.start();


        Runtime.getRuntime().addShutdownHook(new Thread(new ProcessKiller() ) );
        return sc;

View Full Code Here

            }
        } else {
            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec(); 
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            } else if ((op instanceof LOFilter)
                    || (op instanceof LODistinct)
                    || (op instanceof LOSort)

View Full Code Here

    }


    @Override
    public void visit(LOStream stream) throws FrontendException{
        mapToPredLoadFunc(stream);
        StreamingCommand command = ((LOStream)stream).getStreamingCommand();
        HandleSpec streamOutputSpec = command.getOutputSpec(); 
        FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
        setLoadFuncForUids(stream.getSchema(), streamLoaderSpec);
        rel2InputFuncMap.put(stream, streamLoaderSpec);
    }

View Full Code Here

                    exp.add(cast);
                    FuncSpec loadFuncSpec = null;
                    if(op instanceof LOLoad) {
                        loadFuncSpec = ((LOLoad)op).getFileSpec().getFuncSpec();
                    } else if (op instanceof LOStream) {
                        StreamingCommand command = ((LOStream)op).getStreamingCommand();
                        HandleSpec streamOutputSpec = command.getOutputSpec(); 
                        loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                    } else {
                        String msg = "TypeCastInserter invoked with an invalid operator class name: " + innerPlan.getClass().getSimpleName();
                        throw new FrontendException(msg, 2242);
                    }

View Full Code Here

    }


    StreamingCommand buildCommand(SourceLocation loc, String cmd, List<String> shipPaths, List<String> cachePaths,
            List<HandleSpec> inputHandleSpecs, List<HandleSpec> outputHandleSpecs,
            String logDir, Integer limit) throws RecognitionException {
        StreamingCommand command = null;
        try {
            command = buildCommand( loc, cmd );
            
            // Process ship paths
            if( shipPaths != null ) {
                if( shipPaths.size() == 0 ) {
                    command.setShipFiles( false );
                } else {
                    for( String path : shipPaths )
                        command.addPathToShip( path );
                }
            }
            
            // Process cache paths
            if( cachePaths != null ) {
                for( String path : cachePaths )
                    command.addPathToCache( path );
            }
            
            // Process input handle specs
            if( inputHandleSpecs != null ) {
                for( HandleSpec spec : inputHandleSpecs )
                    command.addHandleSpec( Handle.INPUT, spec );
            }
            
            // Process output handle specs
            if( outputHandleSpecs != null ) {
                for( HandleSpec spec : outputHandleSpecs )
                    command.addHandleSpec( Handle.OUTPUT, spec );
            }
            
            // error handling
            if( logDir != null )
                command.setLogDir( logDir );
            if( limit != null )
                command.setLogFilesLimit( limit );
        } catch(IOException e) {
            throw new PlanGenerationFailureException( intStream, loc, e );
        }
        
        return command;

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.pig.impl.streaming.StreamingCommand

org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOStreamVisitor$StreamFileVisitor

org.apache.pig.impl.builtin.StreamingUDF

org.apache.pig.impl.logicalLayer.LOStream

org.apache.pig.impl.logicalLayer.optimizer.streaming.LoadOptimizer

org.apache.pig.impl.logicalLayer.optimizer.streaming.StoreOptimizer

org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer

org.apache.pig.impl.logicalLayer.optimizer.TypeCastInserter

org.apache.pig.impl.logicalLayer.validators.TypeCheckingVisitor

org.apache.pig.newplan.logical.rules.TypeCastInserter$TypeCastInserterTransformer

org.apache.pig.newplan.logical.visitor.LineageFindRelVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.