Package org.apache.pig.impl.streaming

Examples of org.apache.pig.impl.streaming.StreamingCommand$HandleSpec

621
622
623
624
625
626
627
628
629
630
631
    }
   
    StreamingCommand buildCommand(SourceLocation loc, String cmd) throws RecognitionException {
        try {
            String[] args = StreamingCommandUtils.splitArgs( cmd );
            StreamingCommand command = new StreamingCommand( pigContext, args );
            StreamingCommandUtils validator = new StreamingCommandUtils( pigContext );
            validator.checkAutoShipSpecs( command, args );
            return command;
        } catch (ParserException e) {
            throw new InvalidCommandException( intStream, loc, cmd );
View Full Code Here
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
            }
        } else {
            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            } else if ((op instanceof LOFilter)
                    || (op instanceof LODistinct)
                    || (op instanceof LOSort)
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
        // optimize only B's input spec since it is immediately following a store
        // and is conducive to optimization. At this point we clone and make
        // sure only B's "command" gets optimized while C's "command" remains
        // untouched.

        StreamingCommand optimizedCommand = (StreamingCommand)command.clone();
       
        if (handle == Handle.INPUT) {
            HandleSpec streamInputSpec = optimizedCommand.getInputSpec();
            streamInputSpec.setSpec(spec);
        } else if (handle == Handle.OUTPUT) {
            HandleSpec streamOutputSpec = optimizedCommand.getOutputSpec();
            streamOutputSpec.setSpec(spec);
        }
       
        command = optimizedCommand;
    }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
                            cast.setFieldSchema(fs.clone());
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
                            } else if (lo instanceof LOStream) {
                                StreamingCommand command = ((LOStream)lo).getStreamingCommand();
                                HandleSpec streamOutputSpec = command.getOutputSpec();
                                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                            } else {
                                int errCode = 2006;
                                String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
                                throw new OptimizerException(msg, errCode, PigException.BUG);
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
View Full Code Here
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
            }
        } else {
            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            }
           
            Schema s = op.getSchema();
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
                            cast.setFieldSchema(fs.clone());
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
                            } else if (lo instanceof LOStream) {
                                StreamingCommand command = ((LOStream)lo).getStreamingCommand();
                                HandleSpec streamOutputSpec = command.getOutputSpec();
                                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                            } else {
                                int errCode = 2006;
                                String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
                                throw new OptimizerException(msg, errCode, PigException.BUG);
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.streaming.StreamingCommand$HandleSpec

  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOStreamVisitor$StreamFileVisitor
  • org.apache.pig.impl.builtin.StreamingUDF
  • org.apache.pig.impl.logicalLayer.LOStream
  • org.apache.pig.impl.logicalLayer.optimizer.streaming.LoadOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.streaming.StoreOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.TypeCastInserter
  • org.apache.pig.impl.logicalLayer.validators.TypeCheckingVisitor
  • org.apache.pig.newplan.logical.rules.TypeCastInserter$TypeCastInserterTransformer
  • org.apache.pig.newplan.logical.visitor.LineageFindRelVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.