Package org.apache.pig.impl.streaming.StreamingCommand

Examples of org.apache.pig.impl.streaming.StreamingCommand.HandleSpec

161
162
163
164
165
166
167
168
169
170
171
172
                    FuncSpec loadFuncSpec = null;
                    if(op instanceof LOLoad) {
                        loadFuncSpec = ((LOLoad)op).getFileSpec().getFuncSpec();
                    } else if (op instanceof LOStream) {
                        StreamingCommand command = ((LOStream)op).getStreamingCommand();
                        HandleSpec streamOutputSpec = command.getOutputSpec();
                        loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                    } else {
                        String msg = "TypeCastInserter invoked with an invalid operator class name: " + innerPlan.getClass().getSimpleName();
                        throw new FrontendException(msg, 2242);
                    }
                    cast.setFuncSpec(loadFuncSpec);
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
        processError("\nEnd time: " + new Date(System.currentTimeMillis()));
        processError("\nExit code: " + exitCode);

        List<HandleSpec> inputSpecs = command.getHandleSpecs(Handle.INPUT);
        HandleSpec inputSpec =
            (inputSpecs != null) ? inputSpecs.get(0) : null;
        if (inputSpec == null ||
            !inputSpec.getSpec().contains("BinaryStorage")) {
            processError("\nInput records: " + inputRecords);
        }
        processError("\nInput bytes: " + inputBytes + " bytes " +
                    ((inputSpec != null) ?
                            "(" + inputSpec.getName() + " using " +
                                inputSpec.getSpec() + ")"
                            : ""));

        List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
        HandleSpec outputSpec =
            (outputSpecs != null) ? outputSpecs.get(0) : null;
        if (outputSpec == null ||
            !outputSpec.getSpec().contains("BinaryStorage")) {
            processError("\nOutput records: " + outputRecords);
        }
        processError("\nOutput bytes: " + outputBytes + " bytes " +
                     ((outputSpec != null) ?
                         "(" + outputSpec.getName() + " using " +
                             outputSpec.getSpec() + ")"
                         : ""));
        if (outputSpecs != null) {
            for (int i=1; i < outputSpecs.size(); ++i) {
                HandleSpec spec = outputSpecs.get(i);
                processError("\n           " + new File(spec.getName()).length()
                             + " bytes using " + spec.getSpec());
            }
        }

        processError("\n=====          * * *          =====\n");
    }
View Full Code Here
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
        } else {
            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            }
           
            Schema s = op.getSchema();
            if(null != s) {
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
                            } else if (lo instanceof LOStream) {
                                StreamingCommand command = ((LOStream)lo).getStreamingCommand();
                                HandleSpec streamOutputSpec = command.getOutputSpec();
                                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                            } else {
                                int errCode = 2006;
                                String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
                                throw new OptimizerException(msg, errCode, PigException.BUG);
                            }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
        // untouched.

        StreamingCommand optimizedCommand = (StreamingCommand)command.clone();
       
        if (handle == Handle.INPUT) {
            HandleSpec streamInputSpec = optimizedCommand.getInputSpec();
            streamInputSpec.setSpec(spec);
        } else if (handle == Handle.OUTPUT) {
            HandleSpec streamOutputSpec = optimizedCommand.getOutputSpec();
            streamOutputSpec.setSpec(spec);
        }
       
        command = optimizedCommand;
    }
View Full Code Here
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
        } else {
            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            }
           
            Schema s = op.getSchema();
            if(null != s) {
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
190
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
                            } else if (lo instanceof LOStream) {
                                StreamingCommand command = ((LOStream)lo).getStreamingCommand();
                                HandleSpec streamOutputSpec = command.getOutputSpec();
                                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                            } else {
                                int errCode = 2006;
                                String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
                                throw new OptimizerException(msg, errCode, PigException.BUG);
                            }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
145
146
147
        // untouched.

        StreamingCommand optimizedCommand = (StreamingCommand)command.clone();
       
        if (handle == Handle.INPUT) {
            HandleSpec streamInputSpec = optimizedCommand.getInputSpec();
            streamInputSpec.setSpec(spec);
        } else if (handle == Handle.OUTPUT) {
            HandleSpec streamOutputSpec = optimizedCommand.getOutputSpec();
            streamOutputSpec.setSpec(spec);
        }
       
        command = optimizedCommand;
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
            EvalSpec spec = eval.getSpec();
            if (spec instanceof StreamSpec) {
                // Try and optimize if the store and stream output specs match
                StreamSpec streamSpec = (StreamSpec)spec;
                StreamingCommand command = streamSpec.getCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
               
                FileSpec storeFileSpec = s.getOutputFileSpec();
               
                // Instantiate both to compare them for equality
                LoadFunc streamLoader =
                    (LoadFunc)PigContext.instantiateFuncFromSpec(
                            streamOutputSpec.getSpec());
               
                StoreFunc outputStorer = (StoreFunc)PigContext.instantiateFuncFromSpec(
                                                storeFileSpec.getFuncSpec());
               
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        if (optimizedCommand == null) {
            optimizedCommand = (StreamingCommand)command.clone();
        }
       
        if (handle == Handle.INPUT) {
            HandleSpec streamInputSpec = optimizedCommand.getInputSpec();
            streamInputSpec.setSpec(spec);
        } else if (handle == Handle.OUTPUT) {
            HandleSpec streamOutputSpec = optimizedCommand.getOutputSpec();
            streamOutputSpec.setSpec(spec);
        }
       
        command = optimizedCommand;
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.streaming.StreamingCommand.HandleSpec

  • org.apache.pig.backend.hadoop.streaming.HadoopExecutableManager
  • org.apache.pig.impl.eval.StreamSpec
  • org.apache.pig.impl.logicalLayer.LOStream
  • org.apache.pig.impl.logicalLayer.optimizer.streaming.LoadOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.streaming.StoreOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.TypeCastInserter
  • org.apache.pig.impl.logicalLayer.validators.TypeCheckingVisitor
  • org.apache.pig.impl.streaming.HandlerFactory
  • org.apache.pig.newplan.logical.rules.TypeCastInserter$TypeCastInserterTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.