Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOStore

147
148
149
150
151
152
153
154
155
156
157
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, dfs, true) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
       
        plan.add(load) ;
        plan.add(store) ;
       
        plan.connect(load, store) ;    
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        List<LogicalOperator> successors = mPlan.getSuccessors(stream);
        if(successors == null)
            return;
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization";
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        List<LogicalOperator> successors = mPlan.getSuccessors(stream);
        if(successors == null)
            return;
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
230
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization";
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, null, true) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
       
        plan.add(load) ;
        plan.add(store) ;
       
        plan.connect(load, store) ;    
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
288
        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
       
        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOStore

  • org.apache.pig.backend.hadoop.executionengine.MapreducePlanCompiler
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.impl.plan.ProjectionMap
  • org.apache.pig.impl.plan.RequiredFields
  • org.apache.pig.PigServer
  • org.apache.pig.StoreFunc
  • org.apache.pig.test.TestInputOutputFileValidator
  • org.apache.pig.test.TestStore
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.