Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOLoad

146
147
148
149
150
151
152
153
154
155
156
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, dfs, true) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
       
        plan.add(load) ;
        plan.add(store) ;
       
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
        if((predecessors == null) || (predecessors.size() <= 0)) {
            return;
        }
        LogicalOperator predecessor = predecessors.get(0);
        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
                    (lo == null ? lo : lo.getClass().getName()));
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
        // else
        throw new AssertionError("Unknown operator type") ;
    }

    private LOLoad createLOLoad(DotNode node, LogicalPlan plan) {
        LOLoad load = null ;
        FileSpec fileSpec = new FileSpec("pi",
                                         new FuncSpec(PigStorage.class.getName())) ;
        try {
            load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
            fillSchema(load, node.attributes) ;
        }
        catch (IOException ioe) {
            throw new AssertionError("Dummy data is not good") ;
        }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
public class TypeCheckingTestUtil {

    public static LOLoad genDummyLOLoad(LogicalPlan plan)  {
        String pigStorage = PigStorage.class.getName() ;
        try {
            LOLoad load = new LOLoad(plan,
                                      genNewOperatorKey(),
                                      new FileSpec("pi", new FuncSpec(pigStorage)),
                                      ConfigurationUtil.toConfiguration(new Properties())) ;
            return load ;
        } catch (IOException e) {
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        if((predecessors == null) || (predecessors.size() <= 0)) {
            return;
        }
        LogicalOperator predecessor = predecessors.get(0);
        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
                    lo.getClass().getName());
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
public class TypeCheckingTestUtil {

    public static LOLoad genDummyLOLoad(LogicalPlan plan)  {
        String pigStorage = PigStorage.class.getName() ;
        try {
            LOLoad load = new LOLoad(plan,
                                      genNewOperatorKey(),
                                      new FileSpec("pi", new FuncSpec(pigStorage)),
                                      null, null, true) ;
            return load ;
        } catch (IOException e) {
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, null, true) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
       
        plan.add(load) ;
        plan.add(store) ;
       
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
                                       fileNameMap);
        Assert.assertTrue(lp.size()>0);
        LogicalOperator op = lp.getRoots().get(0);
       
        Assert.assertTrue(op instanceof LOLoad);
        LOLoad load = (LOLoad)op;

        String p = load.getInputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");

        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOLoad

  • org.apache.hadoop.mapreduce.Job
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.impl.logicalLayer.schema.TupleSchema
  • org.apache.pig.impl.plan.ProjectionMap
  • org.apache.pig.impl.plan.RequiredFields
  • org.apache.pig.LoadMetadata
  • org.apache.pig.LoadPushDown.RequiredField
  • org.apache.pig.LoadPushDown.RequiredFieldResponse
  • org.apache.pig.ResourceSchema
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.