Examples of HDataStorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

40
41
42
43
44
45
46
47
48
49
50
public class TypeCheckingTestUtil {

    public static LOLoad genDummyLOLoad(LogicalPlan plan)  {
        String pigStorage = PigStorage.class.getName() ;
        DataStorage dfs = new HDataStorage(new Properties());
        try {
            LOLoad load = new LOLoad(plan,
                                      genNewOperatorKey(),
                                      new FileSpec("pi", new FuncSpec(pigStorage)),
                                      null, dfs, true) ;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

443
444
445
446
447
448
449
450
451
452
453
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null, new HDataStorage(new Properties()), true) ;

        // set schemas
        load1.setEnforcedSchema(null) ;

        // Create inner plan
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

507
508
509
510
511
512
513
514
515
516
517
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null, new HDataStorage(new Properties()), true) ;

        // set schemas
        load1.setEnforcedSchema(null) ;

        // Create inner plan
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
       
        // Pass loader signature to slicer
          List<String> inpSignatureLists = (ArrayList<String>)ObjectSerializer.deserialize(job.get("pig.inpSignatures"));
                if (inpSignatureLists.get(i)!=null)
                    store.getConfiguration().setProperty("pig.loader.signature", inpSignatureLists.get(i));

        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    public void connect() throws ExecException {

        switch (execType) {
            case LOCAL:
            {
                lfs = new HDataStorage(URI.create("file:///"),
                                       new Properties());
               
                dfs = lfs;
                executionEngine = new LocalExecutionEngine(this);
            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        new Properties());               
            }
            break;
           
            default:
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

170
171
172
173
174
175
176
177
178
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

194
195
196
197
198
199
200
201
202
203
204
205
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    }

    @SuppressWarnings("unchecked")
    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        // if the execution is against Mapred DFS, set
        // working dir to /user/<userid>
        if(execType == ExecType.MAPREDUCE)
            store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
        List<String> inpSignatureLists = (ArrayList<String>)ObjectSerializer.deserialize(job.get("pig.inpSignatures"));
        if (inpSignatureLists.get(index)!=null) {
            // Pass loader signature to slice
            store.getConfiguration().setProperty("pig.loader.signature", inpSignatureLists.get(index));
            job.set("pig.loader.signature", inpSignatureLists.get(index));
        }
       
        // Get the UDF specific context
        UDFContext udfc = UDFContext.getUDFContext();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

165
166
167
168
169
170
171
172
173
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

182
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.