Package org.apache.pig.backend.hadoop.datastorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

300
301
302
303
304
305
306
307
308
309
310
                executionEngine.init();

                dfs = executionEngine.getDataStorage();

                lfs = new HDataStorage(URI.create("file:///"),
                                        properties);
            }
            break;

            default:
View Full Code Here
412
413
414
415
416
417
418
419
420
421
422
423
424
        String[] locations = getPathStrings(location);
        for (String loc : locations) {
            // since local mode now is implemented as hadoop's local mode
            // we can treat either local or hadoop mode as hadoop mode - hence
            // we can use HDataStorage and FileLocalizer.openDFSFile below
            HDataStorage storage;
            try {
              storage = new HDataStorage((new org.apache.hadoop.fs.Path(loc)).toUri(), props);
            } catch (RuntimeException e) {
                throw new IOException(e);
            }
            if (!FileLocalizer.fileExists(loc, storage)) {
                return null;
View Full Code Here
158
159
160
161
162
163
164
165
166
        return openDFSFile(fileName, ConfigurationUtil.toProperties(wrapper.getJobConf()));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
                properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
            }
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        // So we need to reconstruct the configuration object for the non HOD case
        // In the HOD case, this is the first time the configuration object will be created
        configuration = ConfigurationUtil.toConfiguration(properties);
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
397
398
        String[] locations = getPathStrings(location);
        for (String loc : locations) {
            // since local mode now is implemented as hadoop's local mode
            // we can treat either local or hadoop mode as hadoop mode - hence
            // we can use HDataStorage and FileLocalizer.openDFSFile below
            HDataStorage storage;
            try {
              storage = new HDataStorage((new org.apache.hadoop.fs.Path(loc)).toUri(), props);
            } catch (RuntimeException e) {
                throw new IOException(e);
            }
            if (!FileLocalizer.fileExists(loc, storage)) {
                return null;
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        String[] locations = LoadFunc.getPathStrings(path);
        for (String loc : locations) {
            DataStorage storage;
           
            storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));
           
            String fullPath = FileLocalizer.fullPath(loc, storage);

            if(storage.isContainer(fullPath)) {
                ElementDescriptor metaFilePath = storage.asElement(fullPath, metaname);
View Full Code Here
270
271
272
273
274
275
276
277
278
279
280
281
    // Implementation of StoreMetaData interface

    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        // constructor sets DEFAULT_REPLICATION_FACTOR_KEY
        ds = new HDataStorage(properties);
               
        if(cluster != null && !cluster.equalsIgnoreCase(LOCAL)){
            log.info("Connecting to map-reduce job tracker at: " + jc.get(JOB_TRACKER_LOCATION));
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.datastorage.HDataStorage

  • org.apache.hadoop.dfs.DistributedFileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hdfs.DistributedFileSystem
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper
  • org.apache.pig.builtin.BinStorage
  • org.apache.pig.builtin.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.