Package org.apache.pig.backend.hadoop.datastorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

165
166
167
168
169
170
171
172
173
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        properties);
            }
            break;
           
            default:
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        properties);
            }
            break;
           
            default:
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);
       
           
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
379
380
        String[] locations = getPathStrings(location);
        for (String loc : locations) {
            // since local mode now is implemented as hadoop's local mode
            // we can treat either local or hadoop mode as hadoop mode - hence
            // we can use HDataStorage and FileLocalizer.openDFSFile below
            HDataStorage storage;
            try {
                storage = new HDataStorage(new URI(loc), props);
            } catch (URISyntaxException e) {
                throw new IOException(e);
            }
            if (!FileLocalizer.fileExists(loc, storage)) {
                return null;
View Full Code Here
165
166
167
168
169
170
171
172
173
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
        return lastConf;
    }

    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        wrapped.init(store);
       
        // Mimic org.apache.hadoop.mapred.FileSplit if feasible...
        String[] locations = wrapped.getLocations();
        if (locations.length > 0) {
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
                properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
            }
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        // So we need to reconstruct the configuration object for the non HOD case
        // In the HOD case, this is the first time the configuration object will be created
        configuration = ConfigurationUtil.toConfiguration(properties);
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.datastorage.HDataStorage

  • org.apache.hadoop.dfs.DistributedFileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hdfs.DistributedFileSystem
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper
  • org.apache.pig.builtin.BinStorage
  • org.apache.pig.builtin.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.