Examples of HDataStorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

165
166
167
168
169
170
171
172
173
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

182
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

211
212
213
214
215
216
217
218
219
220
221
            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);
       
           
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

183
184
185
186
187
188
189
190
191
192
193
                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        new Properties());
            }
            break;
           
            default:
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

158
159
160
161
162
163
164
165
166
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

392
393
394
395
396
397
398
399
400
401
402
        Configuration conf = job.getConfiguration();
        Properties props = ConfigurationUtil.toProperties(conf);
        // since local mode now is implemented as hadoop's local mode
        // we can treat either local or hadoop mode as hadoop mode - hence
        // we can use HDataStorage and FileLocalizer.openDFSFile below
        HDataStorage storage = new HDataStorage(props);
        if (!FileLocalizer.fileExists(location, storage)) {
            // At compile time in batch mode, the file may not exist
            // (such as intermediate file). Just return null - the
            // same way as we would if we did not get a valid record
            return null;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

175
176
177
178
179
180
181
182
183
184
185
186
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

209
210
211
212
213
214
215
216
217
218
219
            }
            properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
        }
    
        log.info("Connecting to hadoop file system at: "  + (nameNode==null? LOCAL: nameNode) )  ;
        ds = new HDataStorage(properties);
               
        // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
        configuration = ConfigurationUtil.toConfiguration(properties);
       
           
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
    public void connect() throws ExecException {

        switch (execType) {
            case LOCAL:
            {
                lfs = new HDataStorage(URI.create("file:///"),
                                       new Properties());
               
                dfs = lfs;
               
                executionEngine = new LocalExecutionEngine(this);
            }
            break;
   
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);
   
                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
                lfs = new HDataStorage(URI.create("file:///"),
                                        new Properties());               
            }
            break;
           
            default:
View Full Code Here

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage

153
154
155
156
157
158
159
160
161
        return openDFSFile(fileName, ConfigurationUtil.toProperties(wrapper.getJobConf()));
       
    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.