Package org.apache.pig.backend.hadoop.datastorage

Examples of org.apache.pig.backend.hadoop.datastorage.HDataStorage.asElement()

195
196
197
198
199
200
201
202
203
204
205
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
   
    public static long getSize(String fileName) throws IOException {
      Configuration conf = PigMapReduce.sJobConfInternal.get();
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }

    public static long getSize(String fileName) throws IOException {
      Configuration conf = PigMapReduce.sJobConfInternal.get();
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }

    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);

        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);

        long size = 0;
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
                    String fn;
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }

    public static long getSize(String fileName) throws IOException {
      Configuration conf = PigMapReduce.sJobConfInternal.get();
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }

    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);

        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);

        long size = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.