Package org.apache.pig

Examples of org.apache.pig.ResourceStatistics$ResourceFieldStatistics

203
204
205
206
207
208
209
210
211
212
213
        } else {
            log.warn("Could not find stat file for "+location);
            return null;
        }
        log.debug("Found stat file "+statFile.toString());
        ResourceStatistics resourceStats = null;       
        try {
            resourceStats = new ObjectMapper().readValue(statFile.open(), ResourceStatistics.class);
        } catch (JsonParseException e) {
            log.warn("Unable to load Resource Statistics for "+location);
            e.printStackTrace();
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
        } else {
            log.warn("Could not find stat file for "+location);
            return null;
        }
        log.info("Found stat file "+statFile.toString());
        ResourceStatistics resourceStats = null;       
        try {
            resourceStats = new ObjectMapper().readValue(statFile.open(), ResourceStatistics.class);
        } catch (JsonParseException e) {
            log.warn("Unable to load Resource Statistics for "+location);
            e.printStackTrace();
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
   * Get statistics about the data to be loaded. Only input data size is implemented at this time.
   */
  @Override
  public ResourceStatistics getStatistics(String location, Job job) throws IOException {
    try {
      ResourceStatistics stats = new ResourceStatistics();
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
        job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
      stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
      return stats;
    } catch (Exception e) {
      throw new IOException(e);
    }
  }
View Full Code Here
401
402
403
404
405
406
407
408
    Job job = new Job();
    HCatLoader hCatLoader = new HCatLoader();
    hCatLoader.setUDFContextSignature(this.getName());
    hCatLoader.setLocation(SPECIFIC_SIZE_TABLE, job);
    ResourceStatistics statistics = hCatLoader.getStatistics(file.getAbsolutePath(), job);
    assertEquals(2048, (long) statistics.getmBytes());
  }
View Full Code Here
404
405
406
407
408
409
410
411
    Job job = new Job();
    HCatLoader hCatLoader = new HCatLoader();
    hCatLoader.setUDFContextSignature(this.getName());
    hCatLoader.setLocation(SPECIFIC_SIZE_TABLE, job);
    ResourceStatistics statistics = hCatLoader.getStatistics(file.getAbsolutePath(), job);
    assertEquals(2048, (long) statistics.getmBytes());
  }
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
   * Get statistics about the data to be loaded. Only input data size is implemented at this time.
   */
  @Override
  public ResourceStatistics getStatistics(String location, Job job) throws IOException {
    try {
      ResourceStatistics stats = new ResourceStatistics();
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
        job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
      stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
      return stats;
    } catch (Exception e) {
      throw new IOException(e);
    }
  }
View Full Code Here
396
397
398
399
400
401
402
403
        Job job = new Job();
        HCatLoader hCatLoader = new HCatLoader();
        hCatLoader.setUDFContextSignature(this.getName());
        hCatLoader.setLocation(SPECIFIC_SIZE_TABLE, job);
        ResourceStatistics statistics = hCatLoader.getStatistics(file.getAbsolutePath(), job);
        assertEquals(2048, (long) statistics.getmBytes());
    }
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
     * Get statistics about the data to be loaded. Only input data size is implemented at this time.
     */
    @Override
    public ResourceStatistics getStatistics(String location, Job job) throws IOException {
        try {
            ResourceStatistics stats = new ResourceStatistics();
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
                job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
            stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
            return stats;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
View Full Code Here
39
40
41
42
43
44
45
46
47
        loc = location;
    }

    @Override
    public ResourceStatistics getStatistics(String location, Job job) throws IOException {
        ResourceStatistics stats = new ResourceStatistics();
        stats.setSizeInBytes(getInputSizeInBytes());
        return stats;
    }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
                || ld.getLFile() == null
                || ld.getLFile().getFileName() == null) {
            return -1;
        }

        ResourceStatistics statistics;
        try {
            statistics = ((LoadMetadata) ld.getLoadFunc())
                        .getStatistics(ld.getLFile().getFileName(), job);
        } catch (Exception e) {
            log.warn("Couldn't get statistics from LoadFunc: " + ld.getLoadFunc(), e);
            return -1;
        }

        if (statistics == null || statistics.getSizeInBytes() == null) {
            return -1;
        }

        return statistics.getSizeInBytes();
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.ResourceStatistics$ResourceFieldStatistics

  • org.apache.hcatalog.pig.HCatLoader
  • org.apache.hcatalog.pig.TestHCatLoader
  • org.apache.hive.hcatalog.pig.HCatLoader
  • org.apache.hive.hcatalog.pig.TestHCatLoader
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator
  • org.apache.pig.builtin.JsonMetadata
  • org.apache.pig.piggybank.storage.JsonMetadata
  • org.apache.pig.test.PigStorageWithStatistics
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.