Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.Counters

166
167
168
169
170
171
172
173
174
175
176
177
    // delete the output directory if it exists already
    fs.delete(new Path(outputPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.findCounter(Blocks.Total).getCounter();

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(outputPath + "/part-00000")));
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
    // delete the output directory if it exists already
    fs.delete(outputPath, true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.findCounter(Blocks.Total).getCounter();

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(outputPath + "/part-00000")));
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
    // Delete the output directory if it exists already.
    fs.delete(new Path(tmpPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.getCounter(Blocks.Total);

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(tmpPath + "/part-00000")));
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
    // delete the output directory if it exists already
    FileSystem.get(conf).delete(new Path(outputPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int numDocs = (int) counters.findCounter(Count.DOCS).getCounter();

    String inputFile = outputPath + "/" + "part-00000";

    sLogger.info("Writing " + numDocs + " doc offseta to " + indexFile);
    LineReader reader = new LineReader(fs.open(new Path(inputFile)));
View Full Code Here
47
48
49
50
51
52
53
  private Mapper<K1, V1, K2, V2> myMapper;
  private Counters counters;

  public MapDriver(final Mapper<K1, V1, K2, V2> m) {
    myMapper = m;
    counters = new Counters();
  }
View Full Code Here
51
52
53
54
55
56
57
    myMapper = m;
    counters = new Counters();
  }

  public MapDriver() {
    counters = new Counters();
  }
View Full Code Here
48
49
50
51
52
53
54
  private Reducer<K1, V1, K2, V2> myReducer;
  private Counters counters;

  public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
    myReducer = r;
    counters = new Counters();
  }
View Full Code Here
52
53
54
55
56
57
58
    myReducer = r;
    counters = new Counters();
  }

  public ReduceDriver() {
    counters = new Counters();
  }
View Full Code Here
105
106
107
108
109
110
111
    public static long getDefaultBlockSize(FileSystem fs, Path path) {
        return fs.getDefaultBlockSize(path);
    }

    public static Counters getCounters(Job job) throws IOException, InterruptedException {
        return new Counters(job.getJob().getCounters());
    }
View Full Code Here
298
299
300
301
302
303
304
305
306
307
308
309
310
     * Parse and add the job counters
     */
    @SuppressWarnings("deprecation")
    private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
        try {
            Counters counterGroups = Counters.fromEscapedCompactString(counters);
            for (Group otherGroup : counterGroups) {
                Group group = counterGroups.getGroup(otherGroup.getName());
                for (Counter otherCounter : otherGroup) {
                    Counter counter = group.getCounterForName(otherCounter.getName());
                    job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
                }
            }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Counters

  • cascading.stats.hadoop.HadoopNodeCounterCache
  • cascading.stats.hadoop.HadoopSliceStats
  • cc.mrlda.ParseCorpus
  • cc.mrlda.polylda.VariationalInference
  • cc.mrlda.VariationalInference
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.RunningJobEmul
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer
  • com.m6d.filecrush.crush.CountersInputFormat$CountersReader
  • com.m6d.filecrush.crush.CountersMapperTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.