Examples of com.google.appengine.tools.mapreduce.Counters

Package com.google.appengine.tools.mapreduce

Examples of com.google.appengine.tools.mapreduce.Counters

com.google.appengine.tools.mapreduce.Counters
Collection of all counters.

      WorkerShardTask<I, O, MapOnlyMapperContext<O>> task = new MapOnlyShardTask<>(
          id, shard, readers.size(), readers.get(shard), getCopyOfMapper(), writers.get(shard),
          Long.MAX_VALUE);
      tasks.add(task);
    }
    final Counters counters = new CountersImpl();
    InProcessShardedJobRunner.runJob(tasks.build(), new ShardedJobController<
        WorkerShardTask<I, O, MapOnlyMapperContext<O>>>() {
          // Not really meant to be serialized, but avoid warning.
          private static final long serialVersionUID = 661198005749484951L;


          @Override
          public void failed(Status status) {
            throw new UnsupportedOperationException();
          }


          @Override
          public void completed(Iterator<WorkerShardTask<I, O, MapOnlyMapperContext<O>>> tasks) {
            while (tasks.hasNext()) {
              counters.addAll(tasks.next().getContext().getCounters());
            }
          }
        });
    log.info("Map completed");
    log.info("combined counters=" + counters);

View Full Code Here

      WorkerShardTask<I, KeyValue<K, V>, MapperContext<K, V>> task = new MapShardTask<>(
          id, shard, inputs.size(), inputs.get(shard), getCopyOfMapper(), writers.get(shard),
          Long.MAX_VALUE);
      tasks.add(task);
    }
    final Counters counters = new CountersImpl();
    InProcessShardedJobRunner.runJob(tasks.build(), new ShardedJobController<
        WorkerShardTask<I, KeyValue<K, V>, MapperContext<K, V>>>() {
          // Not really meant to be serialized, but avoid warning.
          private static final long serialVersionUID = 661198005749484951L;


          @Override
          public void failed(Status status) {
            throw new UnsupportedOperationException();
          }


          @Override
          public void completed(
              Iterator<WorkerShardTask<I, KeyValue<K, V>, MapperContext<K, V>>> tasks) {
            while (tasks.hasNext()) {
              counters.addAll(tasks.next().getContext().getCounters());
            }
          }
        });
    log.info("Map phase completed");
    log.info("combined counters=" + counters);

View Full Code Here

      WorkerShardTask<KeyValue<K, Iterator<V>>, O, ReducerContext<O>> task =
          new ReduceShardTask<>(id, shard, outputs.size(), getReducerInputReader(inputs.get(shard)),
              getCopyOfReducer(), outputs.get(shard), Long.MAX_VALUE);
      tasks.add(task);
    }
    final Counters counters = new CountersImpl();
    counters.addAll(mapCounters);
    InProcessShardedJobRunner.runJob(tasks.build(), new ShardedJobController<
        WorkerShardTask<KeyValue<K, Iterator<V>>, O, ReducerContext<O>>>() {
      // Not really meant to be serialized, but avoid warning.
      private static final long serialVersionUID = 575338448598450119L;


      @Override
      public void failed(Status status) {
        throw new UnsupportedOperationException();
      }


      @Override
      public void completed(
          Iterator<WorkerShardTask<KeyValue<K, Iterator<V>>, O, ReducerContext<O>>> tasks) {
        while (tasks.hasNext()) {
          counters.addAll(tasks.next().getContext().getCounters());
        }
      }
    });
    log.info("Reduce phase completed, reduce counters=" + counters);
    log.info("combined counters=" + counters);

View Full Code Here

      JSONObject mapperSpec = new JSONObject();
      mapperSpec.put("mapper_params", mapperParams);
      jobObject.put("mapper_spec", mapperSpec);


      JSONArray shardArray = new JSONArray();
      Counters totalCounters = new CountersImpl();
      int i = 0;
      long[] workerCallCounts = new long[state.getTotalTaskCount()];
      Iterator<IncrementalTaskState<IncrementalTask>> tasks = shardedJobService.lookupTasks(state);
      while (tasks.hasNext()) {
        IncrementalTaskState<?> taskState = tasks.next();
        JSONObject shardObject = new JSONObject();
        shardObject.put("shard_number", i);
        shardObject.put("shard_description", taskState.getTaskId());
        shardObject.put("updated_timestamp_ms", taskState.getMostRecentUpdateMillis());
        if (taskState.getStatus().isActive()) {
          shardObject.put("active", true);
        } else {
          shardObject.put("active", false);
          shardObject.put("result_status", taskState.getStatus().getStatusCode());
        }
        IncrementalTask task = taskState.getTask();
        if (task instanceof IncrementalTaskWithContext) {
          IncrementalTaskContext context = ((IncrementalTaskWithContext) task).getContext();
          totalCounters.addAll(context.getCounters());
          workerCallCounts[i] = context.getWorkerCallCount();
          shardObject.put("last_work_item", context.getLastWorkItemString());
        }
        shardArray.put(shardObject);
        i++;

View Full Code Here

        log.warning("----- Count: " + count);


        Assert.assertTrue(count instanceof MapReduceResult);
        MapReduceResult result = MapReduceResult.class.cast(count);
        int[] chars = toChars(payloads);
        Counters counters = result.getCounters();
        for (int i = 0; i < chars.length; i++) {
            char ch = (char) ('a' + i);
            Counter c = counters.getCounter(CountMapper.toKey(ch));
            Assert.assertEquals(String.format("Invalid count for '%s'.", ch), chars[i], c.getValue());
        }
    }

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.mapreduce.Counters

com.google.appengine.tck.mapreduce.MapReduceTest

com.google.appengine.tools.mapreduce.impl.handlers.StatusHandler

com.google.appengine.tools.mapreduce.impl.InProcessMap

com.google.appengine.tools.mapreduce.impl.InProcessMapReduce

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.