Examples of com.google.appengine.tools.mapreduce.impl.CountersImpl

Package com.google.appengine.tools.mapreduce.impl

Examples of com.google.appengine.tools.mapreduce.impl.CountersImpl

com.google.appengine.tools.mapreduce.impl.CountersImpl

          specification.getMapper(), writers.get(i), settings.getMillisPerSlice()));
    }
    ShardedJobSettings shardedJobSettings = settings.toShardedJobSettings(jobId, getPipelineKey());
    PromisedValue<ResultAndStatus<R>> resultAndStatus = newPromise();
    WorkerController<I, O, R, MapOnlyMapperContext<O>> workerController = new WorkerController<>(
        jobId, new CountersImpl(), output, resultAndStatus.getHandle());
    ShardedJob<?> shardedJob =
        new ShardedJob<>(jobId, mapTasks.build(), workerController, shardedJobSettings);
    FutureValue<Void> shardedJobResult = futureCall(shardedJob, settings.toJobSettings());
    JobSetting[] jobSetting = settings.toJobSettings(waitFor(shardedJobResult),
            statusConsoleUrl(shardedJobSettings.getMapReduceStatusUrl()), maxAttempts(1));

View Full Code Here

      JSONObject mapperSpec = new JSONObject();
      mapperSpec.put("mapper_params", mapperParams);
      jobObject.put("mapper_spec", mapperSpec);


      JSONArray shardArray = new JSONArray();
      Counters totalCounters = new CountersImpl();
      int i = 0;
      long[] workerCallCounts = new long[state.getTotalTaskCount()];
      Iterator<IncrementalTaskState<IncrementalTask>> tasks = shardedJobService.lookupTasks(state);
      while (tasks.hasNext()) {
        IncrementalTaskState<?> taskState = tasks.next();
        JSONObject shardObject = new JSONObject();
        shardObject.put("shard_number", i);
        shardObject.put("shard_description", taskState.getTaskId());
        shardObject.put("updated_timestamp_ms", taskState.getMostRecentUpdateMillis());
        if (taskState.getStatus().isActive()) {
          shardObject.put("active", true);
        } else {
          shardObject.put("active", false);
          shardObject.put("result_status", taskState.getStatus().getStatusCode());
        }
        IncrementalTask task = taskState.getTask();
        if (task instanceof IncrementalTaskWithContext) {
          IncrementalTaskContext context = ((IncrementalTaskWithContext) task).getContext();
          totalCounters.addAll(context.getCounters());
          workerCallCounts[i] = context.getWorkerCallCount();
          shardObject.put("last_work_item", context.getLastWorkItemString());
        }
        shardArray.put(shardObject);
        i++;

View Full Code Here

      ShardedJobSettings shardedJobSettings =
          settings.toShardedJobSettings(shardedJobId, getPipelineKey());


      PromisedValue<ResultAndStatus<FilesByShard>> resultAndStatus = newPromise();
      WorkerController<I, KeyValue<K, V>, FilesByShard, MapperContext<K, V>> workerController =
          new WorkerController<>(mrJobId, new CountersImpl(), output, resultAndStatus.getHandle());
      ShardedJob<?> shardedJob =
          new ShardedJob<>(shardedJobId, mapTasks.build(), workerController, shardedJobSettings);
      FutureValue<Void> shardedJobResult = futureCall(shardedJob, settings.toJobSettings());
      return futureCall(new ExamineStatusAndReturnResult<FilesByShard>(shardedJobId),
          resultAndStatus, settings.toJobSettings(waitFor(shardedJobResult),

View Full Code Here

      try {
        outputResult = output.finish(closedWriters.build());
      } catch (IOException e) {
        throw new RuntimeException(output + ".finish() threw IOException");
      }
      CountersImpl totalCounters = new CountersImpl();
      totalCounters.addAll(initialCounters);
      totalCounters.addAll(finalCombinedResult.getCounters());
      ResultAndCounters<R> result = new ResultAndCounters<R>(outputResult, totalCounters);
      // HACK(ohler): It seems that pipeline doesn't always allow a promise to
      // be filled in the job that created the promise.  We attempt that in the
      // case where there are 0 shards, like in one of our test cases.  Or maybe
      // it's our lack of retries on NoSuchObjectException.  Need to investigate

View Full Code Here

          readers.size(),
          mrSpec.getIntermediateKeyMarshaller(),
          mrSpec.getIntermediateValueMarshaller());
      startShardedJob(mrSpec.getJobName() + " (map phase)",
          shardedJobId,
          new CountersImpl(),
          readers, output, Util.createWriters(output),
          new TaskCreator<I, KeyValue<K, V>, MapperContext<K, V>>() {
            @Override public WorkerShardTask<I, KeyValue<K, V>, MapperContext<K, V>> createTask(
                int shard, int shardCount,
                InputReader<I> reader, OutputWriter<KeyValue<K, V>> writer) {

View Full Code Here

    jobService.startJob(
        "testGetJobDetail_populated", ImmutableList.of(one, two), controller, settings);
    ShardedJobState<?, WorkerResult<? extends Serializable>> state =
        jobService.getJobState("testGetJobDetail_populated");
    assertEquals(2, state.getActiveTaskCount());
    assertEquals(new CountersImpl(), state.getAggregateResult().getCounters());
    assertEquals(2, state.getTotalTaskCount());
    assertEquals(Status.RUNNING, state.getStatus());
    JSONObject jobDetail = StatusHandler.handleGetJobDetail("testGetJobDetail_populated");
    assertNotNull(jobDetail);
    assertEquals("testGetJobDetail_populated", jobDetail.getString("mapreduce_id"));

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.mapreduce.impl.CountersImpl

com.google.appengine.tools.mapreduce.Counter

com.google.appengine.tools.mapreduce.impl.handlers.StatusHandler

com.google.appengine.tools.mapreduce.impl.handlers.StatusHandlerTest

com.google.appengine.tools.mapreduce.MapJob

com.google.appengine.tools.mapreduce.MapReduceJob$MapJob

com.google.appengine.tools.mapreduce.MapReduceJob$WorkerController

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.