Package com.google.appengine.tools.pipeline

Examples of com.google.appengine.tools.pipeline.PipelineService

97
98
99
100
101
102
103
104
105
    }

    int entities = parseInt(req.getParameter("entities"));
    int bytesPerEntity = parseInt(req.getParameter("payloadBytesPerEntity"));
    int shardCount = parseInt(req.getParameter("shardCount"));
    PipelineService service = PipelineServiceFactory.newPipelineService();
    redirectToPipelineStatus(resp, service.startNewPipeline(
        new ChainedMapReduceJob(bucket, DATASTORE_TYPE, shardCount, entities, bytesPerEntity)));
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        .setMapper(new TestMapper()).setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller())
        .setReducer(ValueProjectionReducer.<String, Long>create())
        .setOutput(new CustomOutput())
        .setNumReducers(17);
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    MapReduceSettings mrSettings = new MapReduceSettings.Builder().build();
    String jobId = pipelineService.startNewPipeline(
        new MapReduceJob<>(mrSpecBuilder.build(), mrSettings));
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    assertEquals(State.COMPLETED_SUCCESSFULLY, info.getJobState());
    @SuppressWarnings("unchecked")
    MapReduceResult<Boolean> result = (MapReduceResult<Boolean>) info.getOutput();
    assertNotNull(result);
    assertTrue(result.getOutputResult());
View Full Code Here
86
87
88
89
90
91
92
93
            + "<br> <a href=index.html>back</a><br>" + "</body></html>");
      }
      return;
    }

    PipelineService service = PipelineServiceFactory.newPipelineService();
    redirectToPipelineStatus(resp, service.startNewPipeline(new BigQueryLoadJob(rowCount)));
  }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
    String module = getStringParam(req, "module", "mapreduce");
    String bucket = getBucketParam(req);
    long start = getLongParam(req, "start", 0);
    long limit = getLongParam(req, "limit", 100 * 1000 * 1000);
    int shards = Math.max(1, Math.min(100, Ints.saturatedCast(getLongParam(req, "shards", 30))));
    PipelineService service = PipelineServiceFactory.newPipelineService();
    String pipelineId = service.startNewPipeline(new MyPipelineJob(bucket, start, limit, shards),
        new JobSetting.OnQueue(queue), new JobSetting.OnModule(module));
    resp.sendRedirect("/_ah/pipeline/status.html?root=" + pipelineId);
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
  public static <I, O, R> String start(MapSpecification<I, O, R> specification,
      MapSettings settings) {
    if (settings.getWorkerQueueName() == null) {
      settings = new MapSettings.Builder(settings).setWorkerQueueName("default").build();
    }
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    return pipelineService.startNewPipeline(
        new MapJob<>(specification, settings), settings.toJobSettings());
  }
View Full Code Here
38
39
40
41
42
43
44
45
    // TODO(user): should not be needed once b/9940384 is fixed
    return futureCall(new DeletePipelineJob(getPipelineKey().getName()), settings);
  }

  public static void cleanup(List<GcsFilename> toDelete, JobSetting... settings) {
    PipelineService service = PipelineServiceFactory.newPipelineService();
    service.startNewPipeline(new CleanupPipelineJob(), toDelete, settings);
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    DeferredTask deleteRecordsTask = new DeferredTask() {
      private static final long serialVersionUID = -7510918963650055768L;

      @Override
      public void run() {
        PipelineService service = PipelineServiceFactory.newPipelineService();
        try {
          service.deletePipelineRecords(key);
          log.info("Deleted pipeline: " + key);
        } catch (IllegalStateException e) {
          log.info("Failed to delete pipeline: " + key);
          HttpServletRequest request = DeferredTaskContext.getCurrentRequest();
          if (request != null) {
            int attempts = request.getIntHeader("X-AppEngine-TaskExecutionCount");
            if (attempts <= 5) {
              log.info("Request to retry deferred task #" + attempts);
              DeferredTaskContext.markForRetry();
              return;
            }
          }
          try {
            service.deletePipelineRecords(key, true, false);
            log.info("Force deleted pipeline: " + key);
          } catch (Exception ex) {
            log.log(Level.WARNING, "Failed to force delete pipeline: " + key, ex);
          }
        } catch (NoSuchObjectException e) {
View Full Code Here
87
88
89
90
91
92
93
94
95
  public static <I, K, V, O, R> String start(
      MapReduceSpecification<I, K, V, O, R> specification, MapReduceSettings settings) {
    if (settings.getWorkerQueueName() == null) {
      settings = new MapReduceSettings.Builder(settings).setWorkerQueueName("default").build();
    }
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    return pipelineService.startNewPipeline(
        new MapReduceJob<>(specification, settings), settings.toJobSettings());
  }
View Full Code Here
276
277
278
279
280
281
282
283
284
285
        callCompleted(jobState);
      } else {
        log.info("Calling failed for " + jobId + ", status=" + jobState.getStatus());
        jobState.getController().failed(jobState.getStatus());
      }
      PipelineService pipeline = PipelineServiceFactory.newPipelineService();
      pipeline.startNewPipeline(
          new FinalizeShardedJob(jobId, jobState.getTotalTaskCount(), jobState.getStatus()));
    }
  }
View Full Code Here
662
663
664
665
666
667
668
669
670
671
672
673
    if (jobState.getStatus().isActive()) {
      return false;
    }
    int taskCount = jobState.getTotalTaskCount();
    if (taskCount > 0) {
      PipelineService pipeline = PipelineServiceFactory.newPipelineService();
      pipeline.startNewPipeline(new DeleteShardedJob(jobId, taskCount));
    }
    final Key jobKey = ShardedJobStateImpl.ShardedJobSerializer.makeKey(jobId);
    RetryHelper.runWithRetries(callable(new Runnable() {
      @Override
      public void run() {
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.pipeline.PipelineService

  • com.google.appengine.demos.mapreduce.bigqueryload.RandomDataLoadServlet
  • com.google.appengine.demos.mapreduce.entitycount.DeleteMatchingBlobs
  • com.google.appengine.demos.mapreduce.entitycount.ExampleServlet
  • com.google.appengine.demos.mapreduce.randomcollisions.UsingPipelineServlet
  • com.google.appengine.tck.mapreduce.MapReduceTestBase
  • com.google.appengine.tools.mapreduce.CustomOutputTest
  • com.google.appengine.tools.mapreduce.impl.pipeline.CleanupPipelineJob
  • com.google.appengine.tools.mapreduce.impl.pipeline.DeletePipelineJob
  • com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunner
  • com.google.appengine.tools.mapreduce.MapJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.