Package com.google.appengine.tools.pipeline.impl.backend

Examples of com.google.appengine.tools.pipeline.impl.backend.UpdateSpec$Group


          caughtException);
    }
    if (jobRecord.isIgnoreException()) {
      return;
    }
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    ExceptionRecord exceptionRecord = new ExceptionRecord(
        jobRecord.getRootJobKey(), jobRecord.getKey(), currentRunGUID, caughtException);
    updateSpec.getNonTransactionalGroup().includeException(exceptionRecord);
    Key exceptionKey = exceptionRecord.getKey();
    jobRecord.setExceptionKey(exceptionKey);
    if (jobRecord.isCallExceptionHandler() || attemptNumber >= maxAttempts) {
      jobRecord.setState(State.STOPPED);
      updateSpec.getNonTransactionalGroup().includeJob(jobRecord);
      if (jobRecord.isExceptionHandlerSpecified()) {
        cancelChildren(jobRecord, null);
        executeExceptionHandler(updateSpec, jobRecord, caughtException, false);
      } else {
        if (null != jobRecord.getExceptionHandlingAncestorKey()) {
          cancelChildren(jobRecord, null);
          // current job doesn't have an error handler. So just delegate it to the
          // nearest ancestor that has one.
          Task handleChildExceptionTask = new HandleChildExceptionTask(
              jobRecord.getExceptionHandlingAncestorKey(), jobRecord.getKey(),
              jobRecord.getQueueSettings());
          updateSpec.getFinalTransaction().registerTask(handleChildExceptionTask);
        } else {
          rootJobRecord.setState(State.STOPPED);
          rootJobRecord.setExceptionKey(exceptionKey);
          updateSpec.getNonTransactionalGroup().includeJob(rootJobRecord);
        }
      }
      backEnd.save(updateSpec, jobRecord.getQueueSettings());
    } else {
      jobRecord.setState(State.RETRY);
      int backoffFactor = jobRecord.getBackoffFactor();
      int backoffSeconds = jobRecord.getBackoffSeconds();
      RunJobTask task =
          new RunJobTask(jobRecord.getKey(), attemptNumber, jobRecord.getQueueSettings());
      task.getQueueSettings().setDelayInSeconds(
          backoffSeconds * (long) Math.pow(backoffFactor, attemptNumber));
      updateSpec.getFinalTransaction().includeJob(jobRecord);
      updateSpec.getFinalTransaction().registerTask(task);
      backEnd.saveWithJobStateCheck(updateSpec, jobRecord.getQueueSettings(), jobRecord.getKey(),
          State.WAITING_TO_RUN, State.RETRY);
    }
  }
View Full Code Here


   * @return The pipelineID of the newly created pipeline, also known as the
   *         rootJobID.
   */
  public static String startNewPipeline(
      JobSetting[] settings, Job<?> jobInstance, Object... params) {
    UpdateSpec updateSpec = new UpdateSpec(null);
    String graphGUID = null; // The root job graph doesn't have a GUID
    JobRecord parentJobKey = null; // The root job graph doesn't have a parent
    // Create the root Job and its associated Barriers and Slots
    // Create HandleSlotFilledTasks for the input parameters.
    JobRecord jobRecord =
        registerNewJobRecord(updateSpec, settings, parentJobKey, graphGUID, jobInstance, params);
    updateSpec.setRootJobKey(jobRecord.getRootJobKey());
    // Save the Pipeline model objects and enqueue the tasks that start the
    // Pipeline executing.
    backEnd.save(updateSpec);
    return jobRecord.getKey().getName();
  }
View Full Code Here

  public static void stopJob(String jobHandle) throws NoSuchObjectException {
    checkNonEmpty(jobHandle, "jobHandle");
    Key key = KeyFactory.createKey(JobRecord.DATA_STORE_KIND, jobHandle);
    JobRecord jobRecord = backEnd.queryJob(key, JobRecord.InflationType.NONE);
    jobRecord.setState(JobRecord.State.STOPPED);
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    updateSpec.getTransaction("stopJob").includeJob(jobRecord);
    backEnd.save(updateSpec);
  }
View Full Code Here

    }
    if (!childGraphGuid.equals(slot.getGraphGuid())) {
      // The slot has been orphaned
      throw new OrphanedObjectException(promiseHandle);
    }
    UpdateSpec updateSpec = new UpdateSpec(slot.getRootJobKey());
    registerSlotFilled(updateSpec, slot, value);
    backEnd.save(updateSpec);
  }
View Full Code Here

    }

    // Release the run barrier now so any concurrent HandleSlotFilled tasks
    // will stop trying to release it.
    runBarrier.setReleased();
    UpdateSpec updateSpec = new UpdateSpec(rootJobKey);
    updateSpec.getTransaction("releaseRunBarrier").includeBarrier(runBarrier);
    backEnd.save(updateSpec);
    updateSpec = new UpdateSpec(rootJobKey);

    switch (jobState) {
    case WAITING_TO_RUN:
    case RETRY:
        // OK, proceed
        break;
    case WAITING_TO_FINALIZE:
        logger.info("This job has already been run " + jobRecord);
        return;
    case STOPPED:
        logger.info("This job has been stoped. " + jobRecord);
        return;
    }

    // Deserialize the instance of Job and set some values on the instance
    JobInstanceRecord record = jobRecord.getJobInstanceInflated();
    if (null == record) {
      throw new RuntimeException("Internal logic error:" + jobRecord
          + " does not have jobInstanceInflated.");
    }
    Job<?> jobObject = record.getJobInstanceDeserialized();
    setJobRecord(jobObject, jobRecord);
    String currentRunGUID = GUIDGenerator.nextGUID();
    setCurrentRunGuid(jobObject, currentRunGUID);
    setUpdateSpec(jobObject, updateSpec);

    // Get the run() method we will invoke and its arguments
    Object[] params = runBarrier.buildArgumentArray();
    Method runMethod = findAppropriateRunMethod(jobObject.getClass(), params);
    if (logger.isLoggable(Level.FINEST)) {
      StringBuilder builder = new StringBuilder(1024);
      builder.append("Running " + jobRecord + " with params: ");
      builder.append(StringUtils.toString(params));
      logger.finest(builder.toString());
    }

    // Set the Job's start time and save the jobRecord now before we invoke
    // run(). The start time will be displayed in the UI.
    jobRecord.incrementAttemptNumber();
    jobRecord.setStartTime(new Date());
    UpdateSpec tempSpec = new UpdateSpec(jobRecord.getRootJobKey());
    tempSpec.getNonTransactionalGroup().includeJob(jobRecord);
    backEnd.save(tempSpec);

    // Invoke the run() method. This has the side-effect of populating
    // the UpdateSpec with any child job graph generated by the run().
    Value<?> returnValue = null;
View Full Code Here

    int attemptNumber = jobRecord.getAttemptNumber();
    int maxAttempts = jobRecord.getMaxAttempts();
    String message = StringUtils.printStackTraceToString(e);
    jobRecord.setErrorMessage(message);
    Key thisJobKey = jobRecord.getKey();
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    if (attemptNumber >= maxAttempts) {
      jobRecord.setState(JobRecord.State.STOPPED);
      rootJobRecord.setState(JobRecord.State.STOPPED);
      rootJobRecord.setErrorMessage(message);
      updateSpec.getNonTransactionalGroup().includeJob(jobRecord);
      updateSpec.getNonTransactionalGroup().includeJob(rootJobRecord);
      backEnd.save(updateSpec);
    } else {
      jobRecord.setState(JobRecord.State.RETRY);
      updateSpec.getNonTransactionalGroup().includeJob(jobRecord);
      updateSpec.getNonTransactionalGroup().includeJob(rootJobRecord);
      backEnd.save(updateSpec);
      int backoffFactor = jobRecord.getBackoffFactor();
      int backoffSeconds = jobRecord.getBackoffSeconds();
      Task task = new RunJobTask(jobRecord.getKey(), attemptNumber);
      task.setDelaySeconds(backoffSeconds * (long) Math.pow(backoffFactor, attemptNumber));
View Full Code Here

    }

    // release the finalize barrier now so that any concurrent
    // HandleSlotFilled tasks will stop trying
    finalizeBarrier.setReleased();
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    updateSpec.getTransaction("releaseFinalizeBarrier").includeBarrier(finalizeBarrier);
    backEnd.save(updateSpec);
    updateSpec = new UpdateSpec(jobRecord.getRootJobKey());

    // Copy the finalize value to the output slot
    List<Object> finalizeArguments = finalizeBarrier.buildArgumentList();
    int numFinalizeArguments = finalizeArguments.size();
    if (1 != numFinalizeArguments) {
      throw new RuntimeException("Internal logic error: numFinalizeArguments="
          + numFinalizeArguments);
    }
    Object finalizeValue = finalizeArguments.get(0);
    logger.finest("Finalizing " + jobRecord + " with value=" + finalizeValue);
    outputSlot.fill(finalizeValue);

    // Change state of the job to FINALIZED and set the end time
    jobRecord.setState(JobRecord.State.FINALIZED);
    jobRecord.setEndTime(new Date());

    // Propagate the filler of the finalize slot to also be the filler of the
    // output slot. If there is no unique filler of the finalize slot then we
    // resort to assigning the current job as the filler job.
    Key fillerJobKey = getFinalizeSlotFiller(finalizeBarrier);
    if (null == fillerJobKey) {
      fillerJobKey = jobKey;
    }
    outputSlot.setSourceJobKey(fillerJobKey);

    // Save the job and the output slot
    updateSpec.getNonTransactionalGroup().includeJob(jobRecord);
    updateSpec.getNonTransactionalGroup().includeSlot(outputSlot);
    backEnd.save(updateSpec);

    // enqueue a HandleSlotFilled task
    backEnd.enqueue(new HandleSlotFilledTask(outputSlot));
View Full Code Here

    Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
      abuilder.withName("output").withMinimum(1).withMaximum(1).create()).withDescription(
      "The output directory").withShortName("o").create();
   
   
    Group group = gbuilder.withName("Options").withOption(helpOpt).withOption(
        inputDirOpt).withOption(outputOpt).create();
   
    //.withOption(gramSizeOpt).withOption(typeOpt)
   
    try {
View Full Code Here

   
    Option modelOpt = obuilder.withLongName("model").withRequired(true).withArgument(
      abuilder.withName("index").withMinimum(1).withMaximum(1).create()).withDescription(
      "The directory containing the index model").withShortName("m").create();

    Group group = gbuilder.withName("Options").withOption(helpOpt)
        .withOption(inputDirOpt).withOption(modelOpt).create();
   
    try {
      Parser parser = new Parser();
     
View Full Code Here

   
    Option typeOpt = obuilder.withLongName("classifierType").withRequired(false).withArgument(
      abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).withDescription(
      "Type of classifier: knn|tfidf. Default: bayes").withShortName("type").create();
   
    Group group = gbuilder.withName("Options").withOption(gramSizeOpt).withOption(helpOpt).withOption(
        inputDirOpt).withOption(modelOpt).withOption(typeOpt).withOption(contentFieldOpt)
        .withOption(categoryFieldOpt).withOption(maxResultsOpt)
        .create();
   
    try {
View Full Code Here

TOP

Related Classes of com.google.appengine.tools.pipeline.impl.backend.UpdateSpec$Group

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.