Package eu.stratosphere.nephele.executiongraph

Examples of eu.stratosphere.nephele.executiongraph.ExecutionGraph


    synchronized (this.jobQueue) {

      final Iterator<ExecutionGraph> it = this.jobQueue.iterator();
      while (it.hasNext()) {

        final ExecutionGraph executionGraph = it.next();
        if (executionGraph.getJobID().equals(jobID)) {
          return executionGraph;
        }
      }
    }
View Full Code Here


    synchronized (this.jobQueue) {

      final Iterator<ExecutionGraph> it = this.jobQueue.iterator();
      while (it.hasNext()) {

        final ExecutionGraph executionGraph = it.next();
        if (executionGraph.getJobID().equals(executionGraphToRemove.getJobID())) {
          removedFromQueue = true;
          it.remove();
          break;
        }
      }
View Full Code Here

    synchronized (this.jobQueue) {

      final Iterator<ExecutionGraph> it = this.jobQueue.iterator();
      while (it.hasNext()) {

        final ExecutionGraph executionGraph = it.next();
        if (executionGraph.getJobID().equals(jobID)) {
          return executionGraph;
        }
      }
    }
View Full Code Here

        LOG.debug("Available instance is of type DummyInstance!");
        return;
      }
    }

    final ExecutionGraph eg = getExecutionGraphByID(jobID);

    if (eg == null) {
      /*
       * The job have have been canceled in the meantime, in this case
       * we release the instance immediately.
       */
      try {
        for (final AllocatedResource allocatedResource : allocatedResources) {
          getInstanceManager().releaseAllocatedResource(jobID, null, allocatedResource);
        }
      } catch (InstanceException e) {
        LOG.error(e);
      }
      return;
    }

    final Runnable command = new Runnable() {

      /**
       * {@inheritDoc}
       */
      @Override
      public void run() {

        final ExecutionStage stage = eg.getCurrentExecutionStage();

        synchronized (stage) {

          for (final AllocatedResource allocatedResource : allocatedResources) {

            AllocatedResource resourceToBeReplaced = null;
            // Important: only look for instances to be replaced in the current stage
            final Iterator<ExecutionGroupVertex> groupIterator = new ExecutionGroupVertexIterator(eg, true,
              stage.getStageNumber());
            while (groupIterator.hasNext()) {

              final ExecutionGroupVertex groupVertex = groupIterator.next();
              for (int i = 0; i < groupVertex.getCurrentNumberOfGroupMembers(); ++i) {

                final ExecutionVertex vertex = groupVertex.getGroupMember(i);

                if (vertex.getExecutionState() == ExecutionState.SCHEDULED
                  && vertex.getAllocatedResource() != null) {
                  // In local mode, we do not consider any topology, only the instance type
                  if (vertex.getAllocatedResource().getInstanceType().equals(
                    allocatedResource.getInstanceType())) {
                    resourceToBeReplaced = vertex.getAllocatedResource();
                    break;
                  }
                }
              }

              if (resourceToBeReplaced != null) {
                break;
              }
            }

            // For some reason, we don't need this instance
            if (resourceToBeReplaced == null) {
              LOG.error("Instance " + allocatedResource.getInstance() + " is not required for job"
                + eg.getJobID());
              try {
                getInstanceManager().releaseAllocatedResource(jobID, eg.getJobConfiguration(),
                  allocatedResource);
              } catch (InstanceException e) {
                LOG.error(e);
              }
              return;
            }

            // Replace the selected instance
            final Iterator<ExecutionVertex> it = resourceToBeReplaced.assignedVertices();
            while (it.hasNext()) {
              final ExecutionVertex vertex = it.next();
              vertex.setAllocatedResource(allocatedResource);
              vertex.updateExecutionState(ExecutionState.ASSIGNED);
            }
          }
        }

        // Deploy the assigned vertices
        deployAssignedInputVertices(eg);

      }

    };

    eg.executeCommand(command);
  }
View Full Code Here


  @Override
  public void allocatedResourcesDied(final JobID jobID, final List<AllocatedResource> allocatedResources) {

    final ExecutionGraph eg = getExecutionGraphByID(jobID);

    if (eg == null) {
      LOG.error("Cannot find execution graph for job with ID " + jobID);
      return;
    }

    final Runnable command = new Runnable() {

      /**
       * {@inheritDoc}
       */
      @Override
      public void run() {

        synchronized (eg) {

          for (final AllocatedResource allocatedResource : allocatedResources) {

            LOG.info("Resource " + allocatedResource.getInstance().getName() + " for Job " + jobID
              + " died.");

            final ExecutionGraph executionGraph = getExecutionGraphByID(jobID);

            if (executionGraph == null) {
              LOG.error("Cannot find execution graph for job " + jobID);
              return;
            }
View Full Code Here

    if (failedVertex.getExecutionState() != ExecutionState.FAILED) {
      LOG.error("Vertex " + failedVertex + " is requested to be recovered, but is not failed");
      return false;
    }

    final ExecutionGraph eg = failedVertex.getExecutionGraph();
    synchronized (eg) {

      LOG.info("Starting recovery for failed vertex " + failedVertex);

      final Set<ExecutionVertex> verticesToBeCanceled = new HashSet<ExecutionVertex>();
View Full Code Here

  @Override
  public void executionStateChanged(final JobID jobID, final ExecutionVertexID vertexID,
      final ExecutionState newExecutionState, final String optionalMessage) {

    final ExecutionGraph eg = this.executionVertex.getExecutionGraph();

    // Check if we can deploy a new pipeline.
    if (newExecutionState == ExecutionState.FINISHING) {

      final ExecutionPipeline pipeline = this.executionVertex.getExecutionPipeline();
      if (!pipeline.isFinishing()) {
        // Some tasks of the pipeline are still running
        return;
      }

      // Find another vertex in the group which is still in SCHEDULED state and get its pipeline.
      final ExecutionGroupVertex groupVertex = this.executionVertex.getGroupVertex();
      for (int i = 0; i < groupVertex.getCurrentNumberOfGroupMembers(); ++i) {
        final ExecutionVertex groupMember = groupVertex.getGroupMember(i);
        if (groupMember.compareAndUpdateExecutionState(ExecutionState.SCHEDULED, ExecutionState.ASSIGNED)) {

          final ExecutionPipeline pipelineToBeDeployed = groupMember.getExecutionPipeline();
          pipelineToBeDeployed.setAllocatedResource(this.executionVertex.getAllocatedResource());
          pipelineToBeDeployed.updateExecutionState(ExecutionState.ASSIGNED);

          this.scheduler.deployAssignedPipeline(pipelineToBeDeployed);
          return;
        }
      }
    }

    if (newExecutionState == ExecutionState.CANCELED || newExecutionState == ExecutionState.FINISHED) {

      synchronized (eg) {

        if (this.scheduler.getVerticesToBeRestarted().remove(this.executionVertex.getID()) != null) {

          if (eg.getJobStatus() == InternalJobStatus.FAILING) {
            return;
          }

          this.executionVertex.updateExecutionState(ExecutionState.ASSIGNED, "Restart as part of recovery");

          // Run through the deployment procedure
          this.scheduler.deployAssignedVertices(this.executionVertex);
          return;
        }
      }
    }

    if (newExecutionState == ExecutionState.FINISHED || newExecutionState == ExecutionState.CANCELED
      || newExecutionState == ExecutionState.FAILED) {
      // Check if instance can be released
      this.scheduler.checkAndReleaseAllocatedResource(eg, this.executionVertex.getAllocatedResource());
    }

    // In case of an error, check if the vertex shall be recovered
    if (newExecutionState == ExecutionState.FAILED) {
      if (this.executionVertex.decrementRetriesLeftAndCheck()) {

        final Set<ExecutionVertex> assignedVertices = new HashSet<ExecutionVertex>();

        if (RecoveryLogic.recover(this.executionVertex, this.scheduler.getVerticesToBeRestarted(),
          assignedVertices)) {

          if (RecoveryLogic.hasInstanceAssigned(this.executionVertex)) {
            // Run through the deployment procedure
            this.scheduler.deployAssignedVertices(assignedVertices);
          }

        } else {

          // Make sure the map with the vertices to be restarted is cleaned up properly
          synchronized (eg) {

            final Iterator<ExecutionVertex> it = this.scheduler.getVerticesToBeRestarted().values()
              .iterator();

            while (it.hasNext()) {
              if (eg.equals(it.next().getExecutionGraph())) {
                it.remove();
              }
            }
          }
View Full Code Here

   * @throws InstanceException
   *         thrown if the given execution graph is already processing its final stage
   */
  protected void requestInstances(final ExecutionStage executionStage) throws InstanceException {

    final ExecutionGraph executionGraph = executionStage.getExecutionGraph();
    final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();

    synchronized (executionStage) {

      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);

      final Iterator<Map.Entry<InstanceType, Integer>> it = instanceRequestMap.getMinimumIterator();
      LOG.info("Requesting the following instances for job " + executionGraph.getJobID());
      while (it.hasNext()) {
        final Map.Entry<InstanceType, Integer> entry = it.next();
        LOG.info(" " + entry.getKey() + " [" + entry.getValue().intValue() + ", "
          + instanceRequestMap.getMaximumNumberOfInstances(entry.getKey()) + "]");
      }

      if (instanceRequestMap.isEmpty()) {
        return;
      }

      this.instanceManager.requestInstance(executionGraph.getJobID(), executionGraph.getJobConfiguration(),
        instanceRequestMap, null);

      // Switch vertex state to assigning
      final ExecutionGraphIterator it2 = new ExecutionGraphIterator(executionGraph, executionGraph
        .getIndexOfCurrentExecutionStage(), true, true);
      while (it2.hasNext()) {

        it2.next().compareAndUpdateExecutionState(ExecutionState.CREATED, ExecutionState.SCHEDULED);
      }
View Full Code Here

TOP

Related Classes of eu.stratosphere.nephele.executiongraph.ExecutionGraph

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.