Examples of ExecutionVertex


Examples of eu.stratosphere.nephele.executiongraph.ExecutionVertex

      // Restart all predecessors without checkpoint
      final Iterator<ExecutionVertex> cancelIterator = verticesToBeCanceled.iterator();
      while (cancelIterator.hasNext()) {

        final ExecutionVertex vertex = cancelIterator.next();

        if (vertex.compareAndUpdateExecutionState(ExecutionState.FINISHED, getStateToUpdate(vertex))) {
          LOG.info("Vertex " + vertex + " has already finished and will not be canceled");
          if (vertex.getExecutionState() == ExecutionState.ASSIGNED) {
            assignedVertices.add(vertex);
          }
          continue;
        }

        LOG.info(vertex + " is canceled by recovery logic");
        verticesToBeRestarted.put(vertex.getID(), vertex);
        final TaskCancelResult cancelResult = vertex.cancelTask();

        if (cancelResult.getReturnCode() != AbstractTaskResult.ReturnCode.SUCCESS
            && cancelResult.getReturnCode() != AbstractTaskResult.ReturnCode.TASK_NOT_FOUND) {

          verticesToBeRestarted.remove(vertex.getID());
          LOG.error("Unable to cancel vertex" + cancelResult.getDescription());
          return false;
        }
      }
View Full Code Here

Examples of eu.stratosphere.nephele.executiongraph.ExecutionVertex

    final Set<ExecutionVertex> visited = new HashSet<ExecutionVertex>();
    verticesToTest.add(failedVertex);

    while (!verticesToTest.isEmpty()) {

      final ExecutionVertex vertex = verticesToTest.poll();

      // Predecessors must be either checkpoints or need to be restarted, too
      for (int j = 0; j < vertex.getNumberOfPredecessors(); j++) {
        final ExecutionVertex predecessor = vertex.getPredecessor(j);

        if (hasInstanceAssigned(predecessor)) {
          verticesToBeCanceled.add(predecessor);
        }
View Full Code Here

Examples of eu.stratosphere.nephele.executiongraph.ExecutionVertex

      final ExecutionGate outputGate = vertex.getOutputGate(i);
      for (int j = 0; j < outputGate.getNumberOfEdges(); ++j) {

        final ExecutionEdge outputChannel = outputGate.getEdge(j);

        final ExecutionVertex connectedVertex = outputChannel.getInputGate().getVertex();
        if (connectedVertex == null) {
          LOG.error("Connected vertex is null");
          continue;
        }

        final AbstractInstance instance = connectedVertex.getAllocatedResource().getInstance();
        if (instance instanceof DummyInstance) {
          continue;
        }

        Set<ChannelID> channelIDs = entriesToInvalidate.get(instance);
        if (channelIDs == null) {
          channelIDs = new SerializableHashSet<ChannelID>();
          entriesToInvalidate.put(instance, channelIDs);
        }

        channelIDs.add(outputChannel.getInputChannelID());
      }
    }

    for (int i = 0; i < vertex.getNumberOfInputGates(); ++i) {

      final ExecutionGate inputGate = vertex.getInputGate(i);
      for (int j = 0; j < inputGate.getNumberOfEdges(); ++j) {

        final ExecutionEdge inputChannel = inputGate.getEdge(j);

        final ExecutionVertex connectedVertex = inputChannel.getOutputGate().getVertex();
        if (connectedVertex == null) {
          LOG.error("Connected vertex is null");
          continue;
        }

        final AbstractInstance instance = connectedVertex.getAllocatedResource().getInstance();
        if (instance instanceof DummyInstance) {
          continue;
        }

        Set<ChannelID> channelIDs = entriesToInvalidate.get(instance);
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

    return new Instance(ci, new InstanceID(), resources, numSlots);
  }
 
 
  public static Execution getDummyTask() {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    when(vertex.getJobId()).thenReturn(new JobID());
    when(vertex.toString()).thenReturn("TEST-VERTEX");
   
    Execution execution = mock(Execution.class);
    when(execution.getVertex()).thenReturn(vertex);
   
    return execution;
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

   
    return execution;
  }
 
  public static Execution getTestVertex(Iterable<Instance> preferredLocations) {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
   
    when(vertex.getPreferredLocations()).thenReturn(preferredLocations);
    when(vertex.getJobId()).thenReturn(new JobID());
    when(vertex.toString()).thenReturn("TEST-VERTEX");
   
    Execution execution = mock(Execution.class);
    when(execution.getVertex()).thenReturn(vertex);
   
    return execution;
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

   
    return execution;
  }
 
  public static Execution getTestVertex(JobVertexID jid, int taskIndex, int numTasks) {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
   
    when(vertex.getPreferredLocations()).thenReturn(null);
    when(vertex.getJobId()).thenReturn(new JobID());
    when(vertex.getJobvertexId()).thenReturn(jid);
    when(vertex.getParallelSubtaskIndex()).thenReturn(taskIndex);
    when(vertex.getTotalNumberOfParallelSubtasks()).thenReturn(numTasks);
    when(vertex.toString()).thenReturn("TEST-VERTEX");
    when(vertex.getSimpleName()).thenReturn("TEST-VERTEX");
   
    Execution execution = mock(Execution.class);
    when(execution.getVertex()).thenReturn(vertex);
   
    return execution;
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Scheduling task " + task);
    }
   
    final ExecutionVertex vertex = task.getTaskToExecute().getVertex();
 
    synchronized (globalLock) {
   
      // 1)  === If the task has a slot sharing group, schedule with shared slots ===
     
      SlotSharingGroup sharingUnit = task.getSlotSharingGroup();
      if (sharingUnit != null) {
       
        if (queueIfNoResource) {
          throw new IllegalArgumentException("A task with a vertex sharing group was scheduled in a queued fashion.");
        }
       
        final SlotSharingGroupAssignment assignment = sharingUnit.getTaskAssignment();
        final CoLocationConstraint constraint = task.getLocationConstraint();
       
        // get a slot from the group, if the group has one for us (and can fulfill the constraint)
        SubSlot slotFromGroup;
        if (constraint == null) {
          slotFromGroup = assignment.getSlotForTask(vertex);
        }
        else {
          slotFromGroup = assignment.getSlotForTask(vertex, constraint);
        }
       
        AllocatedSlot newSlot = null;
       
        // the following needs to make sure any allocated slot is released in case of an error
        try {
         
          // check whether the slot from the group is already what we want
          if (slotFromGroup != null) {
            // local (or unconstrained in the current group)
            if (slotFromGroup.getLocality() != Locality.NON_LOCAL) {
              updateLocalityCounters(slotFromGroup.getLocality());
              return slotFromGroup;
            }
          }
         
          final Iterable<Instance> locations = (constraint == null || constraint.isUnassigned()) ?
              vertex.getPreferredLocations() : Collections.singleton(constraint.getLocation());
         
          // get a new slot, since we could not place it into the group, or we could not place it locally
          newSlot = getFreeSlotForTask(vertex, locations);
         
          SubSlot toUse;
         
          if (newSlot == null) {
            if (slotFromGroup == null) {
              // both null
              if (constraint == null || constraint.isUnassigned()) {
                throw new NoResourceAvailableException(getNumberOfAvailableInstances(), getTotalNumberOfSlots());
              } else {
                throw new NoResourceAvailableException("Could not allocate a slot on instance " +
                      constraint.getLocation() + ", as required by the co-location constraint.");
              }
            } else {
              // got a non-local from the group, and no new one
              toUse = slotFromGroup;
            }
          }
          else if (slotFromGroup == null || newSlot.getLocality() == Locality.LOCAL) {
            // new slot is preferable
            if (slotFromGroup != null) {
              slotFromGroup.releaseSlot();
            }
           
            if (constraint == null) {
              toUse = assignment.addNewSlotWithTask(newSlot, vertex);
            } else {
              toUse = assignment.addNewSlotWithTask(newSlot, vertex, constraint);
            }
          }
          else {
            // both are available and usable. neither is local
            newSlot.releaseSlot();
            toUse = slotFromGroup;
          }
         
          // assign to the co-location hint, if we have one and it is unassigned
          // if it was assigned before and the new one is not local, it is a fail
          if (constraint != null) {
            if (constraint.isUnassigned() || toUse.getLocality() == Locality.LOCAL) {
              constraint.setSharedSlot(toUse.getSharedSlot());
            } else {
              // the fail
              throw new NoResourceAvailableException("Could not allocate a slot on instance " +
                  constraint.getLocation() + ", as required by the co-location constraint.");
            }
          }
         
          updateLocalityCounters(toUse.getLocality());
          return toUse;
        }
        catch (NoResourceAvailableException e) {
          throw e;
        }
        catch (Throwable t) {
          if (slotFromGroup != null) {
            slotFromGroup.releaseSlot();
          }
          if (newSlot != null) {
            newSlot.releaseSlot();
          }
         
          ExceptionUtils.rethrow(t, "An error occurred while allocating a slot in a sharing group");
        }
      }
   
      // 2) === schedule without hints and sharing ===
     
      AllocatedSlot slot = getFreeSlotForTask(vertex, vertex.getPreferredLocations());
      if (slot != null) {
        updateLocalityCounters(slot.getLocality());
        return slot;
      }
      else {
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

   
    return execution;
  }
 
  public static Execution getTestVertexWithLocation(JobVertexID jid, int taskIndex, int numTasks, Instance... locations) {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
   
    when(vertex.getPreferredLocations()).thenReturn(Arrays.asList(locations));
    when(vertex.getJobId()).thenReturn(new JobID());
    when(vertex.getJobvertexId()).thenReturn(jid);
    when(vertex.getParallelSubtaskIndex()).thenReturn(taskIndex);
    when(vertex.getTotalNumberOfParallelSubtasks()).thenReturn(numTasks);
    when(vertex.toString()).thenReturn("TEST-VERTEX");
   
    Execution execution = mock(Execution.class);
    when(execution.getVertex()).thenReturn(vertex);
   
    return execution;
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

     
      // the slot was properly released, we can allocate a new one from that instance
     
      if (queued != null) {
        ScheduledUnit task = queued.getTask();
        ExecutionVertex vertex = task.getTaskToExecute().getVertex();
       
        try {
          AllocatedSlot newSlot = instance.allocateSlot(vertex.getJobId());
          if (newSlot != null) {
           
            // success, remove from the task queue and notify the future
            taskQueue.poll();
            if (queued.getFuture() != null) {
              try {
                queued.getFuture().setSlot(newSlot);
              }
              catch (Throwable t) {
                LOG.error("Error calling allocation future for task " + vertex.getSimpleName(), t);
                task.getTaskToExecute().fail(t);
              }
            }
          }
        }
View Full Code Here

Examples of org.apache.flink.runtime.executiongraph.ExecutionVertex

    }
   
    ExecutionVertex[] vertices = jobVertex.getTaskVertices();
   
    for (int j = 0; j < vertices.length; j++) {
      ExecutionVertex vertex = vertices[j];
     
      json.append(toJson(vertex));
     
      // print delimiter
      if (j != vertices.length - 1) {
        json.append(",");
      }
     
      // Increment state status count
      int count =  stateCounts.get(vertex.getExecutionState()) + 1;
      stateCounts.put(vertex.getExecutionState(), count);
    }
   
    json.append("],");
    json.append("\"backwardEdges\": [");
   
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.