Examples of VertexEventRouteEvent


Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

              tezEvent.setDestinationInfo(getDestinationMetaData(inputName));
              return tezEvent;
            }
          });
      appContext.getEventHandler().handle(
          new VertexEventRouteEvent(managedVertex.getVertexId(), Lists.newArrayList(tezEvents)));
      // Recovery handling is taken care of by the Vertex.
    }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

  private static class RouteEventTransition  implements
  SingleArcTransition<VertexImpl, VertexEvent> {
    @Override
    public void transition(VertexImpl vertex, VertexEvent event) {
      VertexEventRouteEvent rEvent = (VertexEventRouteEvent) event;
      boolean recovered = rEvent.isRecovered();
      List<TezEvent> tezEvents = rEvent.getEvents();

      if (vertex.getAppContext().isRecoveryEnabled()
          && !recovered
          && !tezEvents.isEmpty()) {
        List<TezEvent> dataMovementEvents =
            Lists.newArrayList();
        for (TezEvent tezEvent : tezEvents) {
          if (!isEventFromVertex(vertex, tezEvent.getSourceInfo())) {
            continue;
          }
          if  (tezEvent.getEventType().equals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT)
            || tezEvent.getEventType().equals(EventType.DATA_MOVEMENT_EVENT)
            || tezEvent.getEventType().equals(EventType.ROOT_INPUT_DATA_INFORMATION_EVENT)) {
            dataMovementEvents.add(tezEvent);
          }
        }
        if (!dataMovementEvents.isEmpty()) {
          VertexDataMovementEventsGeneratedEvent historyEvent =
              new VertexDataMovementEventsGeneratedEvent(vertex.vertexId,
                  dataMovementEvents);
          vertex.appContext.getHistoryHandler().handle(
              new DAGHistoryEvent(vertex.getDAGId(), historyEvent));
        }
      }
      for(TezEvent tezEvent : tezEvents) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Vertex: " + vertex.getName() + " routing event: "
              + tezEvent.getEventType()
              + " Recovered:" + recovered);
        }
        EventMetaData sourceMeta = tezEvent.getSourceInfo();
        switch(tezEvent.getEventType()) {
        case INPUT_FAILED_EVENT:
        case DATA_MOVEMENT_EVENT:
        case COMPOSITE_DATA_MOVEMENT_EVENT:
          {
            if (isEventFromVertex(vertex, sourceMeta)) {
              // event from this vertex. send to destination vertex
              TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID();
              if (tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT) {
                ((DataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else if (tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) {
                ((CompositeDataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else {
                ((InputFailedEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              }
              Vertex destVertex = vertex.getDAG().getVertex(sourceMeta.getEdgeVertexName());
              Edge destEdge = vertex.targetVertices.get(destVertex);
              if (destEdge == null) {
                throw new TezUncheckedException("Bad destination vertex: " +
                    sourceMeta.getEdgeVertexName() + " for event vertex: " +
                    vertex.getVertexId());
              }
              vertex.eventHandler.handle(new VertexEventRouteEvent(destVertex
                  .getVertexId(), Collections.singletonList(tezEvent)));
            } else {
              // event not from this vertex. must have come from source vertex.
              // send to tasks
              if (vertex.tasksNotYetScheduled) {
                vertex.pendingTaskEvents.add(tezEvent);
              } else {
                Edge srcEdge = vertex.sourceVertices.get(vertex.getDAG().getVertex(
                    sourceMeta.getTaskVertexName()));
                if (srcEdge == null) {
                  throw new TezUncheckedException("Bad source vertex: " +
                      sourceMeta.getTaskVertexName() + " for destination vertex: " +
                      vertex.getVertexId());
                }
                srcEdge.sendTezEventToDestinationTasks(tezEvent);
              }
            }
          }
          break;
        case ROOT_INPUT_DATA_INFORMATION_EVENT:
          if (vertex.tasksNotYetScheduled) {
            vertex.pendingTaskEvents.add(tezEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent
                .getEvent();
            Task targetTask = vertex.getTask(riEvent.getTargetIndex());
            targetTask.registerTezEvent(tezEvent);
          }
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(vmEvent.getTargetVertexName());
          Preconditions.checkArgument(target != null,
              "Event sent to unkown vertex: " + vmEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.vertexManager.onVertexManagerEventReceived(vmEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            vertex.eventHandler.handle(new VertexEventRouteEvent(target
                .getVertexId(), Collections.singletonList(tezEvent)));
          }
        }
          break;
        case ROOT_INPUT_INITIALIZER_EVENT:
        {
          InputInitializerEvent riEvent = (InputInitializerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(riEvent.getTargetVertexName());
          Preconditions.checkArgument(target != null,
              "Event sent to unkown vertex: " + riEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.rootInputInitializerManager.handleInitializerEvent(riEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            vertex.eventHandler.handle(new VertexEventRouteEvent(target.getVertexId(),
                Collections.singletonList(tezEvent)));
          }
        }
          break;
        case INPUT_READ_ERROR_EVENT:
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    taskEvents.add(tezEvent1);
    taskEvents.add(tezEvent2);
    // send events and test that they are buffered until some task is scheduled
    dispatcher.getEventHandler().handle(
        new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(2, v3.pendingTaskEvents.size());
    v3.scheduleTasks(Collections.singletonList(new Integer(0)));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
    // send events and test that they are not buffered anymore
    dispatcher.getEventHandler().handle(
        new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
  }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    taskEvents.add(tezEvent1);
    taskEvents.add(tezEvent2);
    // send events and test that they are buffered until some task is scheduled
    dispatcher.getEventHandler().handle(
        new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(2, v3.pendingTaskEvents.size());
    v3.scheduleTasks(Collections.singletonList(new Integer(0)));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
    // send events and test that they are not buffered anymore
    dispatcher.getEventHandler().handle(
        new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
  }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

              new EventMetaData(EventProducerConsumerType.SYSTEM,
                  vertex.getName(),
                  edgeVertex.getName(),
                  attempt.getID())));
        }
        attempt.sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents));
      }
      // Not checking for leafVertex since a READ_ERROR should only be reported for intermediate tasks.
      if (attempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED) {
        (new TerminatedAfterSuccessHelper(FAILED_HELPER)).transition(
            attempt, event);
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

    readLock.lock();
    try {
      tasksNotYetScheduled = false;
      if (!pendingTaskEvents.isEmpty()) {
        VertexImpl.ROUTE_EVENT_TRANSITION.transition(this,
            new VertexEventRouteEvent(getVertexId(), pendingTaskEvents));
        pendingTaskEvents.clear();
      }
      for (Integer taskID : taskIDs) {
        if (tasks.size() <= taskID.intValue()) {
          throw new TezUncheckedException(
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

        ((InputFailedEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
      } else if (tezEvent.getEventType() == EventType.ROOT_INPUT_DATA_INFORMATION_EVENT) {
        if (vertexState == VertexState.RUNNING
            || vertexState == VertexState.INITED) {
          // Only routed if vertex is still running
          eventHandler.handle(new VertexEventRouteEvent(
              this.getVertexId(), Collections.singletonList(tezEvent), true));
        }
        continue;
      }

      Vertex destVertex = getDAG().getVertex(sourceMeta.getEdgeVertexName());
      Edge destEdge = targetVertices.get(destVertex);
      if (destEdge == null) {
        throw new TezUncheckedException("Bad destination vertex: " +
            sourceMeta.getEdgeVertexName() + " for event vertex: " +
            getVertexId());
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("Routing recovered event"
            + ", eventType=" + tezEvent.getEventType()
            + ", sourceInfo=" + sourceMeta
            + ", destinationVertex" + destVertex.getName());
      }
      eventHandler.handle(new VertexEventRouteEvent(destVertex
          .getVertexId(), Collections.singletonList(tezEvent), true));
    }
  }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

    }

    // Vertex will be moving to INITED state, safe to process pending route events.
    if (pendingRouteEventsWhileIniting != null) {
      VertexImpl.ROUTE_EVENT_TRANSITION.transition(this,
          new VertexEventRouteEvent(getVertexId(), pendingRouteEventsWhileIniting));
      pendingRouteEventsWhileIniting = null;
    }
    return vertexState;
  }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

  private static class RouteEventsWhileInitializingTransition implements
      SingleArcTransition<VertexImpl, VertexEvent> {

    @Override
    public void transition(VertexImpl vertex, VertexEvent event) {
      VertexEventRouteEvent re = (VertexEventRouteEvent) event;
      if (vertex.pendingRouteEventsWhileIniting == null) {
        vertex.pendingRouteEventsWhileIniting = Lists.newLinkedList();
      }
      // Store the events for post-init routing, since INIT state is when
      // initial task parallelism will be set
      vertex.pendingRouteEventsWhileIniting.addAll(re.getEvents());
    }
View Full Code Here

Examples of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent

  private static class RouteEventTransition  implements
  SingleArcTransition<VertexImpl, VertexEvent> {
    @Override
    public void transition(VertexImpl vertex, VertexEvent event) {
      VertexEventRouteEvent rEvent = (VertexEventRouteEvent) event;
      boolean recovered = rEvent.isRecovered();
      List<TezEvent> tezEvents = rEvent.getEvents();

      if (vertex.getAppContext().isRecoveryEnabled()
          && !recovered
          && !tezEvents.isEmpty()) {
        List<TezEvent> dataMovementEvents =
            Lists.newArrayList();
        for (TezEvent tezEvent : tezEvents) {
          if (!isEventFromVertex(vertex, tezEvent.getSourceInfo())) {
            continue;
          }
          if  (tezEvent.getEventType().equals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT)
            || tezEvent.getEventType().equals(EventType.DATA_MOVEMENT_EVENT)
            || tezEvent.getEventType().equals(EventType.ROOT_INPUT_DATA_INFORMATION_EVENT)) {
            dataMovementEvents.add(tezEvent);
          }
        }
        if (!dataMovementEvents.isEmpty()) {
          VertexDataMovementEventsGeneratedEvent historyEvent =
              new VertexDataMovementEventsGeneratedEvent(vertex.vertexId,
                  dataMovementEvents);
          vertex.appContext.getHistoryHandler().handle(
              new DAGHistoryEvent(vertex.getDAGId(), historyEvent));
        }
      }
      for(TezEvent tezEvent : tezEvents) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Vertex: " + vertex.getName() + " routing event: "
              + tezEvent.getEventType()
              + " Recovered:" + recovered);
        }
        EventMetaData sourceMeta = tezEvent.getSourceInfo();
        switch(tezEvent.getEventType()) {
        case INPUT_FAILED_EVENT:
        case DATA_MOVEMENT_EVENT:
        case COMPOSITE_DATA_MOVEMENT_EVENT:
          {
            if (isEventFromVertex(vertex, sourceMeta)) {
              // event from this vertex. send to destination vertex
              TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID();
              if (tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT) {
                ((DataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else if (tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) {
                ((CompositeDataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else {
                ((InputFailedEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              }
              Vertex destVertex = vertex.getDAG().getVertex(sourceMeta.getEdgeVertexName());
              Edge destEdge = vertex.targetVertices.get(destVertex);
              if (destEdge == null) {
                throw new TezUncheckedException("Bad destination vertex: " +
                    sourceMeta.getEdgeVertexName() + " for event vertex: " +
                    vertex.getVertexId());
              }
              vertex.eventHandler.handle(new VertexEventRouteEvent(destVertex
                  .getVertexId(), Collections.singletonList(tezEvent)));
            } else {
              // event not from this vertex. must have come from source vertex.
              // send to tasks
              if (vertex.tasksNotYetScheduled) {
                vertex.pendingTaskEvents.add(tezEvent);
              } else {
                Edge srcEdge = vertex.sourceVertices.get(vertex.getDAG().getVertex(
                    sourceMeta.getTaskVertexName()));
                if (srcEdge == null) {
                  throw new TezUncheckedException("Bad source vertex: " +
                      sourceMeta.getTaskVertexName() + " for destination vertex: " +
                      vertex.getVertexId());
                }
                srcEdge.sendTezEventToDestinationTasks(tezEvent);
              }
            }
          }
          break;
        case ROOT_INPUT_DATA_INFORMATION_EVENT:
          checkEventSourceMetadata(vertex, sourceMeta);
          RootInputDataInformationEvent riEvent = (RootInputDataInformationEvent) tezEvent
              .getEvent();
          TezTaskID targetTaskID = TezTaskID.getInstance(vertex.getVertexId(),
              riEvent.getTargetIndex());
          vertex.eventHandler.handle(new TaskEventAddTezEvent(targetTaskID, tezEvent));         
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(vmEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.vertexManager.onVertexManagerEventReceived(vmEvent);
          } else {
            vertex.eventHandler.handle(new VertexEventRouteEvent(target
                .getVertexId(), Collections.singletonList(tezEvent)));
          }
        }
          break;
        case INPUT_READ_ERROR_EVENT:
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.