Examples of HistoryEventType

org.apache.tez.dag.history.HistoryEventType
org.camunda.bpm.engine.impl.history.event.HistoryEventType
An history event type. See {@link HistoryEventTypes} for a set of built-in events @author Daniel Meyer @since 7.2

Examples of org.apache.tez.dag.history.HistoryEventType

    if (stopped.get()) {
      LOG.warn("Igoring event as service stopped, eventType"
          + event.getHistoryEvent().getEventType());
      return;
    }
    HistoryEventType eventType = event.getHistoryEvent().getEventType();


    if (recoveryFatalErrorOccurred.get()) {
      return;
    }


    if (!started.get()) {
      LOG.warn("Adding event of type " + eventType
          + " to queue as service not started");
      eventQueue.add(event);
      return;
    }


    TezDAGID dagId = event.getDagID();
    if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
      DAGSubmittedEvent dagSubmittedEvent =
          (DAGSubmittedEvent) event.getHistoryEvent();
      String dagName = dagSubmittedEvent.getDAGName();
      if (dagName != null
          && dagName.startsWith(
              TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) {
        // Skip recording pre-warm DAG events
        skippedDAGs.add(dagId);
        return;
      }
    }
    if (dagId == null || skippedDAGs.contains(dagId)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Skipping event for DAG"
            + ", eventType=" + eventType
            + ", dagId=" + (dagId == null ? "null" : dagId.toString())
            + ", isSkippedDAG=" + (dagId == null ? "null"
            : skippedDAGs.contains(dagId)));
      }
      return;
    }


    if (event.getHistoryEvent() instanceof SummaryEvent) {
      synchronized (lock) {
        try {
          SummaryEvent summaryEvent = (SummaryEvent) event.getHistoryEvent();
          handleSummaryEvent(dagId, eventType, summaryEvent);
          summaryStream.hflush();
          if (summaryEvent.writeToRecoveryImmediately()) {
            handleRecoveryEvent(event);
            doFlush(outputStreamMap.get(event.getDagID()),
                appContext.getClock().getTime());
          } else {
            if (LOG.isDebugEnabled()) {
              LOG.debug("Queueing Non-immediate Summary/Recovery event of type"
                  + eventType.name());
            }
            eventQueue.add(event);
          }
          if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
            LOG.info("DAG completed"
                + ", dagId=" + event.getDagID()
                + ", queueSize=" + eventQueue.size());
            completedDAGs.add(dagId);
            if (outputStreamMap.containsKey(dagId)) {
              try {
                outputStreamMap.get(dagId).close();
                outputStreamMap.remove(dagId);
              } catch (IOException ioe) {
                LOG.warn("Error when trying to flush/close recovery file for"
                    + " dag, dagId=" + event.getDagID());
              }
            }
          }
        } catch (IOException ioe) {
          LOG.error("Error handling summary event"
              + ", eventType=" + event.getHistoryEvent().getEventType(), ioe);
          Path fatalErrorDir = new Path(recoveryPath, RECOVERY_FATAL_OCCURRED_DIR);
          try {
            LOG.error("Adding a flag to ensure next AM attempt does not start up"
                + ", flagFile=" + fatalErrorDir.toString());
            recoveryFatalErrorOccurred.set(true);
            recoveryDirFS.mkdirs(fatalErrorDir);
            if (recoveryDirFS.exists(fatalErrorDir)) {
              LOG.error("Recovery failure occurred. Skipping all events");
            } else {
              // throw error if fatal error flag could not be set
              throw ioe;
            }
          } catch (IOException e) {
            LOG.fatal("Failed to create fatal error flag dir "
                + fatalErrorDir.toString(), e);
            throw ioe;
          }
          if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
            // Throw error to tell client that dag submission failed
            throw ioe;
          }
        }
      }
    } else {
      // All other events just get queued
      if (LOG.isDebugEnabled()) {
        LOG.debug("Queueing Non-Summary Recovery event of type " + eventType.name());
      }
      eventQueue.add(event);
    }
  }

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

    }
    summaryEvent.toSummaryProtoStream(summaryStream);
  }


  private void handleRecoveryEvent(DAGHistoryEvent event) throws IOException {
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Handling recovery event of type "
          + event.getHistoryEvent().getEventType());
    }
    TezDAGID dagID = event.getDagID();

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

  public void handle(DAGHistoryEvent event) {
    eventQueue.add(event);
  }


  private void handleEvent(DAGHistoryEvent event) {
    HistoryEventType eventType = event.getHistoryEvent().getEventType();


    TezDAGID dagId = event.getDagID();


    if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
      DAGSubmittedEvent dagSubmittedEvent =
          (DAGSubmittedEvent) event.getHistoryEvent();
      String dagName = dagSubmittedEvent.getDAGName();
      if (dagName != null
          && dagName.startsWith(
              TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) {
        // Skip recording pre-warm DAG events
        skippedDAGs.add(dagId);
        return;
      }
    }
    if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
      // Remove from set to keep size small
      // No more events should be seen after this point.
      if (skippedDAGs.remove(dagId)) {
        return;
      }

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

      // Corrupt data
      // reached end
      throw new IOException("Corrupt data found when trying to read next event type"
          + ", eventTypeOrdinal=" + eventTypeOrdinal);
    }
    HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
    HistoryEvent event;
    switch (eventType) {
      case AM_LAUNCHED:
        event = new AMLaunchedEvent();
        break;

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

        }
      } catch (EOFException eof) {
        LOG.info("Reached end of summary stream");
        break;
      }
      HistoryEventType eventType =
          HistoryEventType.values()[proto.getEventType()];
      if (LOG.isDebugEnabled()) {
        LOG.debug("[RECOVERY SUMMARY]"
            + " dagId=" + proto.getDagId()
            + ", timestamp=" + proto.getTimestamp()
            + ", event=" + eventType);
      }
      TezDAGID dagId = TezDAGID.fromString(proto.getDagId());
      if (dagCounter < dagId.getId()) {
        dagCounter = dagId.getId();
      }
      if (!dagSummaryDataMap.containsKey(dagId)) {
        dagSummaryDataMap.put(dagId, new DAGSummaryData(dagId));
      }
      dagSummaryDataMap.get(dagId).handleSummaryEvent(proto);
      proto.writeDelimitedTo(newSummaryStream);
    }
    summaryStream.close();
    newSummaryStream.hsync();
    newSummaryStream.close();


    // Set counter for next set of DAGs & update dagNames Set in DAGAppMaster
    dagAppMaster.setDAGCounter(dagCounter);
    for (DAGSummaryData dagSummaryData: dagSummaryDataMap.values()){
      dagAppMaster.dagNames.add(dagSummaryData.dagName);
    }


    DAGSummaryData lastInProgressDAGData =
        getLastCompletedOrInProgressDAG(dagSummaryDataMap);
    if (lastInProgressDAGData == null) {
      LOG.info("Nothing to recover as no uncompleted/completed DAGs found");
      return null;
    }
    TezDAGID lastInProgressDAG = lastInProgressDAGData.dagId;
    if (lastInProgressDAG == null) {
      LOG.info("Nothing to recover as no uncompleted/completed DAGs found");
      return null;
    }


    LOG.info("Checking if DAG is in recoverable state"
        + ", dagId=" + lastInProgressDAGData.dagId);


    final RecoveredDAGData recoveredDAGData = new RecoveredDAGData();
    if (lastInProgressDAGData.completed) {
      recoveredDAGData.isCompleted = true;
      recoveredDAGData.dagState = lastInProgressDAGData.dagState;
    }


    String nonRecoverableReason = isDAGRecoverable(lastInProgressDAGData);
    if (nonRecoverableReason != null) {
      LOG.warn("Found last inProgress DAG but not recoverable: "
          + lastInProgressDAGData);
      recoveredDAGData.nonRecoverable = true;
      recoveredDAGData.reason = nonRecoverableReason;
    }


    LOG.info("Trying to recover dag from recovery file"
        + ", dagId=" + lastInProgressDAG.toString()
        + ", dataDir=" + previousAttemptRecoveryDataDir
        + ", intoCurrentDir=" + currentAttemptRecoveryDataDir);


    FSDataInputStream dagRecoveryStream = getDAGRecoveryStream(
        previousAttemptRecoveryDataDir, lastInProgressDAG);
    if (dagRecoveryStream == null) {
      // Could not find data to recover
      // Error out
      throw new IOException("Could not find recovery data for last in progress DAG"
          + ", dagId=" + lastInProgressDAG);
    }


    LOG.info("Copying DAG data into Current Attempt directory"
        + ", filePath=" + getDAGRecoveryFilePath(currentAttemptRecoveryDataDir,
        lastInProgressDAG));
    FSDataOutputStream newDAGRecoveryStream =
        getDAGRecoveryOutputStream(currentAttemptRecoveryDataDir, lastInProgressDAG);


    boolean skipAllOtherEvents = false;
    while (true) {
      HistoryEvent event;
      try {
        event = getNextEvent(dagRecoveryStream);
        if (event == null) {
          LOG.info("Reached end of dag recovery stream");
          break;
        }
      } catch (EOFException eof) {
        LOG.info("Reached end of dag recovery stream");
        break;
      } catch (IOException ioe) {
        LOG.warn("Corrupt data found when trying to read next event", ioe);
        break;
      }
      if (event == null || skipAllOtherEvents) {
        // reached end of data
        break;
      }
      HistoryEventType eventType = event.getEventType();
      switch (eventType) {
        case DAG_SUBMITTED:
        {
          DAGSubmittedEvent submittedEvent = (DAGSubmittedEvent) event;
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          recoveredDAGData.recoveredDAG = dagAppMaster.createDAG(submittedEvent.getDAGPlan(),
              lastInProgressDAG);
          recoveredDAGData.cumulativeAdditionalResources = submittedEvent
            .getCumulativeAdditionalLocalResources();
          recoveredDAGData.recoveredDagID = recoveredDAGData.recoveredDAG.getID();
          dagAppMaster.setCurrentDAG(recoveredDAGData.recoveredDAG);
          if (recoveredDAGData.nonRecoverable) {
            skipAllOtherEvents = true;
          }
          break;
        }
        case DAG_INITIALIZED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case VERTEX_GROUP_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case VERTEX_GROUP_COMMIT_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          // If this is seen, nothing to recover
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          recoveredDAGData.isCompleted = true;
          recoveredDAGData.dagState =
              ((DAGFinishedEvent) event).getState();
          skipAllOtherEvents = true;
        }
        case CONTAINER_LAUNCHED:
        {
          // Nothing to do for now
          break;
        }
        case VERTEX_INITIALIZED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexInitializedEvent vEvent = (VertexInitializedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexStartedEvent vEvent = (VertexStartedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_PARALLELISM_UPDATED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexParallelismUpdatedEvent vEvent = (VertexParallelismUpdatedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexCommitStartedEvent vEvent = (VertexCommitStartedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexFinishedEvent vEvent = (VertexFinishedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case TASK_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskStartedEvent tEvent = (TaskStartedEvent) event;
          Task task = recoveredDAGData.recoveredDAG.getVertex(
              tEvent.getTaskID().getVertexID()).getTask(tEvent.getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskFinishedEvent tEvent = (TaskFinishedEvent) event;
          Task task = recoveredDAGData.recoveredDAG.getVertex(
              tEvent.getTaskID().getVertexID()).getTask(tEvent.getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_ATTEMPT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskAttemptStartedEvent tEvent = (TaskAttemptStartedEvent) event;
          Task task =
              recoveredDAGData.recoveredDAG.getVertex(
                  tEvent.getTaskAttemptID().getTaskID().getVertexID())
                      .getTask(tEvent.getTaskAttemptID().getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_ATTEMPT_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskAttemptFinishedEvent tEvent = (TaskAttemptFinishedEvent) event;
          Task task =
              recoveredDAGData.recoveredDAG.getVertex(
                  tEvent.getTaskAttemptID().getTaskID().getVertexID())
                  .getTask(tEvent.getTaskAttemptID().getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexDataMovementEventsGeneratedEvent vEvent =
              (VertexDataMovementEventsGeneratedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        default:
          throw new RuntimeException("Invalid data found, unknown event type "
              + eventType);
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("[DAG RECOVERY]"
            + " dagId=" + lastInProgressDAG
            + ", eventType=" + eventType
            + ", event=" + event.toString());
      }
      newDAGRecoveryStream.writeInt(eventType.ordinal());
      event.toProtoStream(newDAGRecoveryStream);
    }
    dagRecoveryStream.close();
    newDAGRecoveryStream.hsync();
    newDAGRecoveryStream.close();

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

    DAGSummaryData(TezDAGID dagId) {
      this.dagId = dagId;
    }


    void handleSummaryEvent(SummaryEventProto proto) throws IOException {
      HistoryEventType eventType =
          HistoryEventType.values()[proto.getEventType()];
      switch (eventType) {
        case DAG_SUBMITTED:
          completed = false;
          DAGSubmittedEvent dagSubmittedEvent = new DAGSubmittedEvent();

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

  public void handle(DAGHistoryEvent event) {
    eventQueue.add(event);
  }


  private void handleEvent(DAGHistoryEvent event) {
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    try {
      // TODO integrate with ATS
    } catch (Exception e) {
      LOG.warn("Could not handle history event, eventType="
          + eventType, e);

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

        // Corrupt data
        // reached end
        LOG.warn("Corrupt data found when trying to read next event type");
        break;
      }
      HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
      HistoryEvent event = null;
      switch (eventType) {
        case AM_LAUNCHED:
          event = new AMLaunchedEvent();
          break;

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

    }
    if (!started.get()) {
      eventQueue.add(event);
      return;
    }
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    if (eventType.equals(HistoryEventType.DAG_SUBMITTED)
      || eventType.equals(HistoryEventType.DAG_FINISHED)) {
      // handle submissions and completion immediately
      synchronized (lock) {
        try {
          handleEvent(event);
          summaryStream.flush();
          if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
            outputStreamMap.get(event.getDagID()).flush();
          } else if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
            completedDAGs.add(event.getDagID());
            if (outputStreamMap.containsKey(event.getDagID())) {
              try {
                outputStreamMap.get(event.getDagID()).flush();
                outputStreamMap.get(event.getDagID()).close();

View Full Code Here

Examples of org.apache.tez.dag.history.HistoryEventType

    }
  }




  private void handleEvent(DAGHistoryEvent event) {
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Handling recovery event of type "
          + event.getHistoryEvent().getEventType());
    }
    if (event.getDagID() == null) {
      // AM event
      // anything to be done?
      // TODO
      return;
    }


    TezDAGID dagID = event.getDagID();
    if (completedDAGs.contains(dagID)) {
      // Skip events for completed DAGs
      // no need to recover completed DAGs
      return;
    }


    try {


      if (eventType.equals(HistoryEventType.DAG_SUBMITTED)
          || eventType.equals(HistoryEventType.DAG_FINISHED)) {
        if (summaryStream == null) {
          Path summaryPath = new Path(recoveryPath,
              appContext.getApplicationID()
              + TezConfiguration.DAG_RECOVERY_SUMMARY_FILE_SUFFIX);
          summaryStream = recoveryDirFS.create(summaryPath, false,
              bufferSize);
        }
        if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
          DAGSubmittedEvent dagSubmittedEvent =
              (DAGSubmittedEvent) event.getHistoryEvent();
          String dagName = dagSubmittedEvent.getDAGName();
          if (dagName != null
              && dagName.startsWith(

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.