Examples of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

org.apache.tez.runtime.api.events.RootInputDataInformationEvent
Events used by {@link TezRootInputInitializer} implementations to provide thebyte payload for individual tasks running as part of the Vertex for which an Initial Input has been configured.

      if (event instanceof RootInputUpdatePayloadEvent) {
        // this event can never occur. If it does, fail.
        Preconditions.checkState(false);
      } else if (event instanceof RootInputDataInformationEvent) {
        dataInformationEventSeen = true;
        RootInputDataInformationEvent diEvent = (RootInputDataInformationEvent) event;
        dataInformationEvents.add(diEvent);
        FileSplit fileSplit;
        try {
          fileSplit = getFileSplitFromEvent(diEvent);
        } catch (IOException e) {

View Full Code Here

    List<RootInputDataInformationEvent> taskEvents = Lists.newArrayListWithCapacity(finalSplits.size());
    // Re-serialize the splits after grouping.
    int count = 0;
    for (InputSplit inputSplit : finalSplits) {
      MRSplitProto serializedSplit = MRHelpers.createSplitProto(inputSplit);
      RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
          count, serializedSplit.toByteArray());
      diEvent.setTargetIndex(count);
      count++;
      taskEvents.add(diEvent);
    }


    // Replace the Edge Managers

View Full Code Here

            }
          }
          break;
        case ROOT_INPUT_DATA_INFORMATION_EVENT:
          checkEventSourceMetadata(vertex, sourceMeta);
          RootInputDataInformationEvent riEvent = (RootInputDataInformationEvent) tezEvent
              .getEvent();
          TezTaskID targetTaskID = TezTaskID.getInstance(vertex.getVertexId(),
              riEvent.getTargetIndex());
          vertex.eventHandler.handle(new TaskEventAddTezEvent(targetTaskID, tezEvent));          
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();

View Full Code Here

    assertEquals(3, events.size());
    assertTrue(events.get(0) instanceof RootInputUpdatePayloadEvent);
    assertTrue(events.get(1) instanceof RootInputDataInformationEvent);
    assertTrue(events.get(2) instanceof RootInputDataInformationEvent);


    RootInputDataInformationEvent diEvent1 = (RootInputDataInformationEvent) (events.get(1));
    RootInputDataInformationEvent diEvent2 = (RootInputDataInformationEvent) (events.get(2));


    assertNull(diEvent1.getDeserializedUserPayload());
    assertNull(diEvent2.getDeserializedUserPayload());


    assertNotNull(diEvent1.getUserPayload());
    assertNotNull(diEvent2.getUserPayload());


    MRSplitProto event1Proto = MRSplitProto.parseFrom(diEvent1.getUserPayload());
    InputSplit is1 = MRInput.getOldSplitDetailsFromEvent(event1Proto, new Configuration());
    assertTrue(is1 instanceof InputSplitForTest);
    assertEquals(1, ((InputSplitForTest) is1).identifier);


    MRSplitProto event2Proto = MRSplitProto.parseFrom(diEvent2.getUserPayload());
    InputSplit is2 = MRInput.getOldSplitDetailsFromEvent(event2Proto, new Configuration());
    assertTrue(is2 instanceof InputSplitForTest);
    assertEquals(2, ((InputSplitForTest) is2).identifier);
  }

View Full Code Here

    assertEquals(3, events.size());
    assertTrue(events.get(0) instanceof RootInputUpdatePayloadEvent);
    assertTrue(events.get(1) instanceof RootInputDataInformationEvent);
    assertTrue(events.get(2) instanceof RootInputDataInformationEvent);


    RootInputDataInformationEvent diEvent1 = (RootInputDataInformationEvent) (events.get(1));
    RootInputDataInformationEvent diEvent2 = (RootInputDataInformationEvent) (events.get(2));


    assertNull(diEvent1.getUserPayload());
    assertNull(diEvent2.getUserPayload());


    assertNotNull(diEvent1.getDeserializedUserPayload());
    assertNotNull(diEvent2.getDeserializedUserPayload());


    assertTrue(diEvent1.getDeserializedUserPayload() instanceof InputSplitForTest);
    assertEquals(1, ((InputSplitForTest) diEvent1.getDeserializedUserPayload()).identifier);


    assertTrue(diEvent2.getDeserializedUserPayload() instanceof InputSplitForTest);
    assertEquals(2, ((InputSplitForTest) diEvent2.getDeserializedUserPayload()).identifier);
  }

View Full Code Here


      RootInputConfigureVertexTasksEvent configEvent = new RootInputConfigureVertexTasksEvent(
          targetTasks, locationHints);
      events.add(configEvent);
      for (int i = 0; i < targetTasks; i++) {
        RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
            i, null);
        events.add(diEvent);
      }
      eventHandler.handle(new VertexEventRootInputInitialized(vertexID, inputs
          .get(0).getEntityName(), events));

View Full Code Here

    if (sendSerializedEvents) {
      MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto();
      int count = 0;
      for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
        // Unnecessary array copy, can be avoided by using ByteBuffer instead of a raw array.
        RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++,
            mrSplit.toByteArray());
        events.add(diEvent);
      }
    } else {
      int count = 0;
      if (inputSplitInfo.holdsNewFormatSplits()) {
        for (org.apache.hadoop.mapreduce.InputSplit split : inputSplitInfo.getNewFormatSplits()) {
          RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++, split);
          events.add(diEvent);
        }
      } else {
        for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) {
          RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++, split);
          events.add(diEvent);
        }
      }
    }

View Full Code Here


    int count = 0;
    for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
      // Unnecessary array copy, can be avoided by using ByteBuffer instead of a
      // raw array.
      RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
          count++, mrSplit.toByteArray());
      events.add(diEvent);
    }
    return events;
  }

View Full Code Here

    events.add(updatePayloadEvent);
    int count = 0;
    for (MRSplitProto mrSplit : this.splitsProto.getSplitsList()) {
      // Unnecessary array copy, can be avoided by using ByteBuffer instead of a
      // raw array.
      RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
          count++, mrSplit.toByteArray());
      events.add(diEvent);
    }


    return events;

View Full Code Here


      RootInputConfigureVertexTasksEvent configEvent = new RootInputConfigureVertexTasksEvent(
          targetTasks, locationHints);
      events.add(configEvent);
      for (int i = 0; i < targetTasks; i++) {
        RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
            i, null);
        events.add(diEvent);
      }
      eventHandler.handle(new VertexEventRootInputInitialized(vertexID, inputs
          .get(0).getEntityName(), events));

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.tez.common.ProtoConverters

org.apache.tez.dag.app.dag.impl.RootInputVertexManager

org.apache.tez.dag.app.dag.impl.TestVertexImpl$RootInputInitializerRunnerControlled

org.apache.tez.dag.app.dag.impl.VertexImpl$RouteEventTransition

org.apache.tez.mapreduce.common.MRInputAMSplitGenerator

org.apache.tez.mapreduce.common.MRInputSplitDistributor

org.apache.tez.mapreduce.common.TestMRInputSplitDistributor

org.apache.tez.runtime.api.impl.TezEvent

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.