Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

310
311
312
313
314
315
316
317
318
319
320
        .newHashMap();
        TezTaskAttemptID srcAttemptId = tezEvent.getSourceInfo()
            .getTaskAttemptID();
        int srcTaskIndex = srcAttemptId.getTaskID().getId();
        if (isDataMovementEvent) {
          DataMovementEvent dmEvent = (DataMovementEvent)tezEvent.getEvent();
          edgeManager.routeDataMovementEventToDestination(dmEvent,
                srcTaskIndex, destinationVertex.getTotalTasks(),
                inputIndicesToTaskIndices);
        } else {
          edgeManager.routeInputSourceTaskFailedEventToDestination(srcTaskIndex,
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
  }

  private void sendTezEventsToTask(TezTaskID taskId, int numTezEvents) {
    TaskEventAddTezEvent event = null;
    EventMetaData eventMetaData = new EventMetaData();
    DataMovementEvent dmEvent = new DataMovementEvent(null);
    TezEvent tezEvent = new TezEvent(dmEvent, eventMetaData);
    for (int i = 0; i < numTezEvents; i++) {
      event = new TaskEventAddTezEvent(taskId, tezEvent);
      mockTask.handle(event);
    }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
    List<Event> events = Lists.newArrayListWithCapacity(numOutputs+1);
    events.add(vmEvent);

    for (int i = 0; i < numOutputs; i++) {
      DataMovementEvent event = new DataMovementEvent(i, payloadBytes);
      events.add(event);
    }
    return events;
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
      payloadBuilder.setPort(shufflePort);
      payloadBuilder.setPathComponent(outputContext.getUniqueIdentifier());
    }
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();

    DataMovementEvent dmEvent = new DataMovementEvent(0,
        payloadProto.toByteArray());
    List<Event> events = Lists.newArrayListWithCapacity(1);
    events.add(dmEvent);
    return events;
  }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());

    assertTrue(shufflePayload.getOutputGenerated());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
      sEvt.write(out);
    } else {
      byte[] eventBytes = null;
      switch (eventType) {
      case DATA_MOVEMENT_EVENT:
        DataMovementEvent dmEvt = (DataMovementEvent) event;
        eventBytes = DataMovementEventProto.newBuilder()
          .setSourceIndex(dmEvt.getSourceIndex())
          .setTargetIndex(dmEvt.getTargetIndex())
          .setUserPayload(ByteString.copyFrom(dmEvt.getUserPayload()))
          .build().toByteArray();
        break;
      case VERTEX_MANAGER_EVENT:
        VertexManagerEvent vmEvt = (VertexManagerEvent) event;
        eventBytes = VertexManagerEventProto.newBuilder()
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
      in.readFully(eventBytes);
      switch (eventType) {
      case DATA_MOVEMENT_EVENT:
        DataMovementEventProto dmProto =
            DataMovementEventProto.parseFrom(eventBytes);
        event = new DataMovementEvent(dmProto.getSourceIndex(),
            dmProto.getTargetIndex(),
            dmProto.getUserPayload().toByteArray());
        break;
      case VERTEX_MANAGER_EVENT:
        VertexManagerEventProto vmProto =
View Full Code Here
310
311
312
313
314
315
316
317
318
319
320
321
322
            tezEventToSend = tezEvent;
          } else {
            // create new event object for this input index
            Event e;
            if (isDataMovementEvent) {
              DataMovementEvent dmEvent = (DataMovementEvent) event;
              e = DataMovementEvent.create(dmEvent.getSourceIndex(),
                  inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload());
            } else {
              InputFailedEvent ifEvent = ((InputFailedEvent) event);
              e = InputFailedEvent.create(inputIndex, ifEvent.getVersion());
            }
            tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo());
View Full Code Here
370
371
372
373
374
375
376
377
378
379
380
381
382
                  generateCommonDebugString(srcTaskIndex, tezEvent));
        }

        if (routingRequired) {
          if (isDataMovementEvent) {
            DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
            edgeManager.routeDataMovementEventToDestination(dmEvent,
                srcTaskIndex, dmEvent.getSourceIndex(),
                destTaskAndInputIndices);
          } else {
            edgeManager.routeInputSourceTaskFailedEventToDestination(srcTaskIndex,
                destTaskAndInputIndices);
          }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
    // Same Verification via regular DataMovementEvents
    // Reset the mock
    resetTaskMocks(destTasks.values());

    for (int i = 0 ; i < destTasks.size() ; i++) {
      DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
      dmEvent.setVersion(2);
      tezEvent = new TezEvent(dmEvent, srcMeta);
      edge.sendTezEventToDestinationTasks(tezEvent);
    }
    verifyEvents(srcTAID, destTasks);
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.events.DataMovementEvent

  • org.apache.tez.common.ProtoConverters
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestEdge
  • org.apache.tez.dag.app.dag.impl.TestTaskImpl
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
  • org.apache.tez.runtime.api.impl.TezEvent
  • org.apache.tez.runtime.library.output.OnFileSortedOutput
  • org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.