Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

163
164
165
166
167
168
169
170
171
172
173
    assertTaskScheduledState();
  }

  private void sendTezEventsToTask(TezTaskID taskId, int numTezEvents) {
    EventMetaData eventMetaData = new EventMetaData();
    DataMovementEvent dmEvent = DataMovementEvent.create(null);
    TezEvent tezEvent = new TezEvent(dmEvent, eventMetaData);
    for (int i = 0; i < numTezEvents; i++) {
      mockTask.registerTezEvent(tezEvent);
    }
  }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
    verify(mockContext).setVertexParallelism(eq(2), any(VertexLocationHint.class), anyMap(), anyMap());
    Assert.assertEquals(2, newEdgeManagers.size());
   
    EdgeManagerPlugin edgeManager = newEdgeManagers.values().iterator().next();
    Map<Integer, List<Integer>> targets = Maps.newHashMap();
    DataMovementEvent dmEvent = DataMovementEvent.create(1, ByteBuffer.wrap(new byte[0]));
    // 4 source task outputs - same as original number of partitions
    Assert.assertEquals(4, edgeManager.getNumSourceTaskPhysicalOutputs(0));
    // 4 destination task inputs - 2 source tasks + 2 merged partitions
    Assert.assertEquals(4, edgeManager.getNumDestinationTaskPhysicalInputs(0));
    edgeManager.routeDataMovementEventToDestination(dmEvent, 1, dmEvent.getSourceIndex(), targets);
    Assert.assertEquals(1, targets.size());
    Map.Entry<Integer, List<Integer>> e = targets.entrySet().iterator().next();
    Assert.assertEquals(0, e.getKey().intValue());
    Assert.assertEquals(1, e.getValue().size());
    Assert.assertEquals(3, e.getValue().get(0).intValue());
    targets.clear();
    dmEvent = DataMovementEvent.create(2, ByteBuffer.wrap(new byte[0]));
    edgeManager.routeDataMovementEventToDestination(dmEvent, 0, dmEvent.getSourceIndex(), targets);
    Assert.assertEquals(1, targets.size());
    e = targets.entrySet().iterator().next();
    Assert.assertEquals(1, e.getKey().intValue());
    Assert.assertEquals(1, e.getValue().size());
    Assert.assertEquals(0, e.getValue().get(0).intValue());
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
  @Override
  public void handleEvents(List<Event> inputEvents) throws Exception {
    for (Event event : inputEvents) {
      if (event instanceof DataMovementEvent) {
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        numCompletedInputs++;
        LOG.info("Received DataMovement event sourceId : " + dmEvent.getSourceIndex() +
            " targetId: " + dmEvent.getTargetIndex() +
            " version: " + dmEvent.getVersion() +
            " numInputs: " + getNumPhysicalInputs() +
            " numCompletedInputs: " + numCompletedInputs);
        this.completedInputVersion[dmEvent.getTargetIndex()] = dmEvent.getVersion();
        this.inputValues[dmEvent.getTargetIndex()] =
            dmEvent.getUserPayload().getInt();
      } else if (event instanceof InputFailedEvent) {
        InputFailedEvent ifEvent = (InputFailedEvent) event;
        numCompletedInputs--;
        LOG.info("Received InputFailed event targetId: " + ifEvent.getTargetIndex() +
            " version: " + ifEvent.getVersion() +
View Full Code Here
152
153
154
155
156
157
158
159
160
161
      payloadBuilder.setPort(shufflePort);
      payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());
    }
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();

    DataMovementEvent dmEvent = DataMovementEvent.create(0, payloadProto.toByteString().asReadOnlyByteBuffer());
    List<Event> events = Lists.newArrayListWithCapacity(1);
    events.add(dmEvent);
    return events;
  }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
    LOG.info("Sending data movement event with value: " + output);
    ByteBuffer result = ByteBuffer.allocate(4).putInt(output);
    result.flip();
    List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs());
    for (int i = 0; i < getNumPhysicalOutputs(); i++) {
      DataMovementEvent event = DataMovementEvent.create(i, result);
      events.add(event);
    }
    return events;
  }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));

    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
      payloadBuilder.setPort(shufflePort);
      payloadBuilder.setPathComponent(outputContext.getUniqueIdentifier());
    }
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();

    DataMovementEvent dmEvent = new DataMovementEvent(0,
        payloadProto.toByteArray());
    List<Event> events = Lists.newArrayListWithCapacity(1);
    events.add(dmEvent);
    return events;
  }
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    verify(mockContext).setVertexParallelism(eq(2), any(VertexLocationHint.class), anyMap());
    Assert.assertEquals(2, newEdgeManagers.size());
   
    EdgeManager edgeManager = newEdgeManagers.values().iterator().next();
    Map<Integer, List<Integer>> targets = Maps.newHashMap();
    DataMovementEvent dmEvent = new DataMovementEvent(1, new byte[0]);
    edgeManager.routeDataMovementEventToDestination(dmEvent, 1, 2, targets);
    Assert.assertEquals(1, targets.size());
    Map.Entry<Integer, List<Integer>> e = targets.entrySet().iterator().next();
    Assert.assertEquals(3, e.getKey().intValue());
    Assert.assertEquals(1, e.getValue().size());
    Assert.assertEquals(0, e.getValue().get(0).intValue());
    targets.clear();
    dmEvent = new DataMovementEvent(2, new byte[0]);
    edgeManager.routeDataMovementEventToDestination(dmEvent, 0, 2, targets);
    Assert.assertEquals(1, targets.size());
    e = targets.entrySet().iterator().next();
    Assert.assertEquals(0, e.getKey().intValue());
    Assert.assertEquals(1, e.getValue().size());
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());

    assertTrue(shufflePayload.getOutputGenerated());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
278
279
        }
        tezEventToSend = tezEvent;
      } else {
        Event e;
        if (isDataMovementEvent) {
          DataMovementEvent dmEvent = (DataMovementEvent) event;
          e = new DataMovementEvent(dmEvent.getSourceIndex(),
              entry.getKey().intValue(), dmEvent.getVersion(), dmEvent.getUserPayload());
        } else {
          InputFailedEvent ifEvent = ((InputFailedEvent) event);
          e = new InputFailedEvent(entry.getKey().intValue(), ifEvent.getVersion());
        }
        tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo());
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.events.DataMovementEvent

  • org.apache.tez.common.ProtoConverters
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestEdge
  • org.apache.tez.dag.app.dag.impl.TestTaskImpl
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
  • org.apache.tez.runtime.api.impl.TezEvent
  • org.apache.tez.runtime.library.output.OnFileSortedOutput
  • org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.