Examples of DataMovementEvent

org.apache.tez.runtime.api.events.DataMovementEvent
Event used by user code to send information between tasks. An output can generate an Event of this type to sending information regarding output data ( such as URI for file-based output data, port info in case of streaming-based data transfers ) to the Input on the destination vertex.

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

      Task mockTask = taskEntry.getValue();
      ArgumentCaptor<TezEvent> args = ArgumentCaptor.forClass(TezEvent.class);
      verify(mockTask, times(1)).registerTezEvent(args.capture());
      TezEvent capturedEvent = args.getValue();


      DataMovementEvent dmEvent = (DataMovementEvent) capturedEvent.getEvent();
      assertEquals(srcTAID.getId(), dmEvent.getVersion());
      assertEquals(count++, dmEvent.getSourceIndex());
      assertEquals(srcTAID.getTaskID().getId(), dmEvent.getTargetIndex());
      byte[] res = new byte[dmEvent.getUserPayload().limit() - dmEvent.getUserPayload().position()];
      dmEvent.getUserPayload().slice().get(res);
      assertTrue(Arrays.equals("bytes".getBytes(), res));
    }
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }


    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);


    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());


    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());


    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

  public List<Event> close() throws Exception {
    LOG.info("Sending data movement event with value: " + output);
    byte[] result = ByteBuffer.allocate(4).putInt(output).array();
    List<Event> events = Lists.newArrayListWithCapacity(numOutputs);
    for (int i = 0; i < numOutputs; i++) {
      DataMovementEvent event = new DataMovementEvent(i, result);
      events.add(event);
    }
    return events;
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

    return builder.build();
  }


  public static DataMovementEvent convertDataMovementEventFromProto(
      EventProtos.DataMovementEventProto proto) {
    return new DataMovementEvent(proto.getSourceIndex(),
        proto.getTargetIndex(),
        proto.getVersion(),
        proto.getUserPayload() != null ?
            proto.getUserPayload().toByteArray() : null);
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

    builder.setPort(PORT);
    builder.setPathComponent("attempttmp");
    if (emptyPartitionByteString != null) {
      builder.setEmptyPartitions(emptyPartitionByteString);
    }
    Event dme = new DataMovementEvent(srcIndex, targetIndex, 0, builder.build().toByteArray());
    return dme;
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent


  @Override
  public void handleEvents(List<Event> inputEvents) throws Exception {
    for (Event event : inputEvents) {
      if (event instanceof DataMovementEvent) {
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        numCompletedInputs++;
        LOG.info("Received DataMovement event sourceId : " + dmEvent.getSourceIndex() + 
            " targetId: " + dmEvent.getTargetIndex() +
            " version: " + dmEvent.getVersion() +
            " numInputs: " + numInputs +
            " numCompletedInputs: " + numCompletedInputs);
        this.completedInputVersion[dmEvent.getTargetIndex()] = dmEvent.getVersion();
        this.inputValues[dmEvent.getTargetIndex()] = 
            ByteBuffer.wrap(dmEvent.getUserPayload()).getInt();
      } else if (event instanceof InputFailedEvent) {
        InputFailedEvent ifEvent = (InputFailedEvent) event;
        numCompletedInputs--;
        LOG.info("Received InputFailed event sourceId : " + ifEvent.getSourceIndex() + 
            " targetId: " + ifEvent.getTargetIndex() +

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

      Assert.fail("Invalid creation should have errored out");
    } catch (RuntimeException e) {
      // Expected
    }
    List<TezEvent> events =
        Arrays.asList(new TezEvent(new DataMovementEvent(1, null), new EventMetaData(
            EventProducerConsumerType.SYSTEM, "foo", "bar", null)));
    event = new VertexDataMovementEventsGeneratedEvent(
            TezVertexID.getInstance(
                TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 1), events);
    VertexDataMovementEventsGeneratedEvent deserializedEvent =

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

    
    
    // Same Verification via regular DataMovementEvents
    reset(eventHandler);
    for (int i = 0 ; i < destTasks.size() ; i++) {
      DataMovementEvent dmEvent = new DataMovementEvent(i, "bytes".getBytes());
      dmEvent.setVersion(2);
      tezEvent = new TezEvent(dmEvent, srcMeta);
      edge.sendTezEventToDestinationTasks(tezEvent);
    }
    args = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(destTasks.size())).handle(args.capture());

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

      assertTrue(event instanceof TaskEventAddTezEvent);
      TaskEventAddTezEvent taEvent = (TaskEventAddTezEvent) event;
      assertEquals(expEntry.getKey(), taEvent.getTaskID());
      TezEvent tezEvent = taEvent.getTezEvent();


      DataMovementEvent dmEvent = (DataMovementEvent)tezEvent.getEvent();
      assertEquals(srcTAID.getId(), dmEvent.getVersion());
      assertEquals(count, dmEvent.getSourceIndex());
      assertEquals(srcTAID.getTaskID().getId(), dmEvent.getTargetIndex());
      assertTrue(Arrays.equals("bytes".getBytes(), dmEvent.getUserPayload()));


      count++;
    }
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.events.DataMovementEvent

      payloadBuilder.setPort(shufflePort);
      payloadBuilder.setPathComponent(outputContext.getUniqueIdentifier());
    }
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();


    DataMovementEvent dmEvent = new DataMovementEvent(0,
        payloadProto.toByteArray());
    List<Event> events = Lists.newArrayListWithCapacity(1);
    events.add(dmEvent);
    return events;
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.