Examples of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

org.apache.tez.runtime.api.events.CompositeDataMovementEvent
A convenience class to specify multiple DataMovementEvents which share the same payload. A contiguous range of srcIndices can be specified. This event will NOT be seen by downstream Inputs - instead they will see {@link DataMovementEvent}s which are generated based on the range specified in this event. This event should be used by an output which has the same payload for all of the Physical Outputs that it generates.

        outputLargeRecordsCounter.getValue());


    // Validate the event
    assertEquals(1, events.size());
    assertTrue(events.get(0) instanceof CompositeDataMovementEvent);
    CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) events.get(0);
    assertEquals(0, cdme.getSourceIndexStart());
    assertEquals(numPartitions, cdme.getCount());
    DataMovementEventPayloadProto eventProto = DataMovementEventPayloadProto.parseFrom(
        ByteString.copyFrom(cdme
            .getUserPayload()));
    assertFalse(eventProto.hasData());
    BitSet emptyPartitionBits = null;
    if (partitionsWithData.cardinality() != numPartitions) {
      assertTrue(eventProto.hasEmptyPartitions());

View Full Code Here


    BitSet emptyPartitionBits = null;
    // Verify the event returned
    assertEquals(1, events.size());
    assertTrue(events.get(0) instanceof CompositeDataMovementEvent);
    CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) events.get(0);
    assertEquals(0, cdme.getSourceIndexStart());
    assertEquals(numOutputs, cdme.getCount());
    DataMovementEventPayloadProto eventProto =
        DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(
            cdme.getUserPayload()));
    assertFalse(eventProto.hasData());
    if (skippedPartitions == null && numRecordsWritten > 0) {
      assertFalse(eventProto.hasEmptyPartitions());
      emptyPartitionBits = new BitSet(numPartitions);
    } else {

View Full Code Here

        getContext().getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());


    List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs() + 1);
    events.add(vmEvent);


    CompositeDataMovementEvent csdme =
        CompositeDataMovementEvent.create(0, getNumPhysicalOutputs(), payload);
    events.add(csdme);


    return events;
  }

View Full Code Here


  @Test
  public void testGetCount(){
    int numPartitions = 2;
    int startIndex = 2;
    CompositeDataMovementEvent cdme1 =
        CompositeDataMovementEvent.create(startIndex, numPartitions, userPayload);
    Assert.assertEquals(numPartitions, cdme1.getCount());
    Assert.assertEquals(startIndex, cdme1.getSourceIndexStart());
  }

View Full Code Here


  @Test
  public void testGetEvents(){
    int numOutputs = 0;
    int startIndex = 1;
    CompositeDataMovementEvent cdme2 =
        CompositeDataMovementEvent.create(startIndex, numOutputs, userPayload);
    for(DataMovementEvent dme: cdme2.getEvents()){
      numOutputs++;
    }
    Assert.assertEquals(numOutputs, cdme2.getCount());
  }

View Full Code Here

      payloadBuidler.setHost(host);
      payloadBuidler.setPort(shufflePort);
      payloadBuidler.setPathComponent(outputContext.getUniqueIdentifier());
    }


    CompositeDataMovementEvent cDme = CompositeDataMovementEvent.create(0, numPartitions,
        payloadBuidler.build().toByteString().asReadOnlyByteBuffer());
    return cDme;
  }

View Full Code Here

        outputContext.getDestinationVertexName(), vmBuilder.build().toByteArray());    


    List<Event> events = Lists.newArrayListWithCapacity(numOutputs+1);
    events.add(vmEvent);


    CompositeDataMovementEvent csdme = new CompositeDataMovementEvent(0, numOutputs, payloadBytes);
    events.add(csdme);


    return events;
  }

View Full Code Here

    }
  }
  


  private void handleCompositeDataMovementEvent(TezEvent tezEvent) {
    CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
    EventMetaData srcInfo = tezEvent.getSourceInfo();
    
    for (DataMovementEvent dmEvent : compEvent.getEvents()) {
      TezEvent newEvent = new TezEvent(dmEvent, srcInfo);
      sendTezEventToDestinationTasks(newEvent);
    }
  }

View Full Code Here

    }
  }
  


  private void handleCompositeDataMovementEvent(TezEvent tezEvent) {
    CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
    EventMetaData srcInfo = tezEvent.getSourceInfo();
    
    for (DataMovementEvent dmEvent : compEvent.getEvents()) {
      TezEvent newEvent = new TezEvent(dmEvent, srcInfo);
      sendTezEventToDestinationTasks(newEvent);
    }
  }

View Full Code Here

    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0
    
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(),
        ByteBuffer.wrap("bytes".getBytes()));
    cdmEvent.setVersion(2); // AttemptNum
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.


    edge.sendTezEventToDestinationTasks(tezEvent);
    verifyEvents(srcTAID, destTasks);

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

org.apache.tez.common.ProtoConverters

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestEdge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.runtime.api.event.TestCompositeDataMovementEvent

org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter

org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter

org.apache.tez.runtime.library.output.OnFileSortedOutput

org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.