Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.VertexManagerEvent

220
221
222
223
224
225
226
227
228
229
230
                    parallelism = (Integer)sampleMap.get(ESTIMATED_NUM_PARALLELISM);
                }
                String sortingVertex = conf.get(SORT_VERTEX);
                // Should contain only 1 output for sampleAggregation job
                LOG.info("Sending numParallelism " + parallelism + " to " + sortingVertex);
                VertexManagerEvent vmEvent = VertexManagerEvent.create(
                        sortingVertex, ByteBuffer.wrap(Ints.toByteArray(parallelism)));
                List<Event> events = Lists.newArrayListWithCapacity(1);
                events.add(vmEvent);
                getContext().sendEvents(events);
            }
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);

    ByteBuffer payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(5000L).build().toByteString().asReadOnlyByteBuffer();
    VertexManagerEvent vmEvent = VertexManagerEvent.create("Vertex", payload);
    // parallelism not change due to large data size
    manager = createManager(conf, mockContext, 0.1f, 0.1f);
    manager.onVertexStarted(null);
    Assert.assertTrue(manager.pendingTasks.size() == 4); // no tasks scheduled
    Assert.assertTrue(manager.totalNumSourceTasks == 4);
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
    long outputSize = getContext().getCounters()
        .findCounter(TaskCounter.OUTPUT_BYTES).getValue();
    VertexManagerEventPayloadProto.Builder vmBuilder = VertexManagerEventPayloadProto
        .newBuilder();
    vmBuilder.setOutputSize(outputSize);
    VertexManagerEvent vmEvent = VertexManagerEvent.create(
        getContext().getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());

    List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs() + 1);
    events.add(vmEvent);
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
    long outputSize = outputContext.getCounters()
        .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
    VertexManagerEventPayloadProto.Builder vmBuilder = VertexManagerEventPayloadProto
        .newBuilder();
    vmBuilder.setOutputSize(outputSize);
    VertexManagerEvent vmEvent = new VertexManagerEvent(
        outputContext.getDestinationVertexName(), vmBuilder.build().toByteArray());   

    List<Event> events = Lists.newArrayListWithCapacity(numOutputs+1);
    events.add(vmEvent);
View Full Code Here
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
              riEvent.getTargetIndex());
          vertex.eventHandler.handle(new TaskEventAddTezEvent(targetTaskID, tezEvent));         
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(vmEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.vertexManager.onVertexManagerEventReceived(vmEvent);
          } else {
            vertex.eventHandler.handle(new VertexEventRouteEvent(target
                .getVertexId(), Collections.singletonList(tezEvent)));
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);

    byte[] payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(5000L).build().toByteArray();
    VertexManagerEvent vmEvent = new VertexManagerEvent("Vertex", payload);
    // parallelism not change due to large data size
    manager = createManager(conf, mockContext, 0.1f, 0.1f);
    manager.onVertexStarted(null);
    Assert.assertTrue(manager.pendingTasks.size() == 4); // no tasks scheduled
    Assert.assertTrue(manager.numSourceTasks == 4);
    manager.onVertexManagerEventReceived(vmEvent);
    manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
    // managedVertex tasks reduced
    verify(mockContext, times(0)).setVertexParallelism(anyInt(), any(VertexLocationHint.class), anyMap());
    Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled
    Assert.assertEquals(4, scheduledTasks.size());
    Assert.assertEquals(1, manager.numSourceTasksCompleted);
    Assert.assertEquals(5000L, manager.completedSourceTasksOutputSize);
   
   
    // parallelism changed due to small data size
    scheduledTasks.clear();
    payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(500L).build().toByteArray();
    vmEvent = new VertexManagerEvent("Vertex", payload);
   
    manager = createManager(conf, mockContext, 0.5f, 0.5f);
    manager.onVertexStarted(null);
    Assert.assertEquals(4, manager.pendingTasks.size()); // no tasks scheduled
    Assert.assertEquals(4, manager.numSourceTasks);
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
    long outputSize = outputContext.getCounters()
        .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
    VertexManagerEventPayloadProto.Builder vmBuilder = VertexManagerEventPayloadProto
        .newBuilder();
    vmBuilder.setOutputSize(outputSize);
    VertexManagerEvent vmEvent = new VertexManagerEvent(
        outputContext.getDestinationVertexName(), vmBuilder.build().toByteArray());   

    List<Event> events = Lists.newArrayListWithCapacity(numOutputs+1);
    events.add(vmEvent);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    TezTaskAttemptID mockSrcAttemptId31 =
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId3, 0), 0);

    byte[] payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(5000L).build().toByteArray();
    VertexManagerEvent vmEvent = new VertexManagerEvent("Vertex", payload);
    // parallelism not change due to large data size
    scheduler = createScheduler(conf, mockManagedVertex, 0.1f, 0.1f);
    scheduler.onVertexStarted(null);
    Assert.assertTrue(scheduler.pendingTasks.size() == 4); // no tasks scheduled
    Assert.assertTrue(scheduler.numSourceTasks == 4);
    scheduler.onVertexManagerEventReceived(vmEvent);
    scheduler.onSourceTaskCompleted(mockSrcAttemptId11);
    // managedVertex tasks reduced
    verify(mockManagedVertex, times(0)).setParallelism(anyInt(), anyMap());
    Assert.assertEquals(0, scheduler.pendingTasks.size()); // all tasks scheduled
    Assert.assertEquals(4, scheduledTasks.size());
    Assert.assertEquals(1, scheduler.numSourceTasksCompleted); // TODO
    Assert.assertEquals(5000L, scheduler.completedSourceTasksOutputSize);
   
   
    // parallelism changed due to small data size
    scheduledTasks.clear();
    payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(500L).build().toByteArray();
    vmEvent = new VertexManagerEvent("Vertex", payload);
   
    scheduler = createScheduler(conf, mockManagedVertex, 0.5f, 0.5f);
    scheduler.onVertexStarted(null);
    Assert.assertEquals(4, scheduler.pendingTasks.size()); // no tasks scheduled
    Assert.assertEquals(4, scheduler.numSourceTasks);
View Full Code Here
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
            }
          }
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(vmEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.vertexScheduler.onVertexManagerEventReceived(vmEvent);
          } else {
            vertex.eventHandler.handle(new VertexEventRouteEvent(target
                .getVertexId(), Collections.singletonList(tezEvent)));
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
138
139
          .setTargetIndex(dmEvt.getTargetIndex())
          .setUserPayload(ByteString.copyFrom(dmEvt.getUserPayload()))
          .build().toByteArray();
        break;
      case VERTEX_MANAGER_EVENT:
        VertexManagerEvent vmEvt = (VertexManagerEvent) event;
        eventBytes = VertexManagerEventProto.newBuilder()
          .setTargetVertexName(vmEvt.getTargetVertexName())
          .setUserPayload(ByteString.copyFrom(vmEvt.getUserPayload()))
          .build().toByteArray();
        break;
      case INPUT_READ_ERROR_EVENT:
        InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
        eventBytes = InputReadErrorEventProto.newBuilder()
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.events.VertexManagerEvent

  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
  • org.apache.tez.dag.app.dag.impl.VertexImpl$RouteEventTransition
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
  • org.apache.tez.runtime.api.impl.TezEvent
  • org.apache.tez.runtime.library.output.OnFileSortedOutput
  • org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.