Package edu.berkeley.sparrow.thrift

Examples of edu.berkeley.sparrow.thrift.THostPort

50
51
52
53
54
55
56
57
58
  private TEnqueueTaskReservationsRequest createTaskReservationRequest(
      int numTasks, TaskScheduler scheduler, String userId, int priority) {
    String idStr = Integer.toString(requestId++);
    TUserGroupInfo user = new TUserGroupInfo(userId, "group", priority);
    THostPort schedulerAddress = new THostPort("1.2.3.4", 52);
    return new TEnqueueTaskReservationsRequest(
        "appId", user, idStr, schedulerAddress, numTasks);
  }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
      // If the scheduler does not enforce a maximum number of tasks, just use a number of
      // threads equal to the number of cores.
      numThreads = Resources.getSystemCPUCount(conf);
    }
    this.scheduler = scheduler;
    nodeMonitorInternalAddress = new THostPort(Network.getIPAddress(conf), nodeMonitorPort);
    ExecutorService service = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
      service.submit(new TaskLaunchRunnable());
    }
  }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
          LOG.error("Error creating thrift client: " + e.getMessage());
          return;
        }
      }
      BackendService.Client backendClient = backendClients.get(task.appBackendAddress);
      THostPort schedulerHostPort = Network.socketAddressToThrift(task.schedulerAddress);
      TFullTaskId taskId = new TFullTaskId(task.taskSpec.getTaskId(), task.requestId,
          task.appId, schedulerHostPort);
      try {
        backendClient.launchTask(task.taskSpec.bufferForMessage(), taskId, task.user);
      } catch (TException e) {
View Full Code Here
26
27
28
29
30
31
32
import edu.berkeley.sparrow.thrift.THostPort;

public class Network {
 
  public static THostPort socketAddressToThrift(InetSocketAddress address) {
    return new THostPort(address.getAddress().getHostAddress(), address.getPort());
  }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
      InetSocketAddress node = nodeList.get(i);
      debugString.append(node.getAddress().getHostAddress() + ":" + node.getPort());
      debugString.append(";");
      // TODO: this needs to be a count!
      outstandingReservations.put(
          new THostPort(node.getAddress().getHostAddress(), node.getPort()),
          numReservations);
      TEnqueueTaskReservationsRequest request = new TEnqueueTaskReservationsRequest(
          schedulingRequest.getApp(), schedulingRequest.getUser(), requestId,
          schedulerAddress, numReservations);
      requests.put(node, request);
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
      TTaskLaunchSpec taskLaunchSpec = new TTaskLaunchSpec(task.getTaskId(),
                                                           task.bufferForMessage());

      int numEnqueuedNodes = 0;
      for (InetSocketAddress addr : preferredNodes) {
        THostPort hostPort = new THostPort(addr.getAddress().getHostAddress(), addr.getPort());
        if (!unlaunchedConstrainedTasks.containsKey(hostPort)) {
          unlaunchedConstrainedTasks.put(
              hostPort, new LinkedList<TTaskLaunchSpec>());
        }
View Full Code Here
194
195
196
197
198
199
200
201
202
203
  }

  private void populateOutstandingReservations(
      Map<InetSocketAddress, TEnqueueTaskReservationsRequest> requests) {
    for (Entry<InetSocketAddress, TEnqueueTaskReservationsRequest> entry: requests.entrySet()) {
      THostPort hostPort = new THostPort(
          entry.getKey().getAddress().getHostAddress(), entry.getKey().getPort());
      outstandingReservations.put(hostPort, entry.getValue().numTasks);
    }
  }
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
      List<InetSocketAddress> nodes = Lists.newArrayList(requests.keySet());
      assertEquals(nodes.size(), expectedReservations);
      Collections.shuffle(nodes);
      Set<String> taskIdsCopy = Sets.newHashSet(taskIds);
      for (int j = 0; j < expectedReservations; ++j) {
        THostPort hostPort = new THostPort(nodes.get(j).getHostName(), nodes.get(j).getPort());
        List<TTaskLaunchSpec> specs = taskPlacer.assignTask(hostPort);
        assertTrue(specs != null);
        if (j < NUM_TASKS) {
          assertEquals(specs.size(), 1);
          TTaskLaunchSpec spec = specs.get(0);
          assertTrue("Expect to receive a task spec for task " + j + " at " +
                     hostPort.getHost() + ":" + hostPort.getPort(), spec != null);
          assertTrue("Expect list of unlaunched tasks to contain " + spec.getTaskId(),
                     taskIdsCopy.contains(spec.getTaskId()));
          taskIdsCopy.remove(spec.getTaskId());
        } else {
          assertEquals(specs.size(), 0);
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
      Set<String> taskIdsCopy = Sets.newHashSet(taskIds);
      for (int j = 0; j < expectedReservations; ++j) {
        if (j < NUM_TASKS) {
          assertTrue(!taskPlacer.allTasksPlaced());
        }
        THostPort hostPort = new THostPort(nodes.get(j).getHostName(), nodes.get(j).getPort());
        List<TTaskLaunchSpec> specs = taskPlacer.assignTask(hostPort);
        assertTrue(specs != null);
        if (j < NUM_TASKS) {
          assertEquals(1, specs.size());
          TTaskLaunchSpec spec = specs.get(0);
          assertTrue("Expect to receive a task spec for task " + j + " at " +
                     hostPort.getHost() + ":" + hostPort.getPort(), spec != null);
          assertTrue("Expect list of unlaunched tasks to contain " + spec.getTaskId(),
                     taskIdsCopy.contains(spec.getTaskId()));
          taskIdsCopy.remove(spec.getTaskId());
        } else {
          assertTrue(taskPlacer.allTasksPlaced());
View Full Code Here
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
          taskPlacer.getEnqueueTaskReservationsRequests(schedulingRequest, REQUEST_ID, backendNodes,
              SCHEDULER_ADDRESS);

      List<InetSocketAddress> nodes = Lists.newArrayList(requests.keySet());
      for (int taskIndex = 0; taskIndex < NUM_TASKS; ++taskIndex) {
        THostPort hostPort = new THostPort(nodes.get(0).getAddress().getHostAddress(),
                                           nodes.get(0).getPort());
        nodes.remove(0);
        taskPlacer.assignTask(hostPort);
      }

      // Get the node monitors that should be cancelled.  At this point, the remaining entries in
      // nodes should be the set of nodes where requests should be cancelled.
      Set<THostPort> remainingNodeMonitors = taskPlacer.getOutstandingNodeMonitorsForCancellation();
      assertEquals(nodes.size(), remainingNodeMonitors.size());
      for (InetSocketAddress node : nodes) {
        THostPort hostPort = new THostPort(node.getAddress().getHostAddress(), node.getPort());
        assertTrue(remainingNodeMonitors.contains(hostPort));
      }

      // Trying to get the outstanding NMs for cancellation again should return an empty set.
      assertEquals(0, taskPlacer.getOutstandingNodeMonitorsForCancellation().size());
View Full Code Here
TOP

Related Classes of edu.berkeley.sparrow.thrift.THostPort

  • edu.berkeley.sparrow.daemon.nodemonitor.TaskLauncherService
  • edu.berkeley.sparrow.daemon.nodemonitor.TaskLauncherService$TaskLaunchRunnable
  • edu.berkeley.sparrow.daemon.nodemonitor.TestTaskScheduler
  • edu.berkeley.sparrow.daemon.scheduler.ConstrainedTaskPlacer
  • edu.berkeley.sparrow.daemon.scheduler.TestConstrainedTaskPlacer
  • edu.berkeley.sparrow.daemon.scheduler.TestUnconstrainedTaskPlacer
  • edu.berkeley.sparrow.daemon.scheduler.UnconstrainedTaskPlacer
  • edu.berkeley.sparrow.daemon.util.Network
  • org.apache.thrift.protocol.TTupleProtocol
  • java.util.BitSet
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.