Examples of io.druid.indexing.worker.Worker

io.druid.indexing.worker.Worker
A container for worker metadata.


          @Provides
          @LazySingleton
          public Worker getWorker(@Self DruidNode node, WorkerConfig config)
          {
            return new Worker(
                node.getHost(),
                config.getIp(),
                config.getCapacity(),
                config.getVersion()
            );

View Full Code Here

      ForkingTaskRunner taskRunner


  ) throws Exception
  {
    this.enabledWorker = worker;
    this.disabledWorker = new Worker(worker.getHost(), worker.getIp(), worker.getCapacity(), DISABLED_VERSION);
    this.curatorCoordinator = curatorCoordinator;
    this.taskRunner = taskRunner;
  }

View Full Code Here

  @Path("/enabled")
  @Produces("application/json")
  public Response isEnabled()
  {
    try {
      final Worker theWorker = curatorCoordinator.getWorker();
      final boolean enabled = !theWorker.getVersion().equalsIgnoreCase(DISABLED_VERSION);
      return Response.ok(ImmutableMap.of(theWorker.getHost(), enabled)).build();
    }
    catch (Exception e) {
      return Response.serverError().build();
    }
  }

View Full Code Here

    return worker.get().getVersion().compareTo(minVersion) >= 0;
  }


  public void setWorker(Worker newWorker)
  {
    final Worker oldWorker = worker.get();
    Preconditions.checkArgument(newWorker.getHost().equals(oldWorker.getHost()), "Cannot change Worker host");
    Preconditions.checkArgument(newWorker.getIp().equals(oldWorker.getIp()), "Cannot change Worker ip");


    worker.set(newWorker);
  }

View Full Code Here

          new PathChildrenCacheListener()
          {
            @Override
            public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception
            {
              final Worker worker;
              switch (event.getType()) {
                case CHILD_ADDED:
                  worker = jsonMapper.readValue(
                      event.getData().getData(),
                      Worker.class

View Full Code Here

  {
    if (!started) {
      return;
    }
    final RemoteTaskRunnerWorkItem removed = completeTasks.remove(taskId);
    final Worker worker = removed.getWorker();
    if (removed == null || worker == null) {
      log.makeAlert("WTF?! Asked to cleanup nonexistent task")
         .addData("taskId", taskId)
         .emit();
    } else {
      final String workerId = worker.getHost();
      log.info("Cleaning up task[%s] on worker[%s]", taskId, workerId);
      final String statusPath = JOINER.join(zkPaths.getIndexerStatusPath(), workerId, taskId);
      try {
        cf.delete().guaranteed().forPath(statusPath);
      }

View Full Code Here

      final ZkWorker theZkWorker,
      final RemoteTaskRunnerWorkItem taskRunnerWorkItem
  ) throws Exception
  {
    Preconditions.checkArgument(task.getId().equals(taskRunnerWorkItem.getTaskId()), "task id != workItem id");
    final Worker theWorker = theZkWorker.getWorker();


    log.info("Coordinator asking Worker[%s] to add task[%s]", theWorker.getHost(), task.getId());


    byte[] rawBytes = jsonMapper.writeValueAsBytes(task);
    if (rawBytes.length > config.getMaxZnodeBytes()) {
      throw new ISE("Length of raw bytes for task too large[%,d > %,d]", rawBytes.length, config.getMaxZnodeBytes());
    }


    String taskPath = JOINER.join(zkPaths.getIndexerTaskPath(), theWorker.getHost(), task.getId());


    if (cf.checkExists().forPath(taskPath) == null) {
      cf.create()
        .withMode(CreateMode.EPHEMERAL)
        .forPath(
            taskPath, rawBytes
        );
    }


    RemoteTaskRunnerWorkItem workItem = pendingTasks.remove(task.getId());
    if (workItem == null) {
      log.makeAlert("WTF?! Got a null work item from pending tasks?! How can this be?!")
         .addData("taskId", task.getId())
         .emit();
      return;
    }


    RemoteTaskRunnerWorkItem newWorkItem = workItem.withWorker(theWorker);
    runningTasks.put(task.getId(), newWorkItem);
    log.info("Task %s switched from pending to running (on [%s])", task.getId(), newWorkItem.getWorker().getHost());


    // Syncing state with Zookeeper - don't assign new tasks until the task we just assigned is actually running
    // on a worker - this avoids overflowing a worker with tasks
    Stopwatch timeoutStopwatch = Stopwatch.createUnstarted();
    timeoutStopwatch.start();
    synchronized (statusLock) {
      while (!isWorkerRunningTask(theWorker, task.getId())) {
        final long waitMs = config.getTaskAssignmentTimeout().toStandardDuration().getMillis();
        statusLock.wait(waitMs);
        long elapsed = timeoutStopwatch.elapsed(TimeUnit.MILLISECONDS);
        if (elapsed >= waitMs) {
          log.error(
              "Something went wrong! [%s] never ran task [%s]! Timeout: (%s >= %s)!",
              theWorker.getHost(),
              task.getId(),
              elapsed,
              config.getTaskAssignmentTimeout()
          );
          taskComplete(taskRunnerWorkItem, theZkWorker, TaskStatus.failure(task.getId()));

View Full Code Here

                                .compressionProvider(new PotentiallyGzippedCompressionProvider(false))
                                .build();
    cf.start();
    cf.create().creatingParentsIfNeeded().forPath(basePath);


    worker = new Worker(
        "host",
        "ip",
        3,
        "v1"
    );

View Full Code Here

  }


  @Test
  public void testDoDisable() throws Exception
  {
    Worker theWorker = jsonMapper.readValue(cf.getData().forPath(announcementsPath), Worker.class);
    Assert.assertEquals("v1", theWorker.getVersion());


    Response res = workerResource.doDisable();
    Assert.assertEquals(Response.Status.OK.getStatusCode(), res.getStatus());


    theWorker = jsonMapper.readValue(cf.getData().forPath(announcementsPath), Worker.class);
    Assert.assertTrue(theWorker.getVersion().isEmpty());
  }

View Full Code Here

  public void testDoEnable() throws Exception
  {
    // Disable the worker
    Response res = workerResource.doDisable();
    Assert.assertEquals(Response.Status.OK.getStatusCode(), res.getStatus());
    Worker theWorker = jsonMapper.readValue(cf.getData().forPath(announcementsPath), Worker.class);
    Assert.assertTrue(theWorker.getVersion().isEmpty());


    // Enable the worker
    res = workerResource.doEnable();
    Assert.assertEquals(Response.Status.OK.getStatusCode(), res.getStatus());
    theWorker = jsonMapper.readValue(cf.getData().forPath(announcementsPath), Worker.class);
    Assert.assertEquals("v1", theWorker.getVersion());
  }

View Full Code Here

0 1

TOP

Related Classes of io.druid.indexing.worker.Worker

io.druid.cli.CliMiddleManager

io.druid.indexing.overlord.RemoteTaskRunner

io.druid.indexing.overlord.RemoteTaskRunnerTest

io.druid.indexing.overlord.scaling.SimpleResourceManagementStrategyTest$TestZkWorker

io.druid.indexing.overlord.ZkWorker

io.druid.indexing.worker.http.WorkerResource

io.druid.indexing.worker.http.WorkerResourceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.