Examples of org.apache.twill.api.ResourceReport

org.apache.twill.api.ResourceReport
This interface provides a snapshot of the resources an application is using broken down by each runnable.


  private boolean waitForDebugPort(TwillController controller, String runnable, int timeLimit)
    throws InterruptedException {
    long millis = 0;
    while (millis < 1000 * timeLimit) {
      ResourceReport report = controller.getResourceReport();
      if (report == null || report.getRunnableResources(runnable) == null) {
        continue;
      }
      for (TwillRunResources resources : report.getRunnableResources(runnable)) {
        if (resources.getDebugPort() != null) {
          return true;
        }
      }
      TimeUnit.MILLISECONDS.sleep(100);

View Full Code Here

  public List<String> getServices(String flowName) {
    List<String> services = Lists.newArrayList();
    try {
      Iterable<TwillController> controllers = lookupFlow(flowName);
      for (TwillController controller : controllers) {
        ResourceReport report = controller.getResourceReport();
        sleepForZK(report);
        services.addAll(report.getServices());
      }
    } catch (Exception e) {
      LOG.warn(e.getMessage(), e);
    }
    return services;

View Full Code Here

  @Override
  public void setInstances(String flowName, String flowletName, int instanceCount) {
    try {
      Iterable<TwillController> controllers = lookupFlow(flowName);
      for (TwillController controller : controllers) {
        ResourceReport report = controller.getResourceReport();
        sleepForZK(report);
        int oldInstances = report.getResources().get(flowletName).size();
        Program program = Programs.create(location.append(flowName));
        Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, program.getSpecification(),
                                                                              queueAdmin);
        DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater(
          program, controller, queueAdmin, consumerQueues);

View Full Code Here

  public Map<String, Integer> getFlowInfo(String flowName) {
    Map<String, Integer> flowletInfo = Maps.newHashMap();
    try {
      Iterable<TwillController> controllers = lookupFlow(flowName);
      for (TwillController controller : controllers) {
        ResourceReport report = controller.getResourceReport();
        sleepForZK(report);
        for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) {
          flowletInfo.put(entry.getKey(), entry.getValue().size());
        }
      }
    } catch (Exception e) {
      LOG.warn(e.getMessage(), e);

View Full Code Here

    Assert.assertTrue(running.await(30, TimeUnit.SECONDS));


    Iterable<Discoverable> echoServices = controller.discoverService("echo");
    Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 2, 60));
    // check that we have 2 runnables.
    ResourceReport report = controller.getResourceReport();
    Assert.assertEquals(2, report.getRunnableResources("BuggyServer").size());


    // cause a divide by 0 in one server
    Discoverable discoverable = echoServices.iterator().next();
    Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
                               discoverable.getSocketAddress().getPort());
    try {
      PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
      writer.println("0");
    } finally {
      socket.close();
    }


    // takes some time for app master to find out the container completed...
    int count = 0;
    while (count < 20) {
      report = controller.getResourceReport();
      // check that we have 1 runnable, not 2.
      if (report.getRunnableResources("BuggyServer").size() == 1) {
        break;
      }
      LOG.info("Wait for BuggyServer to have 1 instance left. Trial {}.", count);
      count++;
      TimeUnit.SECONDS.sleep(1);

View Full Code Here

    Assert.assertTrue(running.await(30, TimeUnit.SECONDS));


    // wait for 3 echo servers to come up
    Iterable<Discoverable> echoServices = controller.discoverService("echo");
    Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 3, 60));
    ResourceReport report = controller.getResourceReport();
    // make sure resources for echo1 and echo2 are there
    Map<String, Collection<TwillRunResources>> usedResources = report.getResources();
    Assert.assertEquals(2, usedResources.keySet().size());
    Assert.assertTrue(usedResources.containsKey("echo1"));
    Assert.assertTrue(usedResources.containsKey("echo2"));


    Collection<TwillRunResources> echo1Resources = usedResources.get("echo1");
    // 2 instances of echo1
    Assert.assertEquals(2, echo1Resources.size());
    // TODO: check cores after hadoop-2.1.0
    for (TwillRunResources resources : echo1Resources) {
      Assert.assertEquals(128, resources.getMemoryMB());
    }


    Collection<TwillRunResources> echo2Resources = usedResources.get("echo2");
    // 2 instances of echo1
    Assert.assertEquals(1, echo2Resources.size());
    // TODO: check cores after hadoop-2.1.0
    for (TwillRunResources resources : echo2Resources) {
      Assert.assertEquals(256, resources.getMemoryMB());
    }


    // Decrease number of instances of echo1 from 2 to 1
    controller.changeInstances("echo1", 1);
    echoServices = controller.discoverService("echo1");
    Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 1, 60));
    report = controller.getResourceReport();


    // make sure resources for echo1 and echo2 are there
    usedResources = report.getResources();
    Assert.assertEquals(2, usedResources.keySet().size());
    Assert.assertTrue(usedResources.containsKey("echo1"));
    Assert.assertTrue(usedResources.containsKey("echo2"));


    echo1Resources = usedResources.get("echo1");

View Full Code Here

    if (controllers.hasNext()) {
      TwillController controller = controllers.next();
      if (controllers.hasNext()) {
        LOG.warn("Expected at most one live instance of Twill app {} but found at least two.", twillAppName);
      }
      ResourceReport report = controller.getResourceReport();
      if (report != null) {
        DistributedProgramLiveInfo liveInfo = new DistributedProgramLiveInfo(program, type, report.getApplicationId());


        // if program type is flow then the container type is flowlet.
        Containers.ContainerType containerType = ProgramType.FLOW.equals(type) ? FLOWLET :
                                                 Containers.ContainerType.valueOf(type.name());


        for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) {
          for (TwillRunResources resources : entry.getValue()) {
            liveInfo.addContainer(new ContainerInfo(containerType,
                                                    entry.getKey(),
                                                    resources.getInstanceId(),
                                                    resources.getContainerId(),
                                                    resources.getHost(),
                                                    resources.getMemoryMB(),
                                                    resources.getVirtualCores(),
                                                    resources.getDebugPort()));
          }
        }


        // Add a list of announced services and their discoverables to the liveInfo.
        liveInfo.addServices(report.getServices());
        return liveInfo;
      }
    }
    return new NotRunningProgramLiveInfo(program, type);
  }

View Full Code Here

        int containers = 0;
        int memory = 0;
        int vcores = 0;
        // will have multiple controllers if there are multiple runs of the same application
        for (TwillController controller : info.getControllers()) {
          ResourceReport report = controller.getResourceReport();
          if (report == null) {
            continue;
          }
          containers++;
          memory += report.getAppMasterResources().getMemoryMB();
          vcores += report.getAppMasterResources().getVirtualCores();
        }
        sendMetrics(metricContext, containers, memory, vcores);
      }
      reportClusterStorage();
      reportClusterMemory();

View Full Code Here

TOP

Related Classes of org.apache.twill.api.ResourceReport

co.cask.cdap.internal.app.runtime.distributed.DistributedProgramRuntimeService

co.cask.cdap.internal.app.runtime.distributed.DistributedProgramRuntimeService$ClusterResourceReporter

co.cask.tigon.cli.DistributedFlowOperations

org.apache.twill.yarn.DebugTestRun

org.apache.twill.yarn.ResourceReportTestRun

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.