Examples of com.alibaba.jstorm.cluster.StormClusterState

com.alibaba.jstorm.cluster.StormClusterState
all storm in zk operation interface

  public TopologyMetricInfo getTopologyMetric(String topologyId) throws NotAliveException, TException{
    LOG.debug("Nimbus service handler, getTopologyMetric, topology ID: " + topologyId);
    
    TopologyMetricInfo topologyMetricInfo = new TopologyMetricInfo();
        
    StormClusterState clusterState = data.getStormClusterState();
    
    topologyMetricInfo.set_topology_id(topologyId);
    try {
      //update task metrics list
      Map<Integer, TaskInfo> taskInfoList = clusterState.task_info_list(topologyId);
        List<TaskMetricInfo> taskMetricList = clusterState.get_task_metric_list(topologyId);      
        for(TaskMetricInfo taskMetricInfo : taskMetricList) {
          TaskMetricData taskMetricData = new TaskMetricData();
          NimbusUtils.updateTaskMetricData(taskMetricData, taskMetricInfo);
          TaskInfo taskInfo = taskInfoList.get(Integer.parseInt(taskMetricInfo.getTaskId()));
          String componentId = taskInfo.getComponentId();
          taskMetricData.set_component_id(componentId);
          
          topologyMetricInfo.add_to_task_metric_list(taskMetricData);
        }
        
        //update worker metrics list
        List<WorkerMetricInfo> workerMetricList = clusterState.get_worker_metric_list(topologyId);
        for(WorkerMetricInfo workerMetricInfo : workerMetricList) {
          WorkerMetricData workerMetricData = new WorkerMetricData();
          NimbusUtils.updateWorkerMetricData(workerMetricData, workerMetricInfo);
          
          topologyMetricInfo.add_to_worker_metric_list(workerMetricData);

View Full Code Here

   * @param nimbusData
   * @param active_topologys
   * @throws Exception
   */
  public void cleanupDisappearedTopology() throws Exception {
    StormClusterState clusterState = nimbusData.getStormClusterState();


    List<String> active_topologys = clusterState.active_storms();
    if (active_topologys == null) {
      return;
    }


    Set<String> cleanupIds = get_cleanup_ids(clusterState, active_topologys);


    for (String topologyId : cleanupIds) {


      LOG.info("Cleaning up " + topologyId);


      // remove ZK nodes /taskbeats/topologyId and
      // /taskerror/topologyId
      clusterState.teardown_heartbeats(topologyId);
      clusterState.teardown_task_errors(topologyId);


      //
      nimbusData.getTaskHeartbeatsCache().remove(topologyId);


      // get /nimbus/stormdist/topologyId

View Full Code Here

   * @param stormClusterState
   * @param topologyId
   * @throws Exception
   */
  public void setTopologyStatus(TopologyAssignEvent event) throws Exception {
    StormClusterState stormClusterState = nimbusData.getStormClusterState();


    String topologyId = event.getTopologyId();
    String topologyName = event.getTopologyName();
    String group = event.getGroup();


    StormStatus status = new StormStatus(StatusType.active);
    if (event.getOldStatus() != null) {
      status = event.getOldStatus();
    }


    StormBase stormBase = stormClusterState.storm_base(topologyId, null);
    if (stormBase == null) {
      stormBase = new StormBase(topologyName,
          TimeUtils.current_time_secs(), status, group);
      stormClusterState.activate_storm(topologyId, stormBase);


    } else {


      stormClusterState.update_storm(topologyId, status);


      // here exist one hack operation
      // when monitor/rebalance/startup topologyName is null
      if (topologyName == null) {
        event.setTopologyName(stormBase.getStormName());

View Full Code Here

    Map stormConf = new HashMap();
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);


    StormClusterState stormClusterState = nimbusData.getStormClusterState();


    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.allSupervisorInfo(
        stormClusterState, null);
    if (supInfos.size() == 0) {
      throw new FailedAssignTopologyException(
          "Failed to make assignment " + topologyId
              + ", due to no alive supervisor");
    }


    Map<Integer, String> taskToComponent = Cluster.topology_task_info(
        stormClusterState, topologyId);
    ret.setTaskToComponent(taskToComponent);


    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds == null || allTaskIds.size() == 0) {
      String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/"
          + topologyId;
      LOG.warn(errMsg);
      throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);


    Set<Integer> aliveTasks = new HashSet<Integer>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<Integer>();
    Set<Integer> deadTasks = new HashSet<Integer>();


    Assignment existingAssignment = stormClusterState.assignment_info(
        topologyId, null);
    if (existingAssignment != null) {
      aliveTasks = getAliveTasks(topologyId, allTaskIds);
      unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos,
          existingAssignment);


      deadTasks.addAll(allTaskIds);
      deadTasks.removeAll(aliveTasks);
    }


    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);


    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);


    if (existingAssignment == null) {
      ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);


      try {
        AssignmentBak lastAssignment = stormClusterState
            .assignment_bak(event.getTopologyName());
        if (lastAssignment != null) {
          ret.setOldAssignment(lastAssignment.getAssignment());
        }
      } catch (Exception e) {

View Full Code Here

    String codeDir = StormConfig.masterStormdistRoot(nimbusData.getConf(),
        topologyId);


    assignment = new Assignment(codeDir, assignments, nodeHost, startTimes);


    StormClusterState stormClusterState = nimbusData.getStormClusterState();


    stormClusterState.set_assignment(topologyId, assignment);


    // update task heartbeat's start time
    NimbusUtils.updateTaskHbStartTime(nimbusData, assignment, topologyId);


    // Update metrics information in ZK when rebalance or reassignment

View Full Code Here

      if (existingAssignment.getWorkers() != null) {
        oldWorkers = existingAssignment.getWorkers();
      }
    }


    StormClusterState zkClusterState = nimbusData.getStormClusterState();
    Set<Integer> changeTaskIds = getChangeTaskIds(oldWorkers, workers);
    int nowSecs = TimeUtils.current_time_secs();
    for (Integer changedTaskId : changeTaskIds) {
      startTimes.put(changedTaskId, nowSecs);


      zkClusterState.remove_task_heartbeat(topologyId, changedTaskId);
    }


    LOG.info("Task assignment has been changed " + changeTaskIds);
    return startTimes;
  }

View Full Code Here

      TopologyAssignEvent event) {
    String topologyId = event.getTopologyId();
    String topologyName = event.getTopologyName();
    try {


      StormClusterState zkClusterState = nimbusData
          .getStormClusterState();
      // one little problem, get tasks twice when assign one topology
      HashMap<Integer, String> tasks = Cluster.topology_task_info(
          zkClusterState, topologyId);


      Map<String, List<Integer>> componentTasks = JStormUtils
          .reverse_map(tasks);


      for (Entry<String, List<Integer>> entry : componentTasks.entrySet()) {
        List<Integer> keys = entry.getValue();


        Collections.sort(keys);


      }


      AssignmentBak assignmentBak = new AssignmentBak(componentTasks,
          assignment);
      zkClusterState.backup_assignment(topologyName, assignmentBak);


    } catch (Exception e) {
      LOG.warn("Failed to backup " + topologyId + " assignment "
          + assignment, e);
    }

View Full Code Here

  public void metricsMonitor(TopologyAssignEvent event) {
    String topologyId = event.getTopologyId();
    try {
      Map<Object, Object> conf = nimbusData.getConf();
      boolean isEnable =  ConfigExtension.isEnablePerformanceMetrics(conf);
      StormClusterState zkClusterState = nimbusData.getStormClusterState();
      StormMonitor monitor = new StormMonitor(isEnable);
      zkClusterState.set_storm_monitor(topologyId, monitor);
    } catch (Exception e) {
      LOG.warn("Failed to update metrics monitor status of " + topologyId, e);
    }
  }

View Full Code Here

    state = false;
  }


  private synchronized void check() {


    StormClusterState clusterState = data.getStormClusterState();


    try {
      String master_stormdist_root = StormConfig.masterStormdistRoot(data
          .getConf());


      List<String> code_ids = PathUtils
          .read_dir_contents(master_stormdist_root);


      List<String> assignments_ids = clusterState.assignments(callback);


      List<String> done_ids = new ArrayList<String>();


      for (String id : code_ids) {
        if (assignments_ids.contains(id)) {
          done_ids.add(id);
        }
      }


      for (String id : done_ids) {
        assignments_ids.remove(id);
        code_ids.remove(id);
      }


      for (String topologyId : code_ids) {
        deleteLocalTopology(topologyId);
      }


      for (String id : assignments_ids) {
        Assignment assignment = clusterState.assignment_info(id, null);
        downloadCodeFromMaster(assignment, id);
      }
    } catch (IOException e) {
      // TODO Auto-generated catch block
      LOG.error("Get stormdist dir error!", e);

View Full Code Here

   * @throws Exception
   */
  private void checkOwnMaster() throws Exception {
    int retry_times = 10;


    StormClusterState zkClient = data.getStormClusterState();
    for (int i = 0; i < retry_times; i++, JStormUtils.sleepMs(sleepTime)) {


      if (zkClient.leader_existed() == false) {
        continue;
      }


      String zkHost = zkClient.get_leader_host();
      if (hostPort.equals(zkHost) == true) {
        // current process own master
        return;
      }
      LOG.warn("Current Nimbus has start thrift, but fail to own zk master :"

View Full Code Here

0 1 2

TOP

Related Classes of com.alibaba.jstorm.cluster.StormClusterState

com.alibaba.jstorm.daemon.nimbus.NimbusUtils

com.alibaba.jstorm.daemon.nimbus.ServiceHandler

com.alibaba.jstorm.daemon.nimbus.TopologyAssign

com.alibaba.jstorm.daemon.supervisor.Supervisor

com.alibaba.jstorm.daemon.worker.metrics.StormMetricReporter

com.alibaba.jstorm.schedule.FollowerRunnable

com.alibaba.jstorm.schedule.MonitorRunnable

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.