Package models.agent.batch.commands.message

Examples of models.agent.batch.commands.message.BatchResponseFromManager


 
  public BatchResponseFromManager sendAgentCommandToManagerRemote(
      String nodeGroupType,  String agentCommandType, Map<String, NodeGroupDataMap> dataStore,
      boolean localMode, boolean failOver, String directorJobUuid, int maxConcNum, Set<Address> nodeList) {
    if (system == null) return null;
    BatchResponseFromManager agentCommandResponseFromManager = null;
 
    AgentCommandMetadata agentCommandMetadata = AgentDataProvider.agentCommandMetadatas
        .get(agentCommandType);
    NodeGroupDataMap ngdm = dataStore.get(nodeGroupType);
         
    if (agentCommandMetadata == null) {
      models.utils.LogUtils
        .printLogError("!!ERROR in  sendAgentCommandToManager : "
              + directorJobUuid
              + " agentCommandType is NULL!! return. at "
              + DateUtils.getNowDateTimeStr());
      return agentCommandResponseFromManager;
    }
         
    if (ngdm == null) {
      models.utils.LogUtils
          .printLogError("!!ERROR in  sendAgentCommandToManager : "
              + nodeGroupType
              + " NodeGroupDataMap is NULL!! return. at "
              + DateUtils.getNowDateTimeStr());
      return agentCommandResponseFromManager;
    }
   
    models.utils.LogUtils
      .printLogNormal("Before Safety Check: total entry count: "
        + ngdm.getNodeDataMapValid().size());
    Map<String, NodeData> nodeDataMapValidSafe = new HashMap<String, NodeData>();
   
    AgentDataProviderHelper.filterUnsafeOrUnnecessaryRequest(
        ngdm.getNodeDataMapValid(), nodeDataMapValidSafe,
        agentCommandType);
         
    if (localMode || nodeList.size() ==0 )
        localLoad.addAndGet(maxConcNum);
   
    /**
     * Set jobInfo
     */
   
    JobStatus.JobInfo jobInfo = jobStatus.getJobInfo(directorJobUuid);
    jobInfo.startTime = System.currentTimeMillis();
    jobInfo.state = State.processing;
    jobInfo.maxConcNum = maxConcNum;
   
    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    Timestamp now = new Timestamp(jobInfo.startTime);
    jobInfo.timeStamp = df.format(now);
       
    int totJobNum = nodeDataMapValidSafe.size();
    jobInfo.totJobNum = totJobNum;
    /**
     * Prepare for create manager
     */
   
    List<ActorRef> managerList = new ArrayList<ActorRef>();
    Queue<Integer> jobQ = new ConcurrentLinkedQueue<Integer>();
   
    /**
     * @author chunyang
     * Create Managers locally or remotely
     */
   
    ActorRef agentCommandManager = null;
   
    if (localMode || nodeList.size()==0) {
      agentCommandManager =  system.actorOf(
          Props.create(LocalManager.class),"AgentCommandManager-" + UUID.randomUUID().toString()
        );
      jobQ.offer(managerList.size());
      managerList.add(agentCommandManager);
      localMode = true;
    }
    else {
      for (Address m : nodeList) {
        agentCommandManager = system.actorOf(
          Props.create(LocalManager.class).withDeploy(
              new Deploy(
                   new RemoteScope(
                      m
                      )
                  )),
          "AgentCommandManager-" + UUID.randomUUID().toString()
        );
       
        jobQ.offer(managerList.size());
        managerList.add(agentCommandManager);
        ClusterState.memberLoad.get(m).addAndGet(maxConcNum);
      }
    }
   
    /**
     * Dispatch Jobs
     * @author chunyang
     */
    if (!localMode) {
      List<Map<String, NodeData>> jobGroupList = partDataStore(nodeDataMapValidSafe, nodeList.size()==0?
          totJobNum:Math.min(totJobNum/nodeList.size()+1, 1000));
      List<ActorRef> dispatcherList = new ArrayList<ActorRef>();
      int requestChunckSize = jobGroupList.size()/managerList.size() + 1; // Last one do less
     
      for (int i=0; i<Math.min(3, managerList.size()) ; i++) {
        dispatcherList.add(
            system.actorOf(
                Props.create(JobDispatcher.class, managerList, jobGroupList, jobQ, requestChunckSize,
                    nodeGroupType, agentCommandType, directorJobUuid, maxConcNum)
                )
            );
      }
     
      for (ActorRef dispatcher : dispatcherList) {
        dispatcher.tell("start dispatching", null);
      }
    } else {
      Map<String, NodeGroupDataMap> totDataStore = new HashMap<String, NodeGroupDataMap>();
      totDataStore.put(nodeGroupType, new NodeGroupDataMap(nodeGroupType));
      totDataStore.get(nodeGroupType).setNodeDataMapValid(nodeDataMapValidSafe);
      Future<Object> ackFuture = Patterns.ask(managerList.get(0), new InitialRequestToManager(nodeGroupType,
          agentCommandType, directorJobUuid, totDataStore, true, false, maxConcNum), new Timeout(Duration.create(
              15, TimeUnit.SECONDS)));
      try {
        Await.result(ackFuture,  Duration.create(
            15, TimeUnit.SECONDS));
      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      managerList.get(0).tell(new endOfRequest(), null);
    }
   
    ActorRef monitor = system.actorOf(Props.create(Monitor.class, managerList, directorJobUuid, jobStatus), "ProgressMonitor" + directorJobUuid);
   
    ActorRef collector = system.actorOf(Props.create(ResponseCollector.class, jobInfo), "ResultCollector" + directorJobUuid);
   
    final FiniteDuration gatherResponseDuration = Duration.create(
        3600, TimeUnit.SECONDS);
   
    /**
     * Gather Result.
     */
    Future<Object> totResponse = Patterns.ask(collector, new gatherResponse(monitor, totJobNum), new Timeout(gatherResponseDuration));
   
    BatchResponseFromManager responseFromCollecter = null;
    try {
      responseFromCollecter = (BatchResponseFromManager) Await.result(totResponse, gatherResponseDuration);
      System.out.println("Gather Result Back! : " + responseFromCollecter.responseMap.size());
      /**
       * Slave Fail Over
       */
      int failCount = 3;
      while (responseFromCollecter.responseMap.size() < totJobNum && failCount >= 0) {
        System.out.println("Response less than request, fail over @@");
        failCount -- ;
        Map<String, NodeData> failOverMap = gatherFailOverData(nodeDataMapValidSafe, responseFromCollecter);
       
        List<Address> failOverNodeList = new ArrayList<Address>();
       
        int failOverTot = failOverMap.size();
       
        for (Address m : nodeList) {
          if (ClusterState.memberLoad.containsKey(m)) {
            failOverNodeList.add(m);
            failOverTot -= 2000;
            if (failOverTot < 0)
              break;
          }
        } 
       
       
        List<ActorRef> failOverManagerList = new ArrayList<ActorRef>();
        Queue<Integer> failOverJobQ = new ConcurrentLinkedQueue<Integer>();
       
        if (localMode || failOverNodeList.size()==0) {
          agentCommandManager =  system.actorOf(
              Props.create(LocalManager.class),"AgentCommandManager-" + UUID.randomUUID().toString()
            );
          failOverJobQ.offer(failOverManagerList.size());
          failOverManagerList.add(agentCommandManager);
          managerList.add(agentCommandManager);
          localMode = true;
        }
        else {
          for (Address m : failOverNodeList) {
            agentCommandManager = system.actorOf(
              Props.create(LocalManager.class).withDeploy(
                  new Deploy(
                       new RemoteScope(
                          m
                          )
                      )),
              "AgentCommandManager-" + UUID.randomUUID().toString()
            );
           
            failOverJobQ.offer(failOverManagerList.size());
            failOverManagerList.add(agentCommandManager);
            managerList.add(agentCommandManager);
          }
        }
        if (!localMode) {
          List<Map<String, NodeData>> failOverJobGroupList = partDataStore(failOverMap, failOverNodeList.size()==0?
              failOverMap.size():Math.min(failOverMap.size()/failOverNodeList.size()+1, 1000));
          List<ActorRef> failOverDispatcherList = new ArrayList<ActorRef>();
          int failOverRequestChunckSize = failOverJobGroupList.size()/failOverManagerList.size() + 1; // Last one do less
         
          for (int i=0; i<Math.min(3, failOverManagerList.size()) ; i++) {
            failOverDispatcherList.add(
                system.actorOf(
                    Props.create(JobDispatcher.class, failOverManagerList, failOverJobGroupList, failOverJobQ, failOverRequestChunckSize,
                        nodeGroupType, agentCommandType, directorJobUuid, maxConcNum)
                    )
                );
          }
         
          for (ActorRef failOverDispatcher : failOverDispatcherList) {
            failOverDispatcher.tell("start dispatching", null);
          }
        } else {
          Map<String, NodeGroupDataMap> failOverDataStore = new HashMap<String, NodeGroupDataMap>();
          failOverDataStore.put(nodeGroupType, new NodeGroupDataMap(nodeGroupType));
          failOverDataStore.get(nodeGroupType).setNodeDataMapValid(failOverMap);
          Future<Object> ackFuture = Patterns.ask(failOverManagerList.get(0), new InitialRequestToManager(nodeGroupType,
              agentCommandType, directorJobUuid, failOverDataStore, true, false, maxConcNum), new Timeout(Duration.create(
                  15, TimeUnit.SECONDS)));
          try {
            Await.result(ackFuture,  Duration.create(
                15, TimeUnit.SECONDS));
          } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
          }
          failOverManagerList.get(0).tell(new endOfRequest(), null);
        }
       
        jobInfo.state = State.processing;
       
       
        Future<Object> futureFailOverResponse = Patterns.ask(collector, new gatherResponse(monitor, totJobNum), new Timeout(gatherResponseDuration));
        BatchResponseFromManager failOverResponse = (BatchResponseFromManager) Await.result(futureFailOverResponse, gatherResponseDuration);
        System.out.println("FailOver Result Size" + failOverResponse.responseMap.size());
        for (Entry<String, GenericAgentResponse> e : failOverResponse.responseMap.entrySet()) {
          responseFromCollecter.responseMap.put(e.getKey(), e.getValue());
        }
      }
     
      for (Entry<String, GenericAgentResponse> e: responseFromCollecter.getResponseMap().entrySet()) {
        AgentDataProviderHelper.getInstance()
        .updateResponseFromAgentGenericResponse(nodeGroupType,
            agentCommandType, e.getValue(), dataStore);
      }
    } catch (Exception e) {
      System.out.println("Response Collector Timeout");
      responseFromCollecter = new BatchResponseFromManager();
    }
   
    jobInfo.endTime = System.currentTimeMillis();
    jobInfo.aggregationTime = (jobInfo.endTime - jobInfo.finishedNotAggregatedTime)/1000.0;
    jobInfo.state = State.gathered;
View Full Code Here


      getSender().tell(streamResponse(), getSelf());
    }
  }
 
  private BatchResponseFromManager streamResponse() {
    BatchResponseFromManager response = new BatchResponseFromManager();
    ActorRef manager = slaveQ.poll();
    while (manager != null) {
      FiniteDuration duration = Duration.create(
          60, TimeUnit.SECONDS);
      streamRequestToManager streamRequest = new streamRequestToManager(0);
      Future<Object> futureResponse = Patterns.ask(manager, streamRequest, new Timeout(duration));
      System.out.println(manager.path().address().toString());
     
      try {
        BatchResponseFromManager partResponse = (BatchResponseFromManager) Await.result(futureResponse, duration);
        while (partResponse.getResponseMap().size() > 0) {
          collectResult(response, partResponse);
          streamRequest.index += partResponse.getResponseMap().size();
          futureResponse = Patterns.ask(manager, streamRequest, new Timeout(duration));
          partResponse = (BatchResponseFromManager) Await.result(futureResponse, duration);
        }
      } catch (Exception e) {
        System.out.println("Gather Response Timeout");
View Full Code Here

  public void onReceive(Object message) {
    if (message instanceof gatherResponse) {
      gatherResponse gR = (gatherResponse) message;
      this.monitor = gR.monitor;
      director = getSender();
      collectedResponse = new BatchResponseFromManager();
      for (int i=0; i<3; i++) {
        individualCollectors.add(getContext().system().actorOf(
            Props.create(IndividualCollector.class, slaveQ, atomic)
          ));
      }
      getContext().system().scheduler().scheduleOnce(Duration.create(1, TimeUnit.SECONDS),
          new Runnable() {
              @Override
              public void run() {
                getSelf().tell(new collecterCheckResponse(), ActorRef.noSender());
           }
      }, getContext().system().dispatcher());
     
    } else if (message instanceof collecterCheckResponse){
      FiniteDuration duration = Duration.create(
          10, TimeUnit.SECONDS);
      Future<Object> futureList = Patterns.ask(monitor, new queryMonitorForResponse(), new Timeout(duration));
      try {
        List<ActorRef> collectList = (List<ActorRef>) Await.result(futureList, duration);
        if (collectList.size() > 0) {
          for (ActorRef m : collectList) {
            slaveQ.offer(m);
            atomic.incrementAndGet();
          }
          for (ActorRef m : individualCollectors) {
            atomic.incrementAndGet();
            m.tell("start", getSelf());
          }
        }
      } catch (Exception e) {
        System.out.println("Collector ask monitor timeout");
      }
 
      if (atomic.intValue() == 0 && jobInfo.state == State.finishedNotGathered) {
        director.tell(collectedResponse, getSelf());
        for (ActorRef ic : individualCollectors) {
          getContext().stop(ic);
        }
        individualCollectors.clear();
      } else {
        getContext().system().scheduler().scheduleOnce(Duration.create(2, TimeUnit.SECONDS),
            new Runnable() {
                @Override
                public void run() {
                  getSelf().tell(new collecterCheckResponse(), ActorRef.noSender());
             }
        }, getContext().system().dispatcher());
      }
    } else if (message instanceof BatchResponseFromManager) {
      BatchResponseFromManager res = (BatchResponseFromManager) message;
      for (Entry<String, GenericAgentResponse> e : res.responseMap.entrySet()) {
        collectedResponse.responseMap.put(e.getKey(), e.getValue());
      }
      jobInfo.collectedNum += res.responseMap.size();
      atomic.decrementAndGet();
View Full Code Here

          .printLogNormal("!Obtain request! agent command request for nodeGroupType : "
              + nodeGroupType + "  with count: " + requestCount);
      // If there are no seeders in the incoming message, send a
      // message back
      if (requestCount <= 0) {
        getSender().tell(new BatchResponseFromManager(), getSelf());
        return;
      }
     
      director.tell("ACK", getSelf());
   
    } else if (message instanceof AgentResponse) {
   
      try {
        GenericAgentResponse gap = null;
   
        // this guareetee gap is not NULL :-)
        if ((message instanceof GenericAgentResponse)) {
   
          gap = (GenericAgentResponse) message;
        } else {
   
          gap = new GenericAgentResponse((AgentResponse) message,
              DateUtils.getNowDateTimeStrSdsm());
        }
   
        // 20130422 to trim the message if needed; careful, when there
        // are exception: will make the
        // bug fixed: 20130423 check gap.getResponseContent() length
        // ==0; then -1 will make it out of bound!
       
        if (gap != null && gap.getResponseContent() != null
            && !gap.getResponseContent().isEmpty()
   
            && responseExtractIndexStart >= 0
            && responseExtractIndexEnd >= 0
            && responseExtractIndexStart <= responseExtractIndexEnd) {
   
          int trimStartIndex = (int) Math.min(
              responseExtractIndexStart, gap.getResponseContent()
                  .length() - 1);
          int trimEndIndex = (int) Math.min(responseExtractIndexEnd,
              gap.getResponseContent().length() - 1);
          trimStartIndex = (trimStartIndex < 0) ? 0 : trimStartIndex;
          trimEndIndex = (trimEndIndex < 0) ? 0 : trimEndIndex;
          gap.setResponseContent(gap.getResponseContent().substring(
              trimStartIndex, trimEndIndex));
        }
   
        this.responseCount += 1;
        /**
         * 20131009: add feedback of current responseCount to asst
         * manager ResponseCountToBatchSenderAsstManager
         */
        final ResponseCountToBatchSenderAsstManager responseCountToBatchSenderAsstManager = new ResponseCountToBatchSenderAsstManager(
            this.responseCount);
   
        if (batchSenderAsstManager != null) {
          batchSenderAsstManager.tell(
              responseCountToBatchSenderAsstManager, getSelf());
   
          if (VarUtils.IN_DETAIL_DEBUG) {
   
            models.utils.LogUtils
                .printLogError("Send batchSenderAsstManager to responseCountToBatchSenderAsstManager: "
                    + this.responseCount);
          }
        } else {
          models.utils.LogUtils
              .printLogError("batchSenderAsstManager is empty; when trying to send responseCountToBatchSenderAsstManager In manager resonse handling. "
                  + DateUtils.getNowDateTimeStrSdsm());
        }
   
        String hostName = gap.getHost();
        responseListKey.add(hostName);
        responseListValue.add(gap);
   
        String displayResponse = (gap.getResponseContent() == null || gap
            .getResponseContent().length() < 1) ? "RESPONSE_CONTENT_EMPTY_OR_NULL"
            : gap.getResponseContent()
                .substring(
                    0,
                    Math.min(
                        VarUtils.AGNET_RESPONSE_MAX_RESPONSE_DISPLAY_BYTE1,
                        gap.getResponseContent()
                            .length()));
     
   
        if (this.responseCount % 1 == 0) {
          long responseReceiveTime = System.currentTimeMillis();
          // %.5g%n
          double progressPercent = (double) (responseCount)
              / (double) (requestCount) * 100.0;
          String responseReceiveTimeStr = DateUtils
              .getDateTimeStr(new Date(responseReceiveTime));
          String secondElapsedStr = new Double(
              (responseReceiveTime - startTime) / 1000.0)
              .toString();
          // percent is escaped using percent sign
          models.utils.LogUtils
              .printLogNormal(String
                  .format("\n[%d]__RESPONSE_RECV_IN_MGR %d (+%d) / %d (%.5g%%)  after %s SEC for %s at %s , directorJobId : %s , RESPONSE_BRIEF: %s\n",
                      responseCount, responseCount,
                      requestCount - responseCount,
                      requestCount, progressPercent,
                      secondElapsedStr, hostName,
                      responseReceiveTimeStr,
                      directorJobId, displayResponse));
        }
   
        getContext().stop(getSender());
       
        if (this.responseCount == this.requestCount && requestComplete) {
         
          System.out.println("All result back.");
         
          models.utils.LogUtils
              .printLogNormal("SUCCESSFUL GOT ON ALL RESPONSES: Received all the expected messages. Count matches: "
                  + this.requestCount
                  + " at time: "
                  + DateUtils.getNowDateTimeStrSdsm());
   
          // Send message to the future with the result
          long currTime = System.currentTimeMillis();
          models.utils.LogUtils
              .printLogNormal("\nTime taken to get all responses back : "
                  + (currTime - startTime) / 1000.0 + " secs");
   
          // MUST SHUT DOWN: This is a double check. Acutally agent
          // operation worker has already shutdown.
          for (ActorRef worker : workerQ) {
            if (worker!=null)
              getContext().stop(worker);
          }
   
          // 20131010: kill asst manager
   
          if (timeoutMessageCancellable != null) {
            timeoutMessageCancellable.cancel();
          }
       
        }// end when all requests have resonponse
   
      } catch (Throwable t) {
        t.printStackTrace();
      }
      } else if (message instanceof InternalMessageType) {
      switch ((InternalMessageType) message) {
   
      case OPERATION_TIMEOUT:
        cancelRequestAndCancelWorkers();
        break;
      }
    } else if (message instanceof Map<?, ?>) {
     
      System.out.println("Part request!");
     
      requestComplete = false;
      List<NodeData> retryNodeDataList = new ArrayList<NodeData>();
      List<String> retryJobIdList = new ArrayList<String>();
      requestCount += ((Map<?, ?>) message).size();
      for(Entry<String, NodeData> e: ((Map<String, NodeData>) message).entrySet()) {
        retryJobIdList.add(e.getKey());
        retryNodeDataList.add(e.getValue());
        dataStore.get(nodeGroupType).getNodeDataMapValid().put(e.getKey(), e.getValue());
      }
      getSender().tell("ACK", getSelf());
     
    }else if (message instanceof streamRequestToManager) {
      streamRequestToManager sRTM = (streamRequestToManager) message;
     
      System.out.println("StreamRequestToManager : " + sRTM.index);
     
      BatchResponseFromManager partStreamResponse = new BatchResponseFromManager();
      if (sRTM.index >= responseListKey.size()) {
        getSender().tell(partStreamResponse, getSelf());
      } else {
        int chunckSize = Math.min(1024, responseListKey.size()-sRTM.index);
        try {
          for (int i=sRTM.index; i< sRTM.index + chunckSize; i++) {
            partStreamResponse.responseMap.put(responseListKey.get(i), responseListValue.get(i));
          }
          while (sizeof(partStreamResponse) > 15728640 && chunckSize >= 1) {
            chunckSize /= 2;
            partStreamResponse = new BatchResponseFromManager();
            for (int i=sRTM.index; i< sRTM.index + chunckSize; i++) {
              partStreamResponse.responseMap.put(responseListKey.get(i), responseListValue.get(i));
            }
          }
          if (chunckSize == 0) {
View Full Code Here

      if (nodeGroupSourceMetadata == null) {

        String errMessage = "nodeGroupSourceMetadata  is NULL in generateUpdateSendAgentCommandToNodeGroupHelper. EXIT!!"
            + DateUtils.getNowDateTimeStrSdsm();
        models.utils.LogUtils.printLogError(errMessage);
        batchResponseFromManager = new BatchResponseFromManager();

        return batchResponseFromManager;
      }

      // generate content
View Full Code Here

      if (nodeGroupSourceMetadata == null) {

        String errMessage = "nodeGroupSourceMetadata  is NULL in generateUpdateSendAgentCommandToNodeGroupHelper. EXIT!!"
            + DateUtils.getNowDateTimeStrSdsm();
        models.utils.LogUtils.printLogError(errMessage);
        batchResponseFromManager = new BatchResponseFromManager();

        return batchResponseFromManager;
      }

      // generate content
View Full Code Here

      maxConcurrencyAdjusted = iniRequest.maxConcNum;
     
      // If there are no requests in the incoming message, send a
      // message back
      if (requestTotalCount <= 0) {
        originalManager.tell(new BatchResponseFromManager(), getSelf());
        return;
      }
     
      sendMessageUntilStopCount(maxConcurrencyAdjusted);
View Full Code Here

TOP

Related Classes of models.agent.batch.commands.message.BatchResponseFromManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.