Examples of org.apache.uima.ducc.transport.event.common.DuccWorkJob

Package org.apache.uima.ducc.transport.event.common

Examples of org.apache.uima.ducc.transport.event.common.DuccWorkJob

org.apache.uima.ducc.transport.event.common.DuccWorkJob

  }
  
  private void updateJobs(IDuccWork duccWork) {
    String location = "updateJobs";
    DuccId duccId = duccWork.getDuccId();
    DuccWorkJob job = (DuccWorkJob)duccWork;
    if(keyMapJobs.containsKey(duccId)) {
      sortedJobs.remove(keyMapJobs.get(duccId));
      keyMapJobs.remove(duccId);
    }
    JobInfo jobInfo = new JobInfo(job);

View Full Code Here

  }
  
  private void updateServices(IDuccWork duccWork) {
    String location = "updateServices";
    DuccId duccId = duccWork.getDuccId();
    DuccWorkJob service = (DuccWorkJob)duccWork;
    if(keyMapServices.containsKey(duccId)) {
      sortedServices.remove(keyMapServices.get(duccId));
      keyMapServices.remove(duccId);
    }
    JobInfo serviceInfo = new JobInfo(service);
    sortedServices.put(serviceInfo, serviceInfo);
    if(!duccWork.isCompleted()) {
      keyMapServices.put(duccId, serviceInfo);
      logger.debug(location, duccId, "put service");
    }
    //
    ServiceDeploymentType sdt = service.getServiceDeploymentType();
    if(sdt != null) {
      switch(sdt) {
      case other:
        if(keyMapCombinedReservations.containsKey(duccId)) {
          sortedCombinedReservations.remove(keyMapCombinedReservations.get(duccId));

View Full Code Here

  public boolean equals(Object object) {
    boolean retVal = false;
    try {
      JobInfo i1 = this;
      JobInfo i2 = (JobInfo)object;
      DuccWorkJob j1 = i1.getJob();
      DuccWorkJob j2 = i2.getJob();
      String s1 = j1.getDuccId().toString();
      String s2 = j2.getDuccId().toString();
      retVal = s1.equals(s2);
    }
    catch(Throwable t) {  
    }
    return retVal;

View Full Code Here

  }
  
   
  public int hashCode() {
    JobInfo i1 = this;
    DuccWorkJob j1 = i1.getJob();
    String s1 = j1.getDuccId().toString();
    return s1.hashCode();
  }

View Full Code Here

  }
  
  protected void process(OrchestratorAbbreviatedStateDuccEvent duccEvent) {
    String methodName = "process";
    duccOut.trace(methodName, null, duccMsg.fetch("enter"));
    DuccWorkJob job = (DuccWorkJob) duccEvent.getWorkMap().findDuccWork(DuccType.Job, jobId);
    if(dumpProcessMapEnabled) {
      dumpProcessMap(job);
    }
    if(job != null) {
      if(duccId == null) {
        duccId = job.getDuccId();
      }
      duccOut.trace(methodName, duccId, "jd-cmd:"+job.getDriver().getCommandLine());
      duccOut.trace(methodName, duccId, "jp-cmd:"+job.getCommandLine());
      synchronized(jobId) {
        if(thread != null) {
          thread.setJob(job);
        }
        if(!started.get()) {
          started.set(true);
          duccOut.debug(methodName, job.getDuccId(), job.getJobState());
          duccOut.trace(methodName, job.getDuccId(), duccMsg.fetch("creating driver thread"));
          try {
            thread = new JobDriver();
            duccOut.trace(methodName, job.getDuccId(), "thread:"+thread);
            thread.initialize(job, getProcessJmxUrl());
            thread.start();
            jpc = new JobProcessCollection(job);
          }
          catch(Exception e) {
            duccOut.error(methodName, null, e);
            duccOut.error(methodName, job.getDuccId(), summarize(e), e);
          }
          catch(Throwable t) {
            duccOut.error(methodName, null, t);
          }
        }
        if(active.get()) {
          try {
            if(jpc != null) {
              ConcurrentSkipListMap<Long, JobProcessData> map = jpc.transform(job);
              jpc.exportData(map);
            }
          }
          catch(Exception e) {
            duccOut.error(methodName, job.getDuccId(), summarize(e), e);
          }
        }
      }
    }
    else {

View Full Code Here

  public TreeMap<String,ArrayList<DuccId>> getServiceToJobsUsageMap() {
    TreeMap<String,ArrayList<DuccId>> map = new TreeMap<String,ArrayList<DuccId>>();
    DuccData duccData = DuccData.getInstance();
    ConcurrentSkipListMap<JobInfo, JobInfo> jobs = duccData.getSortedJobs();
    for(JobInfo jobInfo : jobs.descendingKeySet()) {
      DuccWorkJob job = jobInfo.getJob();
      if(job.isOperational()) {
        DuccId duccId = job.getDuccId();
        String[] dependencies = job.getServiceDependencies();
        if(dependencies != null) {
          for(String dependency : dependencies) {
            if(!map.containsKey(dependency)) {
              map.put(dependency, new ArrayList<DuccId>());
            }

View Full Code Here

        }
  }
  
  public DuccWorkJob create(CommonConfiguration common, JobRequestProperties jobRequestProperties) {
    String methodName = "create";
    DuccWorkJob job = new DuccWorkJob();
    checkSpec(job, jobRequestProperties);
    // id, type
    String ddCr = jobRequestProperties.getProperty(JobSpecificationProperties.key_driver_descriptor_CR);
    if(ddCr == null) {
      job.setDuccType(DuccType.Service);
      job.setDuccId(duccIdFactory.next());
    }
    else {
      job.setDuccType(DuccType.Job);
      job.setDuccId(duccIdFactory.next());
      
    }
    // driver
    DuccType duccType = job.getDuccType();
    switch(duccType) {
      case Job:
        createDriver(common, jobRequestProperties, job);
        setDebugPorts(common, jobRequestProperties, job);
        break;
      case Service:
        break;
    }
        // Service Deployment Type
        if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom)) {
      job.setServiceDeploymentType(ServiceDeploymentType.custom);
    }
        else if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_other)) {
      job.setServiceDeploymentType(ServiceDeploymentType.other);
    }
        else if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_uima)) {
      job.setServiceDeploymentType(ServiceDeploymentType.uima);
    }
        else {
          job.setServiceDeploymentType(ServiceDeploymentType.unspecified);
        }
    // sweep out leftover logging trash
    logSweeper(jobRequestProperties.getProperty(JobRequestProperties.key_log_directory), job.getDuccId());
    // log
    jobRequestProperties.specification(logger);
    // classpath
        String java_classpath = getDuccClasspath(1);    // for job processes & services 
    String processClasspath = jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
    logger.debug(methodName, job.getDuccId(), "process CP (spec):"+processClasspath);
    logger.debug(methodName, job.getDuccId(), "java CP:"+java_classpath);
    if(processClasspath != null) {
      if(isClasspathOrderUserBeforeDucc(jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath_order),job.getDuccId())) {
        logger.info(methodName, job.getDuccId(), "process:OrderUserBeforeDucc");
        processClasspath=processClasspath+File.pathSeparator+java_classpath;
      }
      else {
        logger.info(methodName, job.getDuccId(), "process:OrderDuccBeforeUser");
        processClasspath=java_classpath+File.pathSeparator+processClasspath;
      }
    }
    else {
      logger.info(methodName, job.getDuccId(), "process:OrderDefault");
      processClasspath=java_classpath;
    }
    logger.debug(methodName, job.getDuccId(), "process CP (combined):"+processClasspath);
    // java command
    String javaCmd = jobRequestProperties.getProperty(JobSpecificationProperties.key_jvm);
    if(javaCmd == null) {
            // Agent will set javaCmd for Driver and Processes
    }
    // standard info
    DuccStandardInfo standardInfo = new DuccStandardInfo();
    job.setStandardInfo(standardInfo);
    standardInfo.setUser(jobRequestProperties.getProperty(JobSpecificationProperties.key_user));
    standardInfo.setSubmitter(jobRequestProperties.getProperty(JobSpecificationProperties.key_submitter_pid_at_host));
    standardInfo.setDateOfSubmission(TimeStamp.getCurrentMillis());
    standardInfo.setDateOfCompletion(null);
    standardInfo.setDescription(jobRequestProperties.getProperty(JobSpecificationProperties.key_description));
    standardInfo.setLogDirectory(jobRequestProperties.getProperty(JobSpecificationProperties.key_log_directory));
    standardInfo.setWorkingDirectory(jobRequestProperties.getProperty(JobSpecificationProperties.key_working_directory));
    String notifications = jobRequestProperties.getProperty(JobSpecificationProperties.key_notifications);
    if(notifications == null) {
      standardInfo.setNotifications(null);
    }
    else {
      String[] notificationsArray = notifications.split(" ,");
      for(String notification : notificationsArray) {
        notification.trim();
      }
      standardInfo.setNotifications(notificationsArray);
    }
    // scheduling info
    DuccSchedulingInfo schedulingInfo = new DuccSchedulingInfo();
    String ducc_rm_share_quantum = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_rm_share_quantum);
    if(ducc_rm_share_quantum != null) {
      ducc_rm_share_quantum = ducc_rm_share_quantum.trim();
      if(ducc_rm_share_quantum.length() > 0) {
        schedulingInfo.setShareMemorySize(ducc_rm_share_quantum);
      }
    }
    job.setSchedulingInfo(schedulingInfo);
    schedulingInfo.setSchedulingClass(jobRequestProperties.getProperty(JobSpecificationProperties.key_scheduling_class));
    schedulingInfo.setSchedulingPriority(jobRequestProperties.getProperty(JobSpecificationProperties.key_scheduling_priority));
    schedulingInfo.setSharesMax(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_deployments_max));
    schedulingInfo.setSharesMin(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_deployments_min));
    schedulingInfo.setThreadsPerShare(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_thread_count));
    schedulingInfo.setShareMemorySize(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_memory_size));
    schedulingInfo.setShareMemoryUnits(MemoryUnits.GB);
    
    if (job.getDuccType() == DuccType.Job){ 
        checkSchedulingLimits(job, schedulingInfo);
    }
    
    // process_initialization_time_max
    String pi_time = jobRequestProperties.getProperty(JobRequestProperties.key_process_initialization_time_max);
    if(pi_time == null) {
      pi_time = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_agent_launcher_process_init_timeout);
    }
    try {
      long value = Long.parseLong(pi_time)*60*1000;
      standardInfo.setProcessInitializationTimeMax(value);
    }
    catch(Exception e) {
      logger.error(methodName, job.getDuccId(), e);
    }
    // jp
    ServiceDeploymentType serviceDeploymentType = job.getServiceDeploymentType();
    if(isJpUima(duccType, serviceDeploymentType)) {
      String process_DD = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_DD);
      if(process_DD != null) {
        // user DD
        IDuccUimaDeploymentDescriptor uimaDeploymentDescriptor = new DuccUimaDeploymentDescriptor(process_DD);
        job.setUimaDeployableConfiguration(uimaDeploymentDescriptor);
        dump(job, uimaDeploymentDescriptor);
      }
      else {
        // UIMA aggregate
        String name = common.jdQueuePrefix+job.getDuccId().toString();
        String description = job.getStandardInfo().getDescription();
        int threadCount = Integer.parseInt(job.getSchedulingInfo().getThreadsPerShare());
        String brokerURL = job.getjobBroker();;
        String endpoint = job.getjobQueue();
        ArrayList<IDuccUimaAggregateComponent> components = new ArrayList<IDuccUimaAggregateComponent>();
        String CMDescriptor = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CM);
        if(CMDescriptor != null) {
          ArrayList<String> CMOverrides = toArrayList(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CM_overrides));
          IDuccUimaAggregateComponent componentCM = new DuccUimaAggregateComponent(CMDescriptor, CMOverrides);
          components.add(componentCM);
        }
        String AEDescriptor = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_AE);
        if(AEDescriptor != null) {
          ArrayList<String> AEOverrides = toArrayList(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_AE_overrides));
          IDuccUimaAggregateComponent componentAE = new DuccUimaAggregateComponent(AEDescriptor, AEOverrides);
          components.add(componentAE);
        }
        String CCDescriptor = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CC);
        if(CCDescriptor != null) {
          ArrayList<String> CCOverrides = toArrayList(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CC_overrides));
          IDuccUimaAggregateComponent componentCC = new DuccUimaAggregateComponent(CCDescriptor, CCOverrides);
          components.add(componentCC);
        }
        IDuccUimaAggregate uimaAggregate = new DuccUimaAggregate(name,description,threadCount,brokerURL,endpoint,components);
        job.setUimaDeployableConfiguration(uimaAggregate);
        dump(job, uimaAggregate);
      }
      // pipelines
      JavaCommandLine pipelineCommandLine = new JavaCommandLine(javaCmd);
      pipelineCommandLine.setClassName("main:provided-by-Process-Manager");
      pipelineCommandLine.setClasspath(processClasspath);
      String process_jvm_args = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_jvm_args);
      ArrayList<String> pTokens = QuotedOptions.tokenizeList(process_jvm_args, true);
      for(String token : pTokens) {
        pipelineCommandLine.addOption(token);
      }
        // Add any site-provided JVM args
          String siteJvmArgs = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_process_jvm_args);
          pTokens = QuotedOptions.tokenizeList(siteJvmArgs, true);   // a null arg is acceptable
          for(String token : pTokens) {
              pipelineCommandLine.addOption(token);
          }
      
      String processEnvironmentVariables = jobRequestProperties.getProperty(JobSpecificationProperties.key_environment);
      int envCountProcess = addEnvironment(job, "process", pipelineCommandLine, processEnvironmentVariables);
      logger.info(methodName, job.getDuccId(), "process env vars: "+envCountProcess);
      logger.debug(methodName, job.getDuccId(), "pipeline: "+pipelineCommandLine.getCommand());
      pipelineCommandLine.setLogDirectory(jobRequestProperties.getProperty(JobSpecificationProperties.key_log_directory));
      job.setCommandLine(pipelineCommandLine);
    }
    else {
      // ducclet (sometimes known as arbitrary process)
      String process_executable = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_executable);
      NonJavaCommandLine executableProcessCommandLine = new NonJavaCommandLine(process_executable);
      String processEnvironmentVariables = jobRequestProperties.getProperty(JobSpecificationProperties.key_environment);
      int envCountProcess = addEnvironment(job, "process", executableProcessCommandLine, processEnvironmentVariables);
      logger.info(methodName, job.getDuccId(), "process env vars: "+envCountProcess);
      logger.debug(methodName, job.getDuccId(), "ducclet: "+executableProcessCommandLine.getCommandLine());
      job.setCommandLine(executableProcessCommandLine);
      // Tokenize arguments string and strip any quotes, then add to command line.
      // Note: placeholders replaced by CLI so can avoid the add method.
      List<String> process_executable_arguments = QuotedOptions.tokenizeList(
              jobRequestProperties.getProperty(JobSpecificationProperties.key_process_executable_args), true);
      executableProcessCommandLine.getArguments().addAll(process_executable_arguments);
    }
    // process_initialization_failures_cap
    String failures_cap = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_initialization_failures_cap);
    try {
      long process_failures_cap = Long.parseLong(failures_cap);
      if(process_failures_cap > 0) {
        job.setProcessInitFailureCap(process_failures_cap);
      }
      else {
        logger.warn(methodName, job.getDuccId(), "invalid "+JobSpecificationProperties.key_process_initialization_failures_cap+": "+failures_cap);
      }
    }
    catch(Exception e) {
      logger.warn(methodName, job.getDuccId(), "invalid "+JobSpecificationProperties.key_process_initialization_failures_cap+": "+failures_cap);
    }
    // process_failures_limit
    String failures_limit = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_failures_limit);
    try {
      long process_failures_limit = Long.parseLong(failures_limit);
      if(process_failures_limit > 0) {
        job.setProcessFailureLimit(process_failures_limit);
      }
      else {
        logger.warn(methodName, job.getDuccId(), "invalid "+JobSpecificationProperties.key_process_failures_limit+": "+failures_limit);
      }
    }
    catch(Exception e) {
      logger.warn(methodName, job.getDuccId(), "invalid "+JobSpecificationProperties.key_process_failures_limit+": "+failures_limit);
    }
        //
        // Set the service dependency, if there is one.
        //
        String depstr = jobRequestProperties.getProperty(JobSpecificationProperties.key_service_dependency);
        if ( depstr == null ) {            
            logger.debug(methodName, job.getDuccId(), "No service dependencies");
        } else {
            logger.debug(methodName, job.getDuccId(), "Adding service dependency", depstr);
            String[] deps = depstr.split("\\s+");      
            job.setServiceDependencies(deps);
        }
        // Service Endpoint
        String ep = jobRequestProperties.getProperty(ServiceRequestProperties.key_service_request_endpoint);
        if ( ep == null ) {                     
            logger.debug(methodName, job.getDuccId(), "No service endpoint");
        } else {
            logger.debug(methodName, job.getDuccId(), "Adding service endpoint", ep);
            job.setServiceEndpoint(ep);
        }
        // Cancel On Interrupt
        if(jobRequestProperties.containsKey(JobSpecificationProperties.key_cancel_on_interrupt)) {
          job.setCancelOnInterrupt();
        }
        else if(jobRequestProperties.containsKey(ReservationSpecificationProperties.key_cancel_managed_reservation_on_interrupt)) {
          job.setCancelOnInterrupt();
        }
    //TODO be sure to clean-up fpath upon job completion!
    return job;
  }

View Full Code Here

    duccLogger.trace(methodName, jobid, "enter");
    TreeMap<String,ArrayList<String>> map = new TreeMap<String,ArrayList<String>>();
    DuccData duccData = DuccData.getInstance();
    ConcurrentSkipListMap<JobInfo, JobInfo> jobs = duccData.getSortedServices();
    for(JobInfo jobInfo : jobs.descendingKeySet()) {
      DuccWorkJob service = jobInfo.getJob();
      if(service.isOperational()) {
        ServiceDeploymentType type = service.getServiceDeploymentType();
        if(type != null) {
          switch(type) {
          case uima:
          case custom:
            DuccId duccId = service.getDuccId();
            String serviceId = getServiceId(duccId);
            if(serviceId != null) {
              String[] dependencies = service.getServiceDependencies();
              if(dependencies != null) {
                for(String dependency : dependencies) {
                  if(!map.containsKey(dependency)) {
                    map.put(dependency, new ArrayList<String>());
                  }

View Full Code Here

  public TreeMap<String,ArrayList<DuccId>> getServiceToReservationsUsageMap() {
    TreeMap<String,ArrayList<DuccId>> map = new TreeMap<String,ArrayList<DuccId>>();
    DuccData duccData = DuccData.getInstance();
    ConcurrentSkipListMap<JobInfo, JobInfo> services = duccData.getSortedServices();
    for(JobInfo jobInfo : services.descendingKeySet()) {
      DuccWorkJob service = jobInfo.getJob();
      if(service.isOperational()) {
        ServiceDeploymentType type = service.getServiceDeploymentType();
        if(type != null) {
          switch(type) {
          case other:
            DuccId duccId = service.getDuccId();
            String[] dependencies = service.getServiceDependencies();
            if(dependencies != null) {
              for(String dependency : dependencies) {
                if(!map.containsKey(dependency)) {
                  map.put(dependency, new ArrayList<DuccId>());
                }

View Full Code Here

    ArrayList<JobProcessInfo> list = new ArrayList<JobProcessInfo>();
    if(nodeName != null) {
      DuccData duccData = DuccData.getInstance();
      ConcurrentSkipListMap<JobInfo, JobInfo> jobs = duccData.getSortedJobs();
      for(JobInfo jobInfo : jobs.descendingKeySet()) {
        DuccWorkJob job = jobInfo.getJob();
        if(job.isOperational()) {
          DuccId jobid = job.getDuccId();
          IDuccProcessMap map = job.getProcessMap();
          Iterator<DuccId> procids = map.keySet().iterator();
          while(procids.hasNext()) {
            DuccId procid = procids.next();
            IDuccProcess proc = map.get(procid);
            if(!proc.isComplete()) {

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.uima.ducc.transport.event.common.DuccWorkJob

org.apache.uima.ducc.jd.JobDriverComponent

org.apache.uima.ducc.orchestrator.JobFactory

org.apache.uima.ducc.orchestrator.OrchestratorComponent

org.apache.uima.ducc.orchestrator.StateManager

org.apache.uima.ducc.pm.ProcessManagerComponent

org.apache.uima.ducc.sm.ServiceHandler

org.apache.uima.ducc.sm.ServiceManagerComponent

org.apache.uima.ducc.ws.DuccData

org.apache.uima.ducc.ws.DuccDataHelper

org.apache.uima.ducc.ws.Info

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.