Examples of com.taobao.top.analysis.node.job.JobTask

Package com.taobao.top.analysis.node.job

Examples of com.taobao.top.analysis.node.job.JobTask

com.taobao.top.analysis.node.job.JobTask
任务的真实需要执行的实例，每个计算节点根据描述的输入获取数据，根据规则来分析数据，最后返回结果到输出 @author fangweng @Email fangweng@taobao.com2011-11-24

  public static void main(String[] args) {
    
    
    HdfsInputAdaptor hdfsInputAdaptor = new HdfsInputAdaptor();
    JobConfig jobConfig = new JobConfig();
    JobTask jobtask = new JobTask(jobConfig);
    jobtask.setInput("hdfs://localhost:9000/user/apple/top/top-access.log");
    
    java.io.BufferedReader reader = null;
    InputStream in = null;
    
    try

View Full Code Here

    
    Map<String, Job> jobs = mixJobBuilder.build();
    
    for(Job job : jobs.values())
    {
      JobTask task = job.getJobTasks().get(0);
      
      job.setJobResult(defaultAnalysisEngine.doAnalysis(task).getResults());  
      
      fileJobExporter.exportReport(job, false);
    }

View Full Code Here

    
    Map<String, Job> jobs = mixJobBuilder.build();
    
    for(Job job : jobs.values())
    {
      JobTask task = job.getJobTasks().get(0);
      
      JobTaskResult jobTaskResult = defaultAnalysisEngine.doAnalysis(task);  
      
      fileJobExporter.exportReport(task, jobTaskResult, false);
    }

View Full Code Here

    fileJobExporter.init();
    
    Map<String, Job> jobs = mixJobBuilder.build();
    Job job = jobs.values().iterator().next();
    
    JobTask task = job.getJobTasks().get(0);
    
    job.setJobResult(defaultAnalysisEngine.doAnalysis(task).getResults());
    
    fileJobExporter.exportEntryData(job);

View Full Code Here

    defaultAnalysisEngine.addInputAdaptor(fileInputAdaptor);
    defaultAnalysisEngine.addInputAdaptor(httpInputAdaptor);
    
    SendResultsRequestEvent jobResponseEvent = new SendResultsRequestEvent("1234");
  
    JobTask task = jobManager.getJobs().values().iterator().next().getJobTasks().get(0);
    
    JobTaskResult jobTaskResult = defaultAnalysisEngine.doAnalysis(task);  
  
    jobResponseEvent.setJobTaskResult(jobTaskResult);
    
    jobManager.addTaskResultToQueue(jobResponseEvent);
    
    
    JobTaskResult jobTaskResult2 = jobManager.getJobTaskResultsQueuePool().get(task.getJobName()).poll();
    
    Assert.assertEquals(jobTaskResult, jobTaskResult2);
    
    SendResultsResponseEvent sendResultsResponseEvent = (SendResultsResponseEvent)tunnel.getSlaveSide().poll();

View Full Code Here

      throw new AnalysisException("generateJobTasks error, jobConfig is null.");
    
    //允许定义多个job通过逗号分割
    if (jobConfig.getInputParams() == null && jobConfig.getInput().indexOf(",") <= 0)
    {
      JobTask jobTask = new JobTask(jobConfig);
      jobTask.setStatisticsRule(job.getStatisticsRule());
      jobTask.setJobName(job.getJobName());
      jobTask.setUrl(jobTask.getInput());
      jobTask.setJobSourceTimeStamp(job.getJobSourceTimeStamp());
      
      jobTask.setInput(generateJobInputAddition(jobTask.getInput(),job));
      jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
      
      /**
       * 目前使用master游标管理方式的只有hub
       */
      Long begin = jobConfig.getBegin();
      if(begin == null)
          begin = 0L;
      if(jobTask.getUrl().startsWith("hub://")) {
          String key = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
          job.getCursorMap().putIfAbsent(key, begin);
          job.getTimestampMap().putIfAbsent(key, -1L);
          jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(key));
      }
      job.addTaskCount();
      jobTask.getTailCursor().set(jobConfig.getInit());
      job.getJobTasks().add(jobTask);
    }
    else
    {
      if (jobConfig.getInputParams() != null)
      {
        String[] p = StringUtils.split(jobConfig.getInputParams(),":");
        String key = new StringBuilder("$").append(p[0]).append("$").toString();
        
        if (p.length != 2 || jobConfig.getInput().indexOf(key) < 0)
          throw new AnalysisException("inputParams invalidate : " + jobConfig.getInputParams());
        
        String[] params = StringUtils.split(p[1],",");
        
        for(String ps : params)
        {
          JobTask jobTask = new JobTask(jobConfig);
          jobTask.setStatisticsRule(job.getStatisticsRule());
          jobTask.setJobName(job.getJobName());
          jobTask.setUrl(jobConfig.getInput().replace(key, ps));
          jobTask.setJobSourceTimeStamp(job.getJobSourceTimeStamp());
          jobTask.setInput(generateJobInputAddition(jobConfig.getInput().replace(key, ps),job));
          jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
          /**
                 * 目前使用master游标管理方式的只有hub
                 */
          Long begin = jobConfig.getBegin();
                if(begin == null)
                    begin = 0L;
                if(jobTask.getUrl().startsWith("hub://")) {
                    String keyU = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
                    job.getCursorMap().putIfAbsent(keyU, begin);
                    job.getTimestampMap().putIfAbsent(keyU, -1L);
                    jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(keyU));
                }
                jobTask.getTailCursor().set(jobConfig.getInit());
          job.addTaskCount();
          job.getJobTasks().add(jobTask);
        }
      }
      else
      {
        String[] inputs = StringUtils.split(jobConfig.getInput(),",");
        
        for(String input : inputs)
        {
          JobTask jobTask = new JobTask(jobConfig);
          jobTask.setStatisticsRule(job.getStatisticsRule());
          jobTask.setJobName(job.getJobName());
          jobTask.setInput(generateJobInputAddition(input,job));
          jobTask.setUrl(input);
          jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
                    /**
                     * 目前使用master游标管理方式的只有hub
                     */
                    Long begin = jobConfig.getBegin();
                    if(begin == null)
                        begin = 0L;
                    if(jobTask.getUrl().startsWith("hub://")) {
                        String key = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
                        job.getCursorMap().putIfAbsent(key, begin);
                        job.getTimestampMap().putIfAbsent(key, -1L);
                        jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(key));
                    }
                    jobTask.getTailCursor().set(jobConfig.getInit());
          job.addTaskCount();
          job.getJobTasks().add(jobTask);
        }
        
      }

View Full Code Here

            Iterator<JobTask> taskIter = undoTaskQueue.iterator();


            while (taskIter.hasNext()) {
//                String taskId = taskIds.next();
//                JobTask jobTask = jobTaskPool.get(taskId);
                JobTask jobTask = taskIter.next();
                if (!jobTaskPool.keySet().contains(jobTask.getTaskId())
                        || jobs.get(jobTask.getJobName()).getEpoch().get() > jobTask.getJobEpoch()
                        || jobs.get(jobTask.getJobName()).getJobTimeOut().get()) {
                    taskIter.remove();
                    continue;
                }
                
                if (jobs.get(jobTask.getJobName()).getJobConfig().getSlaveIpCondition() != null) {
                    try {
                        Channel channel = (Channel) requestEvent.getChannel();
                        if (!channel.getRemoteAddress().toString()
                            .matches(jobs.get(jobTask.getJobName()).getJobConfig().getSlaveIpCondition())) {
                            continue;
                        }
                    }
                    catch (Throwable e) {
                        logger.error(e);
                    }
                }


                if (statusPool.get(jobTask.getTaskId()).equals(JobTaskStatus.UNDO)) {
                    if (statusPool.replace(jobTask.getTaskId(), JobTaskStatus.UNDO, JobTaskStatus.DOING)) {
                        this.allocateTask(jobTask);
                        jobTasks.add(jobTask);
                        taskIter.remove();


                        if (jobTasks.size() >= jobCount)

View Full Code Here

            }


            Iterator<String> iter = jobTaskResult.getTaskIds().iterator();
            while (iter.hasNext()) {
                String taskId = iter.next();
                JobTask jobTask = jobTaskPool.get(taskId);
                
                if (jobTask == null)
                {     
                  logger.error(new StringBuilder("taskId :").append(taskId).append("not exist!").toString());
                  continue;
                }
                
                Job job = jobs.get(jobTask.getJobName());
                if(job == null) {
                    logger.error(new StringBuilder("job :").append(jobTask.getJobName()).append("not exist!").toString());
                    continue;
                }


                if (statusPool.replace(taskId, JobTaskStatus.DOING, JobTaskStatus.DONE)
                        || statusPool.replace(taskId, JobTaskStatus.UNDO, JobTaskStatus.DONE)) {
                    logger.info("task " + jobTask.getJobName() + " of job " + job.getJobName() + " done");
                    jobTask.setStatus(JobTaskStatus.DONE);
                    jobTask.getTailCursor().compareAndSet(true, false);
                    jobTask.setEndTime(System.currentTimeMillis());
                    jobTask.setLastMergedEpoch(job.getEpoch().get());
                    job.getCompletedTaskCount().incrementAndGet();
                } else {
                    if(!this.config.getDispatchMaster()) {
                        jobTask.setStatus(JobTaskStatus.DONE);
                        jobTask.getTailCursor().compareAndSet(true, false);
                        jobTask.setEndTime(System.currentTimeMillis());
                        jobTask.setLastMergedEpoch(job.getEpoch().get());
                        statusPool.put(taskId, JobTaskStatus.DONE);
                        iter.remove();
                    }
                }
                
                //对jobTask的执行结果打点
                StringBuilder log = new StringBuilder(ReportUtil.SLAVE_LOG).append(",timeStamp=")
                          .append(System.currentTimeMillis()).append(",epoch=")
                          .append(job.getEpoch()).append(",jobName=");
                log.append(jobTask.getJobName()).append(",taskId=")
                  .append(jobTask.getTaskId()).append(",recycleCounter=")
                  .append(jobTask.getRecycleCounter().get()).append(",slaveIp=")
                  .append(jobTaskResult.getSlaveIp()).append(",efficiency=")
                  .append(jobTaskResult.getEfficiency()).append(",");
               
                JobTaskExecuteInfo executeInfo = jobTaskResult.getTaskExecuteInfos().get(jobTask.getTaskId());
                
                if (executeInfo != null) {
                    log.append("analysisConsume=").append(executeInfo.getAnalysisConsume()).append(",")
                        .append("jobDataSize=").append(executeInfo.getJobDataSize()).append(",").append("totalLine=")
                        .append(executeInfo.getTotalLine()).append(",").append("errorLine=")
                        .append(executeInfo.getErrorLine()).append(",").append("emptyLine=")
                        .append(executeInfo.getEmptyLine()).append(",fileBegin=").append(executeInfo.getFileBegin())
                        .append(",fileLength=").append(executeInfo.getFileLength());
                    if(jobTask.getInput().startsWith("hub:")) {
                        jobTask.setJobSourceTimeStamp(executeInfo.getTimestamp());
                        job.updateCursor(jobTask.getUrl(), executeInfo.getFileBegin(), executeInfo.getFileLength(), executeInfo.getTimestamp());
                    }
                }
                else
                  logger.error(new StringBuilder().append("taskId : ").
                      append(jobTask.getTaskId()).append(" executeInfo is null!").toString());
                
                ReportUtil.clusterLog(log.toString());
                
                
                //增加一块对于zookeeper的支持
            if (StringUtils.isNotEmpty(config.getZkServer()) && zk != null)
            {
              try
              {             
                ZKUtil.updateOrCreateNode(zk,new StringBuilder()
                      .append(ZKUtil.getGroupMasterZKPath(config.getGroupId()))
                      .append("/").append(config.getMasterName())
                      .append("/runtime/").append(job.getEpoch())
                      .append("/").append(jobTask.getJobName())
                      .append("/").append(jobTask.getTaskId()).toString(),log.toString().getBytes("UTF-8"));
                
              }
              catch(Exception ex)
              {
                logger.error("log to zk error!",ex);

View Full Code Here


        while (taskIds.hasNext()) {
            String taskId = taskIds.next();


            JobTaskStatus taskStatus = statusPool.get(taskId);
            JobTask jobTask = jobTaskPool.get(taskId);


            if (taskStatus == JobTaskStatus.DOING && jobTask.getStartTime() != 0
                    && System.currentTimeMillis() - jobTask.getStartTime() >= jobTask.getTaskRecycleTime() * 1000) {
                if (statusPool.replace(taskId, JobTaskStatus.DOING, JobTaskStatus.UNDO)) {
                    jobTask.setStatus(JobTaskStatus.UNDO);
                    undoTaskQueue.offer(jobTask);
                    jobTask.getRecycleCounter().incrementAndGet();


                    if (logger.isWarnEnabled())
                        logger.warn("Task : " + jobTask.getTaskId() + " can't complete in time, it be recycle.");
                }
            }
        }
    }

View Full Code Here

          final List<JobTaskResult> taskResults = new ArrayList<JobTaskResult>();
          
          //同一个Job的多个Task并行执行
          for(final JobTask jobtask : jobTasks)
          {
            final JobTask j = jobtask;
            
            analysisWorkerThreadPool.execute(
                new Runnable()
                {
                  public void run()

View Full Code Here

TOP

Related Classes of com.taobao.top.analysis.node.job.JobTask

com.taobao.top.analysis.HdfsInputAdaptorTest

com.taobao.top.analysis.node.component.FileJobBuilder

com.taobao.top.analysis.node.component.FileJobExporterTest

com.taobao.top.analysis.node.component.JobManager

com.taobao.top.analysis.node.component.JobManagerTest

com.taobao.top.analysis.node.component.SlaveNode$BundleTasksExecutable

java.util.concurrent.atomic.AtomicInteger

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.