Examples of com.taobao.top.analysis.config.JobConfig

com.taobao.top.analysis.config.JobConfig
任务的配置类 @author fangweng

   */
  public static void main(String[] args) {
    
    
    HdfsInputAdaptor hdfsInputAdaptor = new HdfsInputAdaptor();
    JobConfig jobConfig = new JobConfig();
    JobTask jobtask = new JobTask(jobConfig);
    jobtask.setInput("hdfs://localhost:9000/user/apple/top/top-access.log");
    
    java.io.BufferedReader reader = null;
    InputStream in = null;

View Full Code Here

        for(String j : instances)
        {
                    try {
                        Job job = new Job();
                        Rule rule = new Rule();
                        JobConfig jobconfig = new JobConfig();
                        job.setStatisticsRule(rule);
                        job.setJobConfig(jobconfig);
                        job.setJobName(j);


                        getConfigFromProps(j, jobconfig, prop);


                        if (jobconfig.getReportConfigs() == null
                                || (jobconfig.getReportConfigs() != null && jobconfig.getReportConfigs().length == 0)) {
                            throw new AnalysisException("job Config files should not be null!");
                        }


                        buildRule(jobconfig.getReportConfigs(), rule);
                        
                        //增加一个获得当前临时文件数据源游标的操作
//                        JobDataOperation jobDataOperation = new JobDataOperation(job,
//                            AnalysisConstants.JOBMANAGER_EVENT_LOADDATA,this.config);
//                        jobDataOperation.run();
                        JobDataOperation.getSourceTimeStamp(job, this.config);
//                        JobDataOperation.loadDataToTmp(job, this.config);
//                        JobDataOperation.loadData(job, this.config);
                        
                        buildTasks(job);
                        jobs.put(job.getJobName(), job);
                        this.jobConfigs.put(job.getJobName(),
                            new JobResource(job.getJobName(), jobconfig.getReportConfigs()));
                        if (job.getJobConfig().getSaveTmpResultToFile() == null && this.config != null)
                            job.getJobConfig().setSaveTmpResultToFile(
                                String.valueOf(this.config.getSaveTmpResultToFile()));
                        if (job.getJobConfig().getAsynLoadDiskFilePrecent() < 0 && this.config != null)
                            job.getJobConfig().setAsynLoadDiskFilePrecent(

View Full Code Here

  public void buildTasks(Job job) throws AnalysisException {
    
    if (job.getJobTasks() != null)
      job.getJobTasks().clear();
    
    JobConfig jobConfig = job.getJobConfig();
    
    if (jobConfig == null)
      throw new AnalysisException("generateJobTasks error, jobConfig is null.");
    
    //允许定义多个job通过逗号分割
    if (jobConfig.getInputParams() == null && jobConfig.getInput().indexOf(",") <= 0)
    {
      JobTask jobTask = new JobTask(jobConfig);
      jobTask.setStatisticsRule(job.getStatisticsRule());
      jobTask.setJobName(job.getJobName());
      jobTask.setUrl(jobTask.getInput());
      jobTask.setJobSourceTimeStamp(job.getJobSourceTimeStamp());
      
      jobTask.setInput(generateJobInputAddition(jobTask.getInput(),job));
      jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
      
      /**
       * 目前使用master游标管理方式的只有hub
       */
      Long begin = jobConfig.getBegin();
      if(begin == null)
          begin = 0L;
      if(jobTask.getUrl().startsWith("hub://")) {
          String key = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
          job.getCursorMap().putIfAbsent(key, begin);
          job.getTimestampMap().putIfAbsent(key, -1L);
          jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(key));
      }
      job.addTaskCount();
      jobTask.getTailCursor().set(jobConfig.getInit());
      job.getJobTasks().add(jobTask);
    }
    else
    {
      if (jobConfig.getInputParams() != null)
      {
        String[] p = StringUtils.split(jobConfig.getInputParams(),":");
        String key = new StringBuilder("$").append(p[0]).append("$").toString();
        
        if (p.length != 2 || jobConfig.getInput().indexOf(key) < 0)
          throw new AnalysisException("inputParams invalidate : " + jobConfig.getInputParams());
        
        String[] params = StringUtils.split(p[1],",");
        
        for(String ps : params)
        {
          JobTask jobTask = new JobTask(jobConfig);
          jobTask.setStatisticsRule(job.getStatisticsRule());
          jobTask.setJobName(job.getJobName());
          jobTask.setUrl(jobConfig.getInput().replace(key, ps));
          jobTask.setJobSourceTimeStamp(job.getJobSourceTimeStamp());
          jobTask.setInput(generateJobInputAddition(jobConfig.getInput().replace(key, ps),job));
          jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
          /**
                 * 目前使用master游标管理方式的只有hub
                 */
          Long begin = jobConfig.getBegin();
                if(begin == null)
                    begin = 0L;
                if(jobTask.getUrl().startsWith("hub://")) {
                    String keyU = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
                    job.getCursorMap().putIfAbsent(keyU, begin);
                    job.getTimestampMap().putIfAbsent(keyU, -1L);
                    jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(keyU));
                }
                jobTask.getTailCursor().set(jobConfig.getInit());
          job.addTaskCount();
          job.getJobTasks().add(jobTask);
        }
      }
      else
      {
        String[] inputs = StringUtils.split(jobConfig.getInput(),",");
        
        for(String input : inputs)
        {
          JobTask jobTask = new JobTask(jobConfig);
          jobTask.setStatisticsRule(job.getStatisticsRule());
          jobTask.setJobName(job.getJobName());
          jobTask.setInput(generateJobInputAddition(input,job));
          jobTask.setUrl(input);
          jobTask.setTaskId(getTaskIdFromUrl(job.getJobName(), jobTask.getUrl(), job.getTaskCount()));
                    /**
                     * 目前使用master游标管理方式的只有hub
                     */
                    Long begin = jobConfig.getBegin();
                    if(begin == null)
                        begin = 0L;
                    if(jobTask.getUrl().startsWith("hub://")) {
                        String key = jobTask.getUrl().substring(0, jobTask.getUrl().indexOf('?'));
                        job.getCursorMap().putIfAbsent(key, begin);
                        job.getTimestampMap().putIfAbsent(key, -1L);
                        jobTask.setJobSourceTimeStamp(job.getTimestampMap().get(key));
                    }
                    jobTask.getTailCursor().set(jobConfig.getInit());
          job.addTaskCount();
          job.getJobTasks().add(jobTask);
        }
        
      }

View Full Code Here

TOP

Related Classes of com.taobao.top.analysis.config.JobConfig

com.taobao.top.analysis.HdfsInputAdaptorTest

com.taobao.top.analysis.node.component.FileJobBuilder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.