Examples of org.archive.crawler.framework.CrawlJob


//    @Override
    protected void waitForCrawlFinish() throws Exception {


        Thread.sleep(2000);
        CrawlJob cj = heritrix.getEngine().getJob("selftest-job");
        
        // TODO: pause, checkpoint, launch new job, yadda yadda
        
        // for now, just kill & wait
        cj.terminate();
        super.waitForCrawlFinish();
        
//        invokeAndWait("basic", "requestCrawlPause", CrawlStatus.PAUSED);
//        invokeAndWait("basic", "requestCrawlCheckpoint", CrawlStatus.PAUSED);
//        invokeAndWait("basic", "requestCrawlStop", CrawlStatus.FINISHED);

                return cj1.getValue().compareTo(cj2.getValue());
            }
        });
        
        for(Map.Entry<String,CrawlJob> jobConfig : jobConfigurations) {
            CrawlJob job = jobConfig.getValue();
            Map<String, Object> crawlJobModel = new LinkedHashMap<String, Object>();
            crawlJobModel.put("shortName",job.getShortName());
            crawlJobModel.put("url",urlBaseRef+"job/"+job.getShortName());
            crawlJobModel.put("isProfile",job.isProfile());
            crawlJobModel.put("launchCount",job.getLaunchCount());
            crawlJobModel.put("lastLaunch",job.getLastLaunch());
            crawlJobModel.put("hasApplicationContext",job.hasApplicationContext());
            crawlJobModel.put("statusDescription", job.getJobStatusDescription());
            crawlJobModel.put("isLaunchInfoPartial", job.isLaunchInfoPartial());
            File primaryConfig = FileUtils.tryToCanonicalize(job.getPrimaryConfig());
            crawlJobModel.put("primaryConfig", primaryConfig.getAbsolutePath());
            crawlJobModel.put("primaryConfigUrl", urlBaseRef + "jobdir/" + primaryConfig.getName());
            if (job.getCrawlController() != null) {
                crawlJobModel.put("crawlControllerState", job.getCrawlController().getState());
                if (job.getCrawlController().getState() == State.FINISHED) {
                    crawlJobModel.put("crawlExitStatus", job.getCrawlController().getCrawlExitStatus());
                }
            }
            
            crawlJobModel.put("key", jobConfig.getKey());
            jobList.add(crawlJobModel);

Examples of org.archive.crawler.framework.CrawlJob

Related Classes of org.archive.crawler.framework.CrawlJob