Package org.apache.oodt.cas.crawl

Examples of org.apache.oodt.cas.crawl.AutoDetectProductCrawler


     /* create a ProductCrawler based on whether or not the output dir specifies a MIME_EXTRACTOR_REPO */
      logger.info("Configuring ProductCrawler...");
      ProductCrawler crawler = null;
      if (pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO) != null &&
          pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO).equals("")){
          crawler = new AutoDetectProductCrawler();
          ((AutoDetectProductCrawler)crawler).
            setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));       
      }
      else{
        crawler = new StdProductCrawler();
View Full Code Here


      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(true);
      replay(pgeTask.wm);

      AutoDetectProductCrawler pc = createMock(AutoDetectProductCrawler.class);
      pc.crawl(new File("/tmp/dir1"));
      pc.crawl(new File("/tmp/dir2"));
      expect(pc.getIngestStatus()).andReturn(Collections.<IngestStatus>emptyList());
      replay(pc);

      pgeTask.runIngestCrawler(pc);

      verify(pgeTask.wm);
      verify(pc);

      // Case: UpdateStatus Fail
      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(false);
      replay(pgeTask.wm);

      pc = createMock(AutoDetectProductCrawler.class);
      replay(pc);

      try {
         pgeTask.runIngestCrawler(pc);
         fail("Should have thrown");
      } catch (Exception e) { /* expect throw */ }

      verify(pgeTask.wm);
      verify(pc);

      // Case: UpdateStatus Success, VerifyIngest Fail
      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(true);
      replay(pgeTask.wm);

      pc = createMock(AutoDetectProductCrawler.class);
      pc.crawl(new File("/tmp/dir1"));
      pc.crawl(new File("/tmp/dir2"));
      IngestStatus failedIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Failure";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.FAILURE;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(failedIngestStatus));
      replay(pc);

      try {
         pgeTask.runIngestCrawler(pc);
View Full Code Here

   public void testVerifyIngests() throws Exception {
      PGETaskInstance pgeTask = createTestInstance();

      // Test case failure.
      AutoDetectProductCrawler pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus failedIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Failure";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.FAILURE;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(failedIngestStatus));
      replay(pc);

      try {
         pgeTask.verifyIngests(pc);
         fail("Should have thrown");
      } catch (Exception e) { /* expect throw */ }

      verify(pc);

      // Test case warn failure of precondition, but success overall.
      pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus precondsFailIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Preconditions failed";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.PRECONDS_FAILED;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(precondsFailIngestStatus));
      replay(pc);

      pgeTask.logger = createMock(Logger.class);
      pgeTask.logger.info("Verifying ingests successful...");
      pgeTask.logger.warning(
            "Product was not ingested [file='/tmp/dir1',result='PRECONDS_FAILED',msg='Preconditions failed']");
      pgeTask.logger.info("Ingests were successful");
      replay(pgeTask.logger);

      pgeTask.verifyIngests(pc);

      verify(pc);
      verify(pgeTask.logger);

      // Test case success.
      pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus successIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Success";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.SUCCESS;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(successIngestStatus));
      replay(pc);

      pgeTask.logger = createMock(Logger.class);
      pgeTask.logger.info("Verifying ingests successful...");
View Full Code Here

      return returnCode == 0;
   }

   protected ProductCrawler createProductCrawler() throws Exception {
      logger.info("Configuring ProductCrawler...");
      AutoDetectProductCrawler crawler = new AutoDetectProductCrawler();
      crawler.setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
      crawler.setClientTransferer(pgeMetadata
            .getMetadata(INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
      crawler.setFilemgrUrl(pgeMetadata.getMetadata(INGEST_FILE_MANAGER_URL));
      String crawlerConfigFile = pgeMetadata.getMetadata(CRAWLER_CONFIG_FILE);
      if (!Strings.isNullOrEmpty(crawlerConfigFile)) {
         crawler.setApplicationContext(
               new FileSystemXmlApplicationContext(crawlerConfigFile));
         List<String> actionIds = pgeMetadata.getAllMetadata(ACTION_IDS);
         if (actionIds != null) {
            crawler.setActionIds(actionIds);
         }
      }
      crawler.setRequiredMetadata(pgeMetadata.getAllMetadata(REQUIRED_METADATA));
      crawler.setCrawlForDirs(Boolean.parseBoolean(pgeMetadata
            .getMetadata(CRAWLER_CRAWL_FOR_DIRS)));
      crawler.setNoRecur(!Boolean.parseBoolean(
            pgeMetadata.getMetadata(CRAWLER_RECUR)));
      logger.fine(
            "Passing Workflow Metadata to CAS-Crawler as global metadata . . .");
      crawler.setGlobalMetadata(pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
      logger.fine("Created ProductCrawler ["
            + crawler.getClass().getCanonicalName() + "]");
      return crawler;
   }
View Full Code Here

      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(true);
      replay(pgeTask.wm);

      AutoDetectProductCrawler pc = createMock(AutoDetectProductCrawler.class);
      pc.crawl(new File("/tmp/dir1"));
      pc.crawl(new File("/tmp/dir2"));
      expect(pc.getIngestStatus()).andReturn(Collections.<IngestStatus>emptyList());
      replay(pc);

      pgeTask.runIngestCrawler(pc);

      verify(pgeTask.wm);
      verify(pc);

      // Case: UpdateStatus Fail
      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(false);
      replay(pgeTask.wm);

      pc = createMock(AutoDetectProductCrawler.class);
      replay(pc);

      try {
         pgeTask.runIngestCrawler(pc);
         fail("Should have thrown");
      } catch (Exception e) { /* expect throw */ }

      verify(pgeTask.wm);
      verify(pc);

      // Case: UpdateStatus Success, VerifyIngest Fail
      pgeTask.wm = createMock(XmlRpcWorkflowManagerClient.class);
      expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
            CRAWLING.getWorkflowStatusName())).andReturn(true);
      replay(pgeTask.wm);

      pc = createMock(AutoDetectProductCrawler.class);
      pc.crawl(new File("/tmp/dir1"));
      pc.crawl(new File("/tmp/dir2"));
      IngestStatus failedIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Failure";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.FAILURE;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(failedIngestStatus));
      replay(pc);

      try {
         pgeTask.runIngestCrawler(pc);
View Full Code Here

   public void testVerifyIngests() throws Exception {
      PGETaskInstance pgeTask = createTestInstance();

      // Test case failure.
      AutoDetectProductCrawler pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus failedIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Failure";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.FAILURE;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(failedIngestStatus));
      replay(pc);

      try {
         pgeTask.verifyIngests(pc);
         fail("Should have thrown");
      } catch (Exception e) { /* expect throw */ }

      verify(pc);

      // Test case warn failure of precondition, but success overall.
      pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus precondsFailIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Preconditions failed";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.PRECONDS_FAILED;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(precondsFailIngestStatus));
      replay(pc);

      pgeTask.logger = createMock(Logger.class);
      pgeTask.logger.info("Verifying ingests successful...");
      pgeTask.logger.warning(
            "Product was not ingested [file='/tmp/dir1',result='PRECONDS_FAILED',msg='Preconditions failed']");
      pgeTask.logger.info("Ingests were successful");
      replay(pgeTask.logger);

      pgeTask.verifyIngests(pc);

      verify(pc);
      verify(pgeTask.logger);

      // Test case success.
      pc = createMock(AutoDetectProductCrawler.class);
      IngestStatus successIngestStatus = new IngestStatus() {
         @Override
         public String getMessage() {
            return "Ingest Success";
         }
         @Override
         public File getProduct() {
            return new File("/tmp/dir1");
         }
         @Override
         public Result getResult() {
            return Result.SUCCESS;
         }
      };
      expect(pc.getIngestStatus()).andReturn(
            Lists.newArrayList(successIngestStatus));
      replay(pc);

      pgeTask.logger = createMock(Logger.class);
      pgeTask.logger.info("Verifying ingests successful...");
View Full Code Here

      return returnCode == 0;
   }

   protected ProductCrawler createProductCrawler() throws Exception {
      logger.info("Configuring ProductCrawler...");
      AutoDetectProductCrawler crawler = new AutoDetectProductCrawler();
      crawler.setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
      crawler.setClientTransferer(pgeMetadata
            .getMetadata(INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
      crawler.setFilemgrUrl(pgeMetadata.getMetadata(INGEST_FILE_MANAGER_URL));
      String crawlerConfigFile = pgeMetadata.getMetadata(CRAWLER_CONFIG_FILE);
      if (!Strings.isNullOrEmpty(crawlerConfigFile)) {
         crawler.setApplicationContext(
               new FileSystemXmlApplicationContext(crawlerConfigFile));
         List<String> actionIds = pgeMetadata.getAllMetadata(ACTION_IDS);
         if (actionIds != null) {
            crawler.setActionIds(actionIds);
         }
      }
      crawler.setRequiredMetadata(pgeMetadata.getAllMetadata(REQUIRED_METADATA));
      crawler.setCrawlForDirs(Boolean.parseBoolean(pgeMetadata
            .getMetadata(CRAWLER_CRAWL_FOR_DIRS)));
      crawler.setNoRecur(!Boolean.parseBoolean(
            pgeMetadata.getMetadata(CRAWLER_RECUR)));
      logger.fine(
            "Passing Workflow Metadata to CAS-Crawler as global metadata . . .");
      crawler.setGlobalMetadata(pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
      logger.fine("Created ProductCrawler ["
            + crawler.getClass().getCanonicalName() + "]");
      return crawler;
   }
View Full Code Here

TOP

Related Classes of org.apache.oodt.cas.crawl.AutoDetectProductCrawler

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.