Examples of org.archive.modules.recrawl.FetchHistoryProcessor

org.archive.modules.recrawl.FetchHistoryProcessor
Maintain a history of fetch information inside the CrawlURI's attributes. @author gojomo @version $Date: 2006-09-25 20:19:54 +0000 (Mon, 25 Sep 2006) $, $Revision: 4654 $

        assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since: "));
        assertTrue(curi.getHttpResponseHeader("last-modified").equals("Thu, 01 Jan 1970 00:00:00 GMT"));
        runDefaultChecks(curi, "requestLine");


        // logger.info("before FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory()));
        FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
        fetchHistoryProcessor.process(curi);
        // logger.info("after FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory()));


        fetcher().process(curi);
        // logger.info("\n" + httpRequestString(curi));
        // logger.info("\n" + rawResponseString(curi));
        assertTrue(httpRequestString(curi).contains("If-Modified-Since: Thu, 01 Jan 1970 00:00:00 GMT\r\n"));
        assertTrue(curi.getFetchStatus() == 304);


        assertNull(curi.getRevisitProfile());
        fetchHistoryProcessor.process(curi);
        assertNotNull(curi.getRevisitProfile());
        assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit);
        ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile();
        assertEquals("Thu, 01 Jan 1970 00:00:00 GMT", revisit.getLastModified());
        assertNull(revisit.getETag());

View Full Code Here

        fetcher().process(curi);
        assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match: "));
        assertTrue(curi.getHttpResponseHeader("etag").equals(ETAG_TEST_VALUE));
        runDefaultChecks(curi, "requestLine");


        FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
        fetchHistoryProcessor.process(curi);


        fetcher().process(curi);
        // logger.info("\n" + httpRequestString(curi));
        // logger.info("\n" + rawResponseString(curi));
        assertTrue(httpRequestString(curi).contains("If-None-Match: " + ETAG_TEST_VALUE + "\r\n"));
        
        assertNull(curi.getRevisitProfile());
        fetchHistoryProcessor.process(curi);
        assertNotNull(curi.getRevisitProfile());
        assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit);
        ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile();
        assertEquals(ETAG_TEST_VALUE, revisit.getETag());
        assertNull(revisit.getLastModified());

View Full Code Here

    curi.setFetchStatus(200);
    curi.setFetchBeginTime(System.currentTimeMillis());
    // FetchHistoryProcessor once failed for a revisit case. We'd need to test other cases
    // too (TODO).
    curi.setContentDigest("sha1", digestValue0);
    FetchHistoryProcessor fhp = new FetchHistoryProcessor();
    fhp.process(curi);
  }

View Full Code Here

TOP

Related Classes of org.archive.modules.recrawl.FetchHistoryProcessor

org.archive.modules.CrawlURI

org.archive.modules.fetcher.FetchHTTPTests

org.archive.modules.recrawl.wbm.WbmPersistLoadProcessorTest

org.archive.modules.revisit.IdenticalPayloadDigestRevisit

org.archive.modules.revisit.ServerNotModifiedRevisit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.