Examples of IdenticalPayloadDigestRevisit

Examples of org.archive.modules.revisit.IdenticalPayloadDigestRevisit

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
    @Override
    protected void innerProcess(CrawlURI curi) throws InterruptedException {
        contentDigestHistory.load(curi);

        if (!curi.getContentDigestHistory().isEmpty()) {
          IdenticalPayloadDigestRevisit revisit =
              new IdenticalPayloadDigestRevisit(curi.getContentDigestSchemeString());
      revisit.setRefersToDate((String)curi.getContentDigestHistory().get(A_ORIGINAL_DATE));
      revisit.setRefersToTargetURI((String)curi.getContentDigestHistory().get(A_ORIGINAL_URL));
      String warcRecordId= (String)curi.getContentDigestHistory().get(A_WARC_RECORD_ID);
      if (warcRecordId!=null) {
        revisit.setRefersToRecordID(warcRecordId);
      }
      curi.setRevisitProfile(revisit);
            curi.getAnnotations().add("duplicate:digest");
        }
    }
View Full Code Here

Examples of org.archive.modules.revisit.IdenticalPayloadDigestRevisit

111
112
113
114
115
116
117
118
119
120
121
122
            revisit.setETag((String) latestFetch.get(A_ETAG_HEADER));
            revisit.setLastModified((String) latestFetch.get(A_LAST_MODIFIED_HEADER));
            curi.setRevisitProfile(revisit);
        } else if (hasIdenticalDigest(curi)) {
            curi.getAnnotations().add("duplicate:digest");
            IdenticalPayloadDigestRevisit revisit =
                new IdenticalPayloadDigestRevisit((String)history[1].get(A_CONTENT_DIGEST));
            revisit.setRefersToTargetURI(curi.getURI()); // Matches are always on the same URI
            revisit.setRefersToDate((Long)history[1].get(A_FETCH_BEGAN_TIME));
            curi.setRevisitProfile(revisit);
        }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.