Package org.archive.net

Examples of org.archive.net.UURI

91
92
93
94
95
96
97
98
99
100
101
        }
    }

    public void testCandidateURIWithLoadedAList()
            throws URIException {
        UURI uuri = UURIFactory.getInstance("http://www.archive.org");
        CrawlURI curi = new CrawlURI(uuri);
        curi.setSeed(true);
        curi.getData().put("key", "value");
        assertTrue("Didn't find AList item",
                curi.getData().get("key").equals("value"));
View Full Code Here
403
404
405
406
407
408
409
410
411
412
413
                            newURL = newURL.substring(0,newURL.length()-1);
                        }

                        // And add the URL to speculative embeds.
                        numberOfLinksExtracted.incrementAndGet();
                        UURI dest = UURIFactory.getInstance(newURL);
                        LinkContext lc = LinkContext.SPECULATIVE_MISC;
                        Hop hop = Hop.SPECULATIVE;
                        addOutlink(curi,  dest, lc, hop);
                    }
                    // Reset lookat for next string.
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
   
    @Override
    protected Collection<TestData> makeData(String content, String destURI)
    throws Exception {
        List<TestData> result = new ArrayList<TestData>();
        UURI src = UURIFactory.getInstance("http://www.archive.org/start/");
        CrawlURI euri = new CrawlURI(src, null, null,
                LinkContext.NAVLINK_MISC);
        Recorder recorder = createRecorder(content, "UTF-8");
        euri.setContentType("text/html");
        euri.setRecorder(recorder);
        euri.setContentSize(content.length());
               
        UURI dest = UURIFactory.getInstance(destURI);
        LinkContext context = determineContext(content);
        Hop hop = determineHop(content);
        CrawlURI link = euri.createCrawlURI(dest, context, hop);
        result.add(new TestData(euri, link));
       
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
   
    @Override
    protected Collection<TestData> makeData(String content, String destURI)
    throws Exception {
        List<TestData> result = new ArrayList<TestData>();
        UURI src = UURIFactory.getInstance("http://www.archive.org/foo/dummy.js");
        CrawlURI euri = new CrawlURI(src, null, src, LinkContext.NAVLINK_MISC);
        Recorder recorder = createRecorder(content, "UTF-8");
        euri.setContentType("text/javascript");
        euri.setRecorder(recorder);
        euri.setContentSize(content.length());

        if (destURI != null) {
            UURI dest = UURIFactory.getInstance(destURI);
            CrawlURI link = euri.createCrawlURI(dest, LinkContext.JS_MISC, Hop.SPECULATIVE);
            result.add(new TestData(euri, link));
        } else {
            result.add(new TestData(euri, null));
        }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
     *
     * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.modules.CrawlURI)
     */
    public int costOf(CrawlURI curi) {
        int cost = 1;
        UURI uuri = curi.getUURI();
        if (uuri.hasQuery()) {
            // has query string
            cost++;
            int qIndex = uuri.toString().indexOf('?');
            if (curi.flattenVia().startsWith(uuri.toString().substring(0,qIndex))) {
                // non-query-string portion of URI is same as previous
                cost++;
            }
            // TODO: other potential query-related cost penalties:
            //  - more than X query-string attributes
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
   
    public HopCrossesAssignmentLevelDomainDecideRule() {
    }

    protected boolean evaluate(CrawlURI uri) {
        UURI via = uri.getVia();
        if (via == null) {
            return false;
        }
        try {
            // determine if this hop crosses assignment-level-domain borders
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

    @Override
    protected Collection<TestData> makeData(String content, String uri)
    throws Exception {
        UURI src = UURIFactory.getInstance("http://www.archive.org/start/");
        CrawlURI euri = new CrawlURI(src, null, null, NAVLINK_MISC);
        Recorder recorder = createRecorder(content);
        euri.setContentType("text/css");
        euri.setRecorder(recorder);
        euri.setContentSize(content.length());
       
        // TODO: This test was naively modified to account for the abscense of LINK, but no effort was made to confirm that it is actually testing anything useful
        UURI dest = UURIFactory.getInstance(uri);
        CrawlURI link = euri.createCrawlURI(dest, EMBED_MISC, Hop.EMBED);
        TestData td = new TestData(euri, link);
        return Collections.singleton(td);
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    public AddRedirectFromRootServerToScope() {
    }

    @Override
    protected boolean evaluate(CrawlURI uri) {
        UURI via = uri.getVia();
        if (via == null) {
            return false;
        }
        try {
            String chost = uri.getUURI().getHostBasename();
            if (chost == null) {
                return false;
            }
           
            String viaHost = via.getHostBasename();
            if (viaHost == null) {
                return false;
            }
           
            if (chost.equals(viaHost) && uri.isLocation()
                    && via.getPath().equals(SLASH)) {
                uri.setSeed(true);
                LOGGER.info("Adding " + uri + " to seeds via " + via);
                return true;
            }
        } catch (URIException e) {
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
     * Test a GET FORM ACTION extraction
     *
     * @throws URIException
     */
    public void testFormsLinkGet() throws URIException {
        UURI uuri = UURIFactory.getInstance("http://www.example.org");
        CrawlURI curi = new CrawlURI(uuri);
        CharSequence cs =
          "<form name=\"testform\" method=\"GET\" action=\"redirect_me?form=true\"> " +
          "  <INPUT TYPE=CHECKBOX NAME=\"checked[]\" VALUE=\"1\" CHECKED> "+
          "  <INPUT TYPE=CHECKBOX NAME=\"unchecked[]\" VALUE=\"1\"> " +
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
     * Test a POST FORM ACTION being properly ignored
     *
     * @throws URIException
     */
    public void testFormsLinkIgnorePost() throws URIException {
        UURI uuri = UURIFactory.getInstance("http://www.example.org");
        CrawlURI curi = new CrawlURI(uuri);
        CharSequence cs =
            "<form name=\"testform\" method=\"POST\" action=\"redirect_me?form=true\"> " +
            "  <INPUT TYPE=CHECKBOX NAME=\"checked[]\" VALUE=\"1\" CHECKED> "+
            "  <INPUT TYPE=CHECKBOX NAME=\"unchecked[]\" VALUE=\"1\"> " +
View Full Code Here
TOP

Related Classes of org.archive.net.UURI

  • org.archive.crawler.datamodel.CrawlURITest
  • org.archive.crawler.frontier.AMQPUrlReceiver$UrlConsumer
  • org.archive.crawler.frontier.URIAuthorityBasedQueueAssignmentPolicy
  • org.archive.crawler.frontier.WagCostAssignmentPolicy
  • org.archive.crawler.io.UriErrorFormatter
  • org.archive.crawler.prefetch.PreconditionEnforcer
  • org.archive.crawler.selftest.SelfTestBase
  • org.archive.crawler.util.BdbUriUniqFilterTest
  • org.archive.crawler.util.BloomUriUniqFilterTest
  • org.archive.crawler.util.FPUriUniqFilterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.