Package org.archive.net

Examples of org.archive.net.UURI

129
130
131
132
133
134
135
136
137
138
139
            String build = getFormat();
            CharSequence dest = link.getUURI();
            String implied = extractImplied(dest, trigger, build);
            if (implied != null) {
                try {
                    UURI target = UURIFactory.getInstance(implied);
                    LinkContext lc = LinkContext.INFERRED_MISC;
                    Hop hop = Hop.INFERRED;
                    addOutlink(curi, target, lc, hop);
                    numberOfLinksExtracted.incrementAndGet();
View Full Code Here
76
77
78
79
80
81
82
83
     *
     * @return   a CrawlURI
     * @throws Exception   just in case
     */
    protected CrawlURI defaultURI() throws Exception {
        UURI uuri = UURIFactory.getInstance("http://www.archive.org/start/");
        return new CrawlURI(uuri, null, null, LinkContext.NAVLINK_MISC);
    }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    }
   
   
    private void addLink(CrawlURI curi, String hyperlink) {
        try {
            UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink);
            LinkContext lc = LinkContext.NAVLINK_MISC;
            addOutlink(curi, hyperlink, lc, Hop.NAVLINK);
        } catch (URIException e1) {
            logUriError(e1, curi.getUURI(), hyperlink);
        }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
    /* (non-Javadoc)
     * @see java.util.logging.Formatter#format(java.util.logging.LogRecord)
     */
    public String format(LogRecord lr) {
        UURI uuri = (UURI) lr.getParameters()[0];
        String problem = (String) lr.getParameters()[1];

        return ArchiveUtils.getLog17Date()
        + " "
        + ( (uuri ==null) ? "n/a" : uuri.toString() )
        + " \""
        + lr.getMessage()
        + "\" "
        + problem
        + "\n";
View Full Code Here
361
362
363
364
365
366
367
368
369
370
371
372
        for (ArchiveRecordHeader arh: headers) {
            // ignore 'filedesc:' record
            if(arh.getUrl().startsWith("filedesc:")) {
                continue;
            }
            UURI uuri = UURIFactory.getInstance(arh.getUrl());
            String path = uuri.getPath();
            if (path.startsWith("/")) {
                path = path.substring(1);
            }
            if (arh.getUrl().startsWith("http:")) {
                result.add(path);
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
        }
        return linkSet;
    }
    private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
    UnsupportedEncodingException, IOException {
        UURI testUuri = UURIFactory.getInstance(urlStr);
        CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);
       

        File temp = File.createTempFile("test", ".tmp");
        Recorder recorder = new Recorder(temp, 1024, 1024);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
        // all whois urls in the same queue
        if (curi.getUURI().getScheme().equals("whois")) {
            return "whois...";
        }
       
        UURI basis = curi.getPolicyBasisUURI();
        String candidate = getCoreKey(basis);
       
        if(StringUtils.isEmpty(candidate)) {
            return DEFAULT_CLASS_KEY;
        }
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
        return (ExtractorYoutubeFormatStream)extractor;
    }

    private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
    UnsupportedEncodingException, IOException {
        UURI testUuri = UURIFactory.getInstance(urlStr);
        CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);

        InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream(resourceFileName);
        BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
        StringBuilder content = new StringBuilder();
View Full Code Here
TOP

Related Classes of org.archive.net.UURI

  • org.archive.crawler.datamodel.CrawlURITest
  • org.archive.crawler.frontier.AMQPUrlReceiver$UrlConsumer
  • org.archive.crawler.frontier.URIAuthorityBasedQueueAssignmentPolicy
  • org.archive.crawler.frontier.WagCostAssignmentPolicy
  • org.archive.crawler.io.UriErrorFormatter
  • org.archive.crawler.prefetch.PreconditionEnforcer
  • org.archive.crawler.selftest.SelfTestBase
  • org.archive.crawler.util.BdbUriUniqFilterTest
  • org.archive.crawler.util.BloomUriUniqFilterTest
  • org.archive.crawler.util.FPUriUniqFilterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.