Examples of org.archive.modules.net.CrawlServer

org.archive.modules.net.CrawlServer
Represents a single remote "server". A server is a service on a host. There might be more than one service on a host differentiated by a port number. @author gojomo

     * @param credential Credential to test.
     * @param curi CrawlURI.
     * @return True if already run.
     */
    protected boolean authenticated(final Credential credential, final CrawlURI curi) {
        CrawlServer server = serverCache.getServerFor(curi.getUURI());
        if (!server.hasCredentials()) {
            return false;
        }
        Set<Credential> credentials = server.getCredentials();
        for (Credential cred: credentials) {
            if (cred.getKey().equals(credential.getKey()) 
                    && cred.getClass().isInstance(credential)) {
                return true; 
            }

View Full Code Here

            h.setIP(LOCALHOST, -1);
            return h;
        }
        @Override 
        public CrawlServer getServerFor(String serverKey) {
            CrawlServer s = new CrawlServer(serverKey);
            return s;
        }

View Full Code Here

        try {
            serverKey = CrawlServer.getServerKey(curi.getUURI());
        } catch (URIException e) {
            return false;
        }
        CrawlServer server = fetcher.getServerCache().getServerFor(serverKey);
        if (server.hasCredentials()) {
            for (Credential c: server.getCredentials()) {
                if (c.isEveryTime()) {
                    if (c instanceof HttpAuthenticationCredential) {
                        HttpAuthenticationCredential cred = (HttpAuthenticationCredential) c;
                        AuthScheme authScheme = fetcher.chooseAuthScheme(server.getHttpAuthChallenges(), HttpHeaders.WWW_AUTHENTICATE);
                        populateHttpCredential(targetHost, authScheme, cred.getLogin(), cred.getPassword());
                    } else {
                        populateHtmlFormCredential((HtmlFormCredential) c);
                    }
                }

View Full Code Here

     * Get the {@link CrawlServer} associated with <code>name</code>.
     * @param serverKey Server name we're to return server for.
     * @return CrawlServer instance that matches the passed server name.
     */
    public CrawlServer getServerFor(final String serverKey) {
        CrawlServer cserver = servers.getOrUse(
                serverKey,
                new Supplier<CrawlServer>() {
                    public CrawlServer get() {
                        String skey = new String(serverKey); // ensure private minimal key
                        return new CrawlServer(skey);
                    }});
        return cserver;
    }

View Full Code Here

     * @param curi
     * @param stage
     */
    protected void tally(CrawlURI curi, Stage stage) {
        // Tally per-server, per-host, per-frontier-class running totals
        CrawlServer server = getServerCache().getServerFor(curi.getUURI());
        if (server != null) {
            server.getSubstats().tally(curi, stage);
            server.makeDirty(); 
        }
        try {
            CrawlHost host = getServerCache().getHostFor(curi.getUURI());
            if (host != null) {
                host.getSubstats().tally(curi, stage);

View Full Code Here

0 1

TOP

Related Classes of org.archive.modules.net.CrawlServer

org.archive.crawler.frontier.AbstractFrontier

org.archive.crawler.postprocessor.DispositionProcessor

org.archive.crawler.prefetch.PreconditionEnforcer

org.archive.crawler.prefetch.QuotaEnforcer

org.archive.modules.credential.Credential

org.archive.modules.fetcher.CookieFetchHTTPIntegrationTest$AlwaysLocalhostServerCache

org.archive.modules.fetcher.DefaultServerCache

org.archive.modules.fetcher.FetchHTTP

org.archive.modules.fetcher.FetchHTTPRequest

org.archive.modules.net.CrawlServerTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.