Examples of net.yacy.cora.document.MultiProtocolURI

net.yacy.cora.document.MultiProtocolURI
MultiProtocolURI provides a URL object for multiple protocols like http, https, ftp, smb and file

                        robotsEntry = null;
                    }
                  prop.put("robots-allowed", robotsEntry == null ? 1 : robotsEntry.isDisallowed(theURL) ? 0 : 1);


                    // get the sitemap URL of the domain
                    final MultiProtocolURI sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap();
                    prop.putXML("sitemap", sitemapURL == null ? "" : sitemapURL.toString());
                } catch (final MalformedURLException e) {}
            }


        }
        // return rewrite properties

View Full Code Here

        // return variable that accumulates replacements
        final serverObjects prop = new serverObjects();
        final Switchboard sb = (Switchboard) env;


        // get referer for backlink
        final MultiProtocolURI referer = header.referer();
        prop.put("referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true, true));
        //if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());


        if (post == null) {
            prop.put("info", "1");//no information submitted
            return prop;

View Full Code Here

            prop.put("imgurl", "1");        
            int c = 0;
            final int yOffset = embed ? 0 : 70;
            for (int i = 0; i < fifoSize; i++) {
             
                final MultiProtocolURI baseURL = origins[i].baseURL;
                final MultiProtocolURI imageURL = origins[i].imageEntry.url();
                
                // check if this loads a page from localhost, which must be prevented to protect the server
                // against attacks to the administration interface when localhost access is granted
                if ((Domains.isLocal(baseURL.getHost()) || Domains.isLocal(imageURL.getHost())) &&
                    sb.getConfigBool("adminAccountForLocalhost", false)) continue;
                
                final long z = imgZIndex[i];
                prop.put("imgurl_list_" + c + "_url",
                       "<a href=\"" + baseURL.toNormalform(true, false) + "\">"
                       + "<img src=\"" + imageURL.toNormalform(true, false) + "\" "
                       + "style=\""
                       + ((imgWidth[i] == 0 || imgHeight[i] == 0) ? "" : "width:" + imgWidth[i] + "px;height:" + imgHeight[i] + "px;")
                       + "position:absolute;top:" + (imgPosY[i] + yOffset)
                       + "px;left:" + imgPosX[i]
                       + "px;z-index:" + z + "\" "

View Full Code Here

public class cytag {
    
    public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        
        final Switchboard sb = (Switchboard)env;
        final MultiProtocolURI referer = header.referer();
        
        // harvest request information
        StringBuilder connect = new StringBuilder();
        connect.append('{');
        appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format());
        appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false, false));
        appendJSON(connect, "nick",  (post == null) ? "" : post.get("nick", ""));
        appendJSON(connect, "tag",   (post == null) ? "" : post.get("tag", ""));
        appendJSON(connect, "icon",  (post == null) ? "" : post.get("icon", ""));
        appendJSON(connect, "ip",    header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""));
        appendJSON(connect, "agent", header.get("User-Agent", ""));

View Full Code Here

        if (url.isLocal()) return; // we do this only for global urls
        final Map<MultiProtocolURI, String> hl = document.getHyperlinks();
        final Iterator<MultiProtocolURI> it = hl.keySet().iterator();
        final HashSet<MultiProtocolURI> globalRefURLs = new HashSet<MultiProtocolURI>();
        final String refhost = url.getHost();
        MultiProtocolURI u;
        while (it.hasNext()) {
            u = it.next();
            if (u == null) continue;
            if (refhost != null && u.getHost() != null && !u.getHost().equals(refhost)) {
                // this is a global link
                globalRefURLs.add(u);
            }
        }
        final leanrefObject lro = new leanrefObject(url, globalRefURLs);

View Full Code Here

          // absolute path of form href="/absolute/path/to/linked/page"
            m.appendReplacement(result, init + "/proxy.html?url=http://" + proxyurl.getHost() + url);


        } else {
          // relative path of form href="relative/path"
          MultiProtocolURI target = new MultiProtocolURI(proxyurl.getHost() + directory + "/" + url);
            m.appendReplacement(result, init + "/proxy.html?url=" + target.toString());


        }
      }
      m.appendTail(result);

View Full Code Here

        for (int i = 0; i < transmissionQueueCount; i++) {
            this.transmissionQueue[i] = new ArrayBlockingQueue<SolrInputDocument>(transmissionQueueSize);
        }


        // connect using authentication
        final MultiProtocolURI u = new MultiProtocolURI(this.solrurl);
        this.host = u.getHost();
        this.port = u.getPort();
        this.solrpath = u.getPath();
        final String userinfo = u.getUserInfo();
        if (userinfo == null || userinfo.length() == 0) {
            this.solraccount = ""; this.solrpw = "";
        } else {
            final int p = userinfo.indexOf(':');
            if (p < 0) {

View Full Code Here

    private static byte[] postToFile(final yacySeedDB seedDB, final String targetHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
      return postToFile(seedDB.targetAddress(targetHash), targetHash, filename, parts, timeout);
    }
    private static byte[] postToFile(final String targetAddress, final String targetPeerHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
        final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
        return httpClient.POSTbytes(new MultiProtocolURI("http://" + targetAddress + "/yacy/" + filename), yacySeed.b64Hash2hexHash(targetPeerHash) + ".yacyh", parts, false);
    }

View Full Code Here

            parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt)));
            // send request
            final long start = System.currentTimeMillis();
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 30000);
            final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts, false);
            yacyCore.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + address + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + (System.currentTimeMillis() - start) + " milliseconds");
            result = FileUtils.table(content);
        } catch (final Exception e) {
            if (Thread.currentThread().isInterrupted()) {
                yacyCore.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted.");

View Full Code Here

            parts.put("call", UTF8.StringBody("remotecrawl"));
            parts.put("count", UTF8.StringBody(Integer.toString(maxCount)));
            parts.put("time", UTF8.StringBody(Long.toString(maxTime)));
            // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), (int) maxTime);
            final byte[] result = httpClient.POSTbytes(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false);
            final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
            if (reader == null) {
                yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");
                target.put(yacySeed.RCOUNT, "0");
                seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of net.yacy.cora.document.MultiProtocolURI

Collage

cytag

de.anomic.crawler.RobotsTxt

de.anomic.crawler.RobotsTxtEntry

de.anomic.data.BookmarkHelper

de.anomic.http.server.HTTPDFileHandler

de.anomic.search.QueryParams

de.anomic.search.Switchboard

de.anomic.yacy.graphics.WebStructureGraph

de.anomic.yacy.yacyClient

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.