Examples of net.yacy.kelondro.data.meta.DigestURI

net.yacy.kelondro.data.meta.DigestURI
URI-object providing YaCy-hash computation Hashes for URIs are split in several parts For URIs pointing to resources not globally available, the domainhash-part gets one reserved value

    }


    public static void doHead(final HashMap<String, Object> conProp, final RequestHeader requestHeader, OutputStream respond) {
        
//        ResponseContainer res = null;
        DigestURI url = null;
        try {
            final int reqID = requestHeader.hashCode();
            // remembering the starting time of the request
            final Date requestDate = new Date(); // remember the time...
            conProp.put(HeaderFramework.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
            if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
            sb.proxyLastAccess = System.currentTimeMillis();
            
            // using an ByteCount OutputStream to count the send bytes
            respond = new ByteCountOutputStream(respond,((String) conProp.get(HeaderFramework.CONNECTION_PROP_REQUESTLINE)).length() + 2,"PROXY");                                   
            
            String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
            final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH);
            final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); 
            final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
            
            int port, pos;
            if ((pos = host.indexOf(':')) < 0) {
                port = 80;
            } else {
                port = Integer.parseInt(host.substring(pos + 1));
                host = host.substring(0, pos);
            }
            
            try {
                url = new DigestURI("http", host, port, (args == null) ? path : path + "?" + args);
            } catch (final MalformedURLException e) {
                final String errorMsg = "ERROR: internal error with url generation: host=" +
                                  host + ", port=" + port + ", path=" + path + ", args=" + args;
                log.logSevere(errorMsg);
                HTTPDemon.sendRespondError(conProp,respond,4,501,null,errorMsg,e);

View Full Code Here


    public static void doPost(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream respond, InputStream body) throws IOException {
        assert conProp != null : "precondition violated: conProp != null";
        assert requestHeader != null : "precondition violated: requestHeader != null";
        assert body != null : "precondition violated: body != null";
        DigestURI url = null;
        ByteCountOutputStream countedRespond = null;
        try {
            final int reqID = requestHeader.hashCode();
            // remembering the starting time of the request
            final Date requestDate = new Date(); // remember the time...
            conProp.put(HeaderFramework.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
            if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
            sb.proxyLastAccess = System.currentTimeMillis();
            
            // using an ByteCount OutputStream to count the send bytes
            countedRespond  = new ByteCountOutputStream(respond,((String) conProp.get(HeaderFramework.CONNECTION_PROP_REQUESTLINE)).length() + 2,"PROXY");
                        
            String host    = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
            final String path    = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH);
            final String args    = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
            final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);


            int port, pos;
            if ((pos = host.indexOf(':')) < 0) {
                port = 80;
            } else {
                port = Integer.parseInt(host.substring(pos + 1));
                host = host.substring(0, pos);
            }
            
            try {
                url = new DigestURI("http", host, port, (args == null) ? path : path + "?" + args);
            } catch (final MalformedURLException e) {
                final String errorMsg = "ERROR: internal error with url generation: host=" +
                                  host + ", port=" + port + ", path=" + path + ", args=" + args;
                log.logSevere(errorMsg);
                HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e);

View Full Code Here

        final StringBuilder cpg = new StringBuilder(240);
        assert cpg.length() % 12 == 0 : "cpg.length() = " + cpg.length() + ", cpg = " + cpg.toString();
        //final String refhashp = ASCII.String(lro.url.hash(), 6, 6); // ref hash part
        String nexturlhash;
        for (final MultiProtocolURI u: lro.globalRefURLs) {
            final byte[] nexturlhashb = new DigestURI(u).hash();
            assert nexturlhashb != null;
            if (nexturlhashb != null) {
                nexturlhash = ASCII.String(nexturlhashb);
                assert nexturlhash.length() == 12 : "nexturlhash.length() = " + nexturlhash.length() + ", nexturlhash = " + nexturlhash;
                //assert !nexturlhash.substring(6).equals(refhashp);

View Full Code Here

                port = 80;
            }
            
            String urlString;
            try {
                urlString = (new DigestURI((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
            } catch (final MalformedURLException e) {
                urlString = "invalid URL";
            }


            // set rewrite values

View Full Code Here

                if (post.containsKey("blacklisturls")) {
                    PrintWriter pw;
                    try {
                        final String[] supportedBlacklistTypes = env.getConfig("BlackLists.types", "").split(",");
                        pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true));
                        DigestURI url;
                        for (final byte[] b: urlb) {
                            try {
                                urlHashes.put(b);
                            } catch (final RowSpaceExceededException e) {
                                Log.logException(e);
                            }
                            final URIMetadataRow e = segment.urlMetadata().load(b);
                            segment.urlMetadata().remove(b);
                            if (e != null) {
                                url = e.metadata().url();
                                pw.println(url.getHost() + "/" + url.getFile());
                                for (final String supportedBlacklistType : supportedBlacklistTypes) {
                                    if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklist)) {
                                        Switchboard.urlBlacklist.add(
                                                supportedBlacklistType,
                                                url.getHost(),
                                                url.getFile());
                                    }
                                }
                                SearchEventCache.cleanupEvents(true);
                            }
                        }
                        pw.close();
                    } catch (final IOException e) {
                    }
                }


                if (post.containsKey("blacklistdomains")) {
                    PrintWriter pw;
                    try {
                        final String[] supportedBlacklistTypes = Blacklist.BLACKLIST_TYPES_STRING.split(",");
                        pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true));
                        DigestURI url;
                        for (final byte[] b: urlb) {
                            try {
                                urlHashes.put(b);
                            } catch (final RowSpaceExceededException e) {
                                Log.logException(e);
                            }
                            final URIMetadataRow e = segment.urlMetadata().load(b);
                            segment.urlMetadata().remove(b);
                            if (e != null) {
                                url = e.metadata().url();
                                pw.println(url.getHost() + "/.*");
                                for (final String supportedBlacklistType : supportedBlacklistTypes) {
                                    if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklist)) {
                                        Switchboard.urlBlacklist.add(
                                                supportedBlacklistType,
                                                url.getHost(), ".*");
                                    }
                                }
                            }
                        }
                        pw.close();

View Full Code Here

            prop.put("genUrlList", 2);
            prop.put("searchresult", 3);
            prop.put("genUrlList_flags", (flags == null) ? "" : flags.exportB64());
            prop.put("genUrlList_lines", maxlines);
            int i = 0;
            DigestURI url;
            URIMetadataRow entry;
            String us;
            long rn = -1;
            while (!ranked.isEmpty() && (entry = ranked.takeURL(false, 1000)) != null) {
                if ((entry == null) || (entry.metadata() == null)) continue;
                url = entry.metadata().url();
                if (url == null) continue;
                us = url.toNormalform(false, false);
                if (rn == -1) rn = entry.ranking();
                prop.put("genUrlList_urlList_"+i+"_urlExists", "1");
                prop.put("genUrlList_urlList_"+i+"_urlExists_urlhxCount", i);
                prop.putHTML("genUrlList_urlList_"+i+"_urlExists_urlhxValue", entry.word().urlhash());
                prop.putHTML("genUrlList_urlList_"+i+"_urlExists_keyString", keystring);

View Full Code Here

                            // detect charset of html-files
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();

View Full Code Here

            m.appendReplacement(result, init + url);


        } else if (url.startsWith("http")) {
          // absoulte url of form href="http://domain.com/path"
          if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) {
            if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(url)) != null) {
              continue;
            }
          }


            m.appendReplacement(result, init + "/proxy.html?url=" + url);

View Full Code Here

                row = Supporter.get(urlhash);
                if (row == null) continue;
                
                url = row.getColString(0);
                try {
                    if (Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_SURFTIPS, new DigestURI(url, urlhash.getBytes()))) continue;
                } catch(final MalformedURLException e) {continue;}
                title = row.getColString(1);
                description = row.getColString(2);
                if ((url == null) || (title == null) || (description == null)) continue;
                refid = row.getColString(3);

View Full Code Here

            }


            // add/subtract votes and write record
            if (entry != null) {
                try {
                    urlhash = ASCII.String((new DigestURI(url)).hash());
                } catch (final MalformedURLException e) {
                    urlhash = null;
                }
                if (urlhash == null)
                    try {
                        urlhash = ASCII.String((new DigestURI("http://" + url)).hash());
                    } catch (final MalformedURLException e) {
                        urlhash = null;
                    }
                        if (urlhash==null) {
                            System.out.println("Supporter: bad url '" + url + "' from news record " + record.toString());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of net.yacy.kelondro.data.meta.DigestURI

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.