Package de.anomic.yacy

Examples of de.anomic.yacy.yacySeed


        } else {
            port = env.getConfigLong("port", 8090); //this allows a low port, but it will only get one, if the user edits the config himself.
        }

        // check if peer name already exists
        final yacySeed oldSeed = sb.peers.lookupByName(peerName);
        if (oldSeed == null &&
            !peerName.equals(sb.peers.mySeed().getName()) &&
            Pattern.compile("[A-Za-z0-9\\-_]{3,80}").matcher(peerName).matches()) {
            sb.peers.setMyName(peerName);
            sb.peers.saveMySeed();
View Full Code Here


            prop.put("table_showTitle", (showTitle) ? "1" : "0");
            prop.put("table_showURL", (showURL) ? "1" : "0");

            boolean dark = true;
            String urlstr, urltxt;
            yacySeed initiatorSeed, executorSeed;
            URIMetadataRow urle;
            URIMetadataRow.Components metadata;

            int cnt = 0;
            final Iterator<Map.Entry<String, InitExecEntry>> i = ResultURLs.results(tabletype);
            Map.Entry<String, InitExecEntry> entry;
            while (i.hasNext()) {
                entry = i.next();
                try {
                    urle = sb.indexSegments.urlMetadata(Segments.Process.LOCALCRAWLING).load(UTF8.getBytes(entry.getKey()));
                    if (urle == null) {
                        Log.logWarning("PLASMA", "CrawlResults: URL not in index with url hash " + entry.getKey());
                        urlstr = null;
                        urltxt = null;
                        metadata = null;
                        continue;
                    }
                    metadata = urle.metadata();
                    urlstr = metadata.url().toNormalform(false, true);
                    urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL
                   
                    initiatorSeed = entry.getValue() == null || entry.getValue().initiatorHash == null ? null : sb.peers.getConnected(ASCII.String(entry.getValue().initiatorHash));
                    executorSeed = entry.getValue() == null || entry.getValue().executorHash == null ? null : sb.peers.getConnected(ASCII.String(entry.getValue().executorHash));

                    prop.put("table_indexed_" + cnt + "_dark", (dark) ? "1" : "0");
                    prop.put("table_indexed_" + cnt + "_feedbackpage", "CrawlResults.html");
                    prop.put("table_indexed_" + cnt + "_tabletype", tabletype.getCode());
                    prop.put("table_indexed_" + cnt + "_urlhash", entry.getKey());

                    if (showInit) {
                        prop.put("table_indexed_" + cnt + "_showInit", "1");
                        prop.put("table_indexed_" + cnt + "_showInit_initiatorSeed", (initiatorSeed == null) ? "unknown" : initiatorSeed.getName());
                    } else
                        prop.put("table_indexed_" + cnt + "_showInit", "0");

                    if (showExec) {
                        prop.put("table_indexed_" + cnt + "_showExec", "1");
View Full Code Here

            prop.put("crawler-queue", "1");
            final List<Request> crawlerList = sb.crawlQueues.noticeURL.top(NoticedURL.StackType.LIMIT, showLimit);
           
            Request urle;
            boolean dark = true;
            yacySeed initiator;
            String profileHandle;
            CrawlProfile profileEntry;
            int i, showNum = 0;
            for (i = 0; (i < crawlerList.size()) && (showNum < showLimit); i++) {
                urle = crawlerList.get(i);
                if (urle != null && urle.url() != null) {
                    initiator = sb.peers.getConnected((urle.initiator() == null) ? "" : ASCII.String(urle.initiator()));
                    profileHandle = urle.profileHandle();
                    profileEntry = profileHandle == null ? null : sb.crawler.getActive(profileHandle.getBytes());
                    prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
                    prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
                    prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
                    prop.put("crawler-queue_list_"+showNum+"_depth", urle.depth());
                    prop.put("crawler-queue_list_"+showNum+"_modified", daydate(urle.appdate()) );
                    prop.putHTML("crawler-queue_list_"+showNum+"_anchor", urle.name());
                    prop.putHTML("crawler-queue_list_"+showNum+"_url", urle.url().toNormalform(false, true));
View Full Code Here

            prop.put("loader-set", "0");
        } else {
            prop.put("loader-set", "1");
            boolean dark = true;
            final Request[] w = sb.crawlQueues.activeWorkerEntries();
            yacySeed initiator;
            int count = 0;
            for (int i = 0; i < w.length; i++)  {
                if (w[i] == null) continue;
               
                initiator = sb.peers.getConnected((w[i].initiator() == null) ? "" : ASCII.String(w[i].initiator()));
                prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
                prop.putHTML("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
                prop.put("loader-set_list_"+count+"_depth", w[i].depth());
                prop.put("loader-set_list_"+count+"_status", w[i].getStatus());
                prop.putHTML("loader-set_list_"+count+"_url", w[i].url().toNormalform(true, false));
                dark = !dark;
                count++;
View Full Code Here

           
            if (sb.peers != null) {
                final int maxCount = Math.min(1000, sb.peers.newsPool.size(tableID));
                final Iterator<yacyNewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(tableID, false);
                yacyNewsDB.Record record;
                yacySeed seed;
                int i = 0;
                while ((recordIterator.hasNext()) && (i < maxCount)) {
                    record = recordIterator.next();
                    if (record == null) continue;
                   
                    seed = sb.peers.getConnected(record.originator());
                    if (seed == null) seed = sb.peers.getDisconnected(record.originator());
                    final String category = record.category();
                    prop.put("table_list_" + i + "_id", record.id());
                    prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
                    prop.put("table_list_" + i + "_cre", GenericFormatter.SHORT_SECOND_FORMATTER.format(record.created()));
                    prop.put("table_list_" + i + "_crerfcdate", HeaderFramework.formatRFC1123(record.created()));
                    prop.putHTML("table_list_" + i + "_cat", category);
                    prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : GenericFormatter.SHORT_SECOND_FORMATTER.format(record.received()));
                    prop.put("table_list_" + i + "_dis", record.distributed());
                   
                    final Map<String, String> attributeMap = record.attributes();
                    prop.putHTML("table_list_" + i + "_att", attributeMap.toString());
                    int j = 0;
                    if (!attributeMap.isEmpty()) {
                        for (final Entry<String, String> attribute: attributeMap.entrySet()) {
                            prop.put("table_list_" + i + "_attributes_" + j + "_name", attribute.getKey());
                            prop.putHTML("table_list_" + i + "_attributes_" + j + "_value", attribute.getValue());
                            j++;
                        }
                    }
                    prop.put("table_list_" + i + "_attributes", j);
                                       
                    // generating link / title / description (taken over from Surftips.java)
                    String link, title, description;
                    if (category.equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
                      link = record.attribute("startURL", "");
                      title = (record.attribute("intention", "").isEmpty()) ? link : record.attribute("intention", "");
                      description = "Crawl Start Point";
                    } else if (category.equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) {
                      link = record.attribute("homepage", "");
                      title = "Home Page of " + record.attribute("nickname", "");
                      description = "Profile Update";
                    } else if (category.equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) {
                      link = record.attribute("url", "");
                      title = record.attribute("title", "");
                      description = "Bookmark: " + record.attribute("description", "");
                    } else if (category.equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) {
                      link = record.attribute("url", "");
                      title = record.attribute("title", "");
                      description = "Surf Tipp: " + record.attribute("description", "");
                    } else if (category.equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
                      link = record.attribute("url", "");
                      title = record.attribute("title", "");
                      description = record.attribute("url", "");
                    } else if (category.equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
                      link = (seed == null)? "" : "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
                      title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", "");
                      description = "Wiki Update: " + record.attribute("description", "");
                    } else if (category.equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
                      link = (seed == null)? "" : "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
                      title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", "");
                      description = "Blog Entry: " + record.attribute("subject", "");
                    } else {
                      link = "";            
                      title = "";
View Full Code Here

            prop.put("crawler-queue", "1");
            final List<Request> crawlerList = sb.crawlQueues.noticeURL.top(NoticedURL.StackType.CORE, (int) (showLimit * 1.20));

            Request urle;
            boolean dark = true;
            yacySeed initiator;
            String profileHandle;
            CrawlProfile profileEntry;
            int i;
            for (i = 0; (i < crawlerList.size()) && (showNum < showLimit); i++) {
                urle = crawlerList.get(i);
                if ((urle != null)&&(urle.url()!=null)) {
                    initiator = sb.peers.getConnected(urle.initiator() == null ? "" : ASCII.String(urle.initiator()));
                    profileHandle = urle.profileHandle();
                    profileEntry = profileHandle == null ? null : sb.crawler.getActive(profileHandle.getBytes());
                    prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
                    prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
                    prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
                    prop.put("crawler-queue_list_"+showNum+"_depth", urle.depth());
                    prop.put("crawler-queue_list_"+showNum+"_modified", daydate(urle.appdate()) );
                    prop.putHTML("crawler-queue_list_"+showNum+"_anchor", urle.name());
                    prop.putHTML("crawler-queue_list_"+showNum+"_url", urle.url().toNormalform(false, true));
View Full Code Here

           known       - the resource is already in database, believed to be fresh and not reloaded
           stale       - the resource was reloaded but not processed because source had no changes

        */
       
        final yacySeed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));       

        if ((sb.peers.mySeed() == null) || (!(sb.peers.mySeed().hash.equals(youare)))) {
            // no yacy connection / unknown peers
            prop.put("delay", "3600");
            return prop;
View Full Code Here

        final int entryc      = post.getInt("entryc", 0);                 // number of entries in indexes
        byte[] indexes        = post.get("indexes", "").getBytes();       // the indexes, as list of word entries
        boolean granted       = sb.getConfigBool("allowReceiveIndex", false);
        final boolean blockBlacklist = sb.getConfigBool("indexReceiveBlockBlacklist", false);
        final long cachelimit = sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 100000);
        final yacySeed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));               
       
        // response values
        int pause = 0;
        String result = "ok";
        final StringBuilder unknownURLs = new StringBuilder(6000);
       
        if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) {
          sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash);
            result = "wrong_target";
            pause = 0;
        } else if (otherPeer == null) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted. Other Peer is unknown");
            result = "not_granted";
            pause = 60000;
        } else if (!granted) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Granted is false");
            result = "not_granted";
            pause = 60000;
        } else if (sb.isRobinsonMode()) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted. This peer is in robinson mode");
            result = "not_granted";
            pause = 60000;
        } else if (sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() > cachelimit) {
            // we are too busy to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() + ").");
            granted = false; // don't accept more words if there are too many words to flush
            result = "busy";
            pause = 60000;
        } else if (otherPeer.getVersion() < 0.75005845 && otherPeer.getVersion() >= 0.75005821) {
          // version that sends [B@... hashes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Bad version.");
            result = "not_granted";
            pause = 1800000;
        } else {
View Full Code Here

        long   nwords  = sb.peers.countActiveRWI();
        final double nqpm    = sb.peers.countActiveQPM();
        long   nppm    = sb.peers.countActivePPM();
        double nqph    = 0;

        final yacySeed seed = sb.peers.mySeed();
        if (seed != null){
            name    = seed.get(yacySeed.NAME, "-").toUpperCase();
            links   = seed.getLinkCount();
            words   = seed.getWordCount();
            myppm   = seed.getPPM();
            myqph   = 60d * seed.getQPM();

            if (sb.peers.mySeed().isVirgin()) {
                type = "VIRGIN";
                nqph = Math.round(6000d * nqpm) / 100d;
            } else if(sb.peers.mySeed().isJunior()) {
View Full Code Here

        if (seed.length() > yacySeed.maxsize) {
          yacyCore.log.logInfo("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + yacySeed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")");
            prop.put("message", "your seed is too long (" + seed.length() + ")");
            return prop;
        }
        yacySeed remoteSeed;
        try {
            remoteSeed = yacySeed.genRemoteSeed(seed, key, true, ias.getHostAddress());
        } catch (IOException e) {
            yacyCore.log.logInfo("hello/server: bad seed: " + e.getMessage() + ", time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "bad seed: " + e.getMessage());
            return prop;
        }
       
        if (remoteSeed == null || remoteSeed.hash == null) {
            yacyCore.log.logInfo("hello/server: bad seed: null, time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "cannot parse your seed");
            return prop;
        }
       
//      final String properTest = remoteSeed.isProper();
        // The remote peer might not know its IP yet, so don't abort if the IP check fails
//      if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }

        // we easily know the caller's IP:
        final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
        final String reportedip = remoteSeed.getIP();
        final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
        final float clientversion = remoteSeed.getVersion();

        if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
          // if we are a robinson cluster, answer only if this client is known by our network definition
            prop.put("message", "I am robinson, I do not answer");
            return prop;
        }
       
        long[] callback = new long[]{-1, -1};
        if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));
       
        // if the remote client has reported its own IP address and the client supports
        // the port forwarding feature (if client version >= 0.383) then we try to
        // connect to the reported IP address first
        time = System.currentTimeMillis();
        long time_backping = 0;
        String backping_method = "none";
        if (reportedip.length() > 0 &&
            !clientip.equals(reportedip) &&
            clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING &&
            magic != 0) {           
            serverCore.checkInterruption();
           
            // try first the reportedip, since this may be a connect from a port-forwarding host
            prop.put("yourip", reportedip);
            remoteSeed.setIP(reportedip);
            callback = yacyClient.queryUrlCount(remoteSeed);
            time_backping = System.currentTimeMillis() - time;
            backping_method = "reportedip=" + reportedip;
        } else {
            prop.put("yourip", ias.getHostAddress());
            remoteSeed.setIP(ias.getHostAddress());
        }

        // if the previous attempt (using the reported ip address) was not successful,
        // then try the ip where the request came from
        if (callback[0] < 0 || (magic != 0 && magic != callback[1])) {
            boolean isNotLocal = true;

            // we are only allowed to connect to the client IP address if it's not our own address
            if (serverCore.useStaticIP) {
                    isNotLocal = !ias.isSiteLocalAddress();
            }

            if (isNotLocal) {
                serverCore.checkInterruption();
               
                prop.put("yourip", clientip);
                remoteSeed.setIP(clientip);
                callback = yacyClient.queryUrlCount(remoteSeed);
                time_backping = System.currentTimeMillis() - time;
                backping_method = "clientip=" + clientip;
            }
        }

//      System.out.println("YACYHELLO: YOUR IP=" + clientip);
        // set lastseen value (we have seen that peer, it contacted us!)
        remoteSeed.setLastSeenUTC();
       
        // assign status
        if (callback[0] >= 0) {
            if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
                prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
                remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
            } else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
                prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL);
            } else {
                prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
                remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
            }
            // connect the seed
            yacyCore.log.logInfo("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        } else {
            prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
            remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
            yacyCore.log.logInfo("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            // no connection here, instead store junior in connection cache
            if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) {
                sb.peers.peerActions.peerPing(remoteSeed);
            }
        }
        sb.peers.peerActions.setUserAgent(clientip, userAgent);
        if (!(prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
            yacyCore.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() +
                                                           "' [" + reportedip +
                                             "] peerType from '" + reportedPeerType +
                                                        "' to '" + prop.get(yacySeed.YOURTYPE) + "'.");
        }

        serverCore.checkInterruption();
        final StringBuilder seeds = new StringBuilder(768);
        // attach some more seeds, as requested
        if (sb.peers.sizeConnected() > 0) {
            if (count > sb.peers.sizeConnected()) { count = sb.peers.sizeConnected(); }
            if (count > 100) { count = 100; }
           
            // latest seeds
            final Map<String, yacySeed> ySeeds = PeerSelection.seedsByAge(sb.peers, true, count); // peerhash/yacySeed relation
           
            // attach also my own seed
            seeds.append("seed0=").append(sb.peers.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
            count = 1;           
           
            // attach other seeds
            if (ySeeds != null) {
                seeds.ensureCapacity((ySeeds.size() + 1) * 768);
                final Iterator<yacySeed> si = ySeeds.values().iterator();
                yacySeed s;
                String seedString;
                while (si.hasNext()) {
                  s = si.next();
                    if ((s != null) && (s.isProper(false) == null)) try {
                        seedString = s.genSeedStr(key);
                        if (seedString != null) {
                            seeds.append("seed").append(count).append('=').append(seedString).append(serverCore.CRLF_STRING);
                            count++;
                        }
                    } catch (final ConcurrentModificationException e) {
View Full Code Here

TOP

Related Classes of de.anomic.yacy.yacySeed

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.