Package net.yacy.peers

Examples of net.yacy.peers.Seed


            }
           
            String prot = "http"; // only httpd sessions listed
           
            // determining if the source is a yacy host
            Seed seed = null;
            if (doNameLookup) {
                seed = sb.peers.lookupByIP(userAddress,true,false,false);
                if (seed != null && (seed.hash.equals(sb.peers.mySeed().hash)) &&
                        (!seed.get(Seed.PORT,"").equals(Integer.toString(userPort)))) {
                    seed = null;
                }
            }
           
            prop.put("list_" + idx + "_dark", dark ? "1" : "0");
            dark = !dark;
            try {
                prop.put("list_" + idx + "_serverSessionID",URLEncoder.encode(s.getName(),"UTF8"));
            } catch (final UnsupportedEncodingException e) {
                // TODO Auto-generated catch block
                Log.logException(e);
            }
            prop.putHTML("list_" + idx + "_sessionName", s.getName());
            prop.put("list_" + idx + "_proto", prot);
            if (sessionTime > 1000*60) {
                prop.put("list_" + idx + "_ms", "0");
                prop.put("list_" + idx + "_ms_duration",PeerActions.formatInterval(sessionTime));
            } else {
                prop.put("list_" + idx + "_ms", "1");
                prop.putNum("list_" + idx + "_ms_duration", sessionTime);
            }
            prop.putHTML("list_" + idx + "_source",(seed!=null)?seed.getName()+".yacy":userAddress.getHostAddress()+":"+userPort);
            prop.putHTML("list_" + idx + "_dest", "-");
            if (blockingRequest) {
                prop.put("list_" + idx + "_running", "0");
                prop.putNum("list_" + idx + "_running_reqNr", commandCount+1);
                numActivePending++;
View Full Code Here


            if (otherSeedString.length() == 0) {
                prop.put("response", "-1"); // request rejected
                return prop;
            }
            //Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time
            Seed otherSeed;
            try {
                otherSeed = Seed.genRemoteSeed(otherSeedString, key, false, ias.getHostAddress());
            } catch (final IOException e) {
                prop.put("response", "-1"); // don't accept messages for bad seeds
                return prop;
            }

            String subject = crypt.simpleDecode(post.get("subject", ""), key); // message's subject
            String message = crypt.simpleDecode(post.get("message", ""), key); // message body
            if (subject == null || message == null) {
                prop.put("response", "-1"); // don't accept empty messages
                return prop;
            }
            message = message.trim();
            subject = subject.trim();
            if (subject.length() == 0 || message.length() == 0) {
                prop.put("response", "-1"); // don't accept empty messages
                return prop;
            }

            prop.put("response", "Thank you!");

            // save message
            MessageBoard.entry msgEntry = null;
            byte[] mb;
            mb = UTF8.getBytes(message);
            sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
                    "remote",
                    otherSeed.get(Seed.NAME, "anonymous"), otherSeed.hash,
                    sb.peers.mySeed().getName(), sb.peers.mySeed().hash,
                    subject, mb));

            messageForwardingViaEmail(sb, msgEntry);
View Full Code Here

           known       - the resource is already in database, believed to be fresh and not reloaded
           stale       - the resource was reloaded but not processed because source had no changes

        */

        final Seed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

        if ((sb.peers.mySeed() == null) || (!(sb.peers.mySeed().hash.equals(youare)))) {
            // no yacy connection / unknown peers
            prop.put("delay", "3600");
            return prop;
View Full Code Here

        final int entryc      = post.getInt("entryc", 0);                 // number of entries in indexes
        byte[] indexes        = post.get("indexes", "").getBytes();       // the indexes, as list of word entries
        boolean granted       = sb.getConfigBool("allowReceiveIndex", false);
        final boolean blockBlacklist = sb.getConfigBool("indexReceiveBlockBlacklist", false);
        final long cachelimit = sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 100000);
        final Seed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

        // response values
        int pause = 0;
        String result = "ok";
        final StringBuilder unknownURLs = new StringBuilder(6000);

        if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) {
          sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash);
            result = "wrong_target";
            pause = 0;
        } else if (otherPeer == null) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted. Other Peer is unknown");
            result = "not_granted";
            pause = 60000;
        } else if (!granted) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Granted is false");
            result = "not_granted";
            pause = 60000;
        } else if (sb.isRobinsonMode()) {
            // we dont want to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted. This peer is in robinson mode");
            result = "not_granted";
            pause = 60000;
        } else if (sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() > cachelimit) {
            // we are too busy to receive indexes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() + ").");
            granted = false; // don't accept more words if there are too many words to flush
            result = "busy";
            pause = 60000;
        } else if (otherPeer.getVersion() < 0.75005845 && otherPeer.getVersion() >= 0.75005821) {
          // version that sends [B@... hashes
            sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Bad version.");
            result = "not_granted";
            pause = 1800000;
        } else {
View Full Code Here

        final serverObjects prop = new serverObjects();

        if (post != null) {
            if (post.containsKey("retrieve")) {
                final String peerhash = post.get("peer", null);
                final Seed seed = (peerhash == null) ? null : sb.peers.getConnected(peerhash);
                final RSSFeed feed = (seed == null) ? null : Protocol.queryRemoteCrawlURLs(sb.peers, seed, 20, 60000);
                if (feed != null) {
                    for (final Hit item: feed) {
                        //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate());
                       
View Full Code Here

        return (url == null ? "null" : url.toNormalform(true, false));
    }
   
    private static void listHosts(final Switchboard sb, final serverObjects prop) {
        // list known hosts
        Seed seed;
        int hc = 0;
        if (sb.peers != null && sb.peers.sizeConnected() > 0) {
            final Iterator<Seed> e = PeerSelection.getProvidesRemoteCrawlURLs(sb.peers);
            while (e.hasNext()) {
                seed = e.next();
                if (seed != null) {
                    prop.put("hosts_" + hc + "_hosthash", seed.hash);
                    prop.putHTML("hosts_" + hc + "_hostname", seed.hash + " " + seed.get(Seed.NAME, "nameless") + " (" + seed.getLong(Seed.RCOUNT, 0) + ")");
                    hc++;
                }
            }
            prop.put("hosts", hc);
        } else {
View Full Code Here

        if (seed.length() > Seed.maxsize) {
          Network.log.logInfo("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")");
            prop.put("message", "your seed is too long (" + seed.length() + ")");
            return prop;
        }
        Seed remoteSeed;
        try {
            remoteSeed = Seed.genRemoteSeed(seed, key, true, ias.getHostAddress());
        } catch (final IOException e) {
            Network.log.logInfo("hello/server: bad seed: " + e.getMessage() + ", time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "bad seed: " + e.getMessage());
            return prop;
        }

        if (remoteSeed == null || remoteSeed.hash == null) {
            Network.log.logInfo("hello/server: bad seed: null, time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "cannot parse your seed");
            return prop;
        }

//      final String properTest = remoteSeed.isProper();
        // The remote peer might not know its IP yet, so don't abort if the IP check fails
//      if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }

        // we easily know the caller's IP:
        final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
        final String reportedip = remoteSeed.getIP();
        final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
        final float clientversion = remoteSeed.getVersion();

        if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
          // if we are a robinson cluster, answer only if this client is known by our network definition
            prop.put("message", "I am robinson, I do not answer");
            return prop;
        }

        long[] callback = new long[]{-1, -1};
        if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));

        // if the remote client has reported its own IP address and the client supports
        // the port forwarding feature (if client version >= 0.383) then we try to
        // connect to the reported IP address first
        time = System.currentTimeMillis();
        long time_backping = 0;
        String backping_method = "none";
        if (reportedip.length() > 0 &&
            !clientip.equals(reportedip) &&
            clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING &&
            magic != 0) {
            serverCore.checkInterruption();

            // try first the reportedip, since this may be a connect from a port-forwarding host
            prop.put("yourip", reportedip);
            remoteSeed.setIP(reportedip);
            callback = Protocol.queryUrlCount(remoteSeed);
            time_backping = System.currentTimeMillis() - time;
            backping_method = "reportedip=" + reportedip;
        } else {
            prop.put("yourip", ias.getHostAddress());
            remoteSeed.setIP(ias.getHostAddress());
        }

        // if the previous attempt (using the reported ip address) was not successful,
        // then try the ip where the request came from
        if (callback[0] < 0 || (magic != 0 && magic != callback[1])) {
            boolean isNotLocal = true;

            // we are only allowed to connect to the client IP address if it's not our own address
            if (serverCore.useStaticIP) {
                    isNotLocal = !ias.isSiteLocalAddress();
            }

            if (isNotLocal) {
                serverCore.checkInterruption();

                prop.put("yourip", clientip);
                remoteSeed.setIP(clientip);
                callback = Protocol.queryUrlCount(remoteSeed);
                time_backping = System.currentTimeMillis() - time;
                backping_method = "clientip=" + clientip;
            }
        }

//      System.out.println("YACYHELLO: YOUR IP=" + clientip);
        // set lastseen value (we have seen that peer, it contacted us!)
        remoteSeed.setLastSeenUTC();

        // assign status
        final int connectedBefore = sb.peers.sizeConnected();
        if (callback[0] >= 0) {
            if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
                remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
            } else if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL).equals(Seed.PEERTYPE_PRINCIPAL)) {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_PRINCIPAL);
            } else {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
                remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
            }
            // connect the seed
            Network.log.logInfo("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        } else {
            prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR);
            remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
            Network.log.logInfo("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            // no connection here, instead store junior in connection cache
            if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) {
                sb.peers.peerActions.peerPing(remoteSeed);
            }
        }
        final int connectedAfter = sb.peers.sizeConnected();

        // update event tracker
        EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);

        sb.peers.peerActions.setUserAgent(clientip, userAgent);
        if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) {
            Network.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() +
                                                           "' [" + reportedip +
                                             "] peerType from '" + reportedPeerType +
                                                        "' to '" + prop.get(Seed.YOURTYPE) + "'.");
        }

        serverCore.checkInterruption();
        final StringBuilder seeds = new StringBuilder(768);
        // attach some more seeds, as requested
        if (sb.peers.sizeConnected() > 0) {
            if (count > sb.peers.sizeConnected()) { count = sb.peers.sizeConnected(); }
            if (count > 100) { count = 100; }

            // latest seeds
            final Map<String, Seed> ySeeds = PeerSelection.seedsByAge(sb.peers, true, count); // peerhash/yacySeed relation

            // attach also my own seed
            seeds.append("seed0=").append(sb.peers.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
            count = 1;

            // attach other seeds
            if (ySeeds != null) {
                seeds.ensureCapacity((ySeeds.size() + 1) * 768);
                final Iterator<Seed> si = ySeeds.values().iterator();
                Seed s;
                String seedString;
                while (si.hasNext()) {
                  s = si.next();
                    if ((s != null) && (s.isProper(false) == null)) try {
                        seedString = s.genSeedStr(key);
                        if (seedString != null) {
                            seeds.append("seed").append(count).append('=').append(seedString).append(serverCore.CRLF_STRING);
                            count++;
                        }
                    } catch (final ConcurrentModificationException e) {
View Full Code Here

        long   nwords  = sb.peers.countActiveRWI();
        final double nqpm    = sb.peers.countActiveQPM();
        long   nppm    = sb.peers.countActivePPM();
        double nqph    = 0;

        final Seed seed = sb.peers.mySeed();
        if (seed != null){
            name    = seed.get(Seed.NAME, "-").toUpperCase();
            links   = seed.getLinkCount();
            words   = seed.getWordCount();
            myppm   = seed.getPPM();
            myqph   = 60d * seed.getQPM();

            if (sb.peers.mySeed().isVirgin()) {
                type = "VIRGIN";
                nqph = Math.round(6000d * nqpm) / 100d;
            } else if(sb.peers.mySeed().isJunior()) {
View Full Code Here

        // response values
        String result = "";
        String doublevalues = "0";

        final Seed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

        if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) {
            Network.log.logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash);
            result = "wrong_target";
        } else if ((!granted) || (sb.isRobinsonMode())) {
View Full Code Here

        EventTracker.delete(EventTracker.EClass.SEARCH);
        final HandleSet abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : QueryParams.hashes2Set(abstracts);

        // store accessing peer
        Seed remoteSeed;
        try {
            remoteSeed = Seed.genRemoteSeed(oseed, key, false, client);
        } catch (final IOException e) {
            Network.log.logInfo("yacy.search: access with bad seed: " + e.getMessage());
            remoteSeed = null;
        }
        if (sb.peers == null) {
            Network.log.logSevere("yacy.search: seed cache not initialized");
        } else {
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        }

        // prepare search
        final HandleSet queryhashes = QueryParams.hashes2Set(query);
        final HandleSet excludehashes = (exclude.length() == 0) ? new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0) : QueryParams.hashes2Set(exclude);
        final long timestamp = System.currentTimeMillis();

      // prepare a search profile
        final RankingProfile rankingProfile = (profile.length() == 0) ? new RankingProfile(ContentDomain.contentdomParser(contentdom)) : new RankingProfile("", profile);

        // prepare an abstract result
        final StringBuilder indexabstract = new StringBuilder(6000);
        int indexabstractContainercount = 0;
        int joincount = 0;
        QueryParams theQuery = null;
        SearchEvent theSearch = null;
        ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
        if (query.length() == 0 && abstractSet != null) {
            // this is _not_ a normal search, only a request for index abstracts
            final Segment indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
            theQuery = new QueryParams(
                    null,
                    abstractSet,
                    new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0),
                    null,
                    snippetPattern,
                    null,
                    modifier,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.Searchdom.LOCAL,
                    -1,
                    null,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    indexSegment,
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");

            final long timer = System.currentTimeMillis();
            //final Map<byte[], ReferenceContainer<WordReference>>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls));
            final TreeMap<byte[], ReferenceContainer<WordReference>> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2Handles(urls));

            EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false);
            if (incc != null) {
                final Iterator<Map.Entry<byte[], ReferenceContainer<WordReference>>> ci = incc.entrySet().iterator();
                Map.Entry<byte[], ReferenceContainer<WordReference>> entry;
                byte[] wordhash;
                while (ci.hasNext()) {
                    entry = ci.next();
                    wordhash = entry.getKey();
                    final ReferenceContainer<WordReference> container = entry.getValue();
                    indexabstractContainercount += container.size();
                    indexabstract.append("indexabstract.");
                    indexabstract.append(ASCII.String(wordhash));
                    indexabstract.append("=");
                    indexabstract.append(WordReferenceFactory.compressIndex(container, null, 1000).toString());
                    indexabstract.append(serverCore.CRLF_STRING);
                }
            }

            prop.put("indexcount", "");
            prop.put("joincount", "0");
            prop.put("references", "");

        } else {
            // retrieve index containers from search request
            theQuery = new QueryParams(
                    null,
                    queryhashes,
                    excludehashes,
                    null,
                    snippetPattern,
                    null,
                    modifier,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.Searchdom.LOCAL,
                    -1,
                    constraint,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    sb.indexSegments.segment(Segments.Process.PUBLIC),
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
            EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));

            // make event
            theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));

            // set statistic details of search result and find best result index set
View Full Code Here

TOP

Related Classes of net.yacy.peers.Seed

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.