Examples of Seed


Examples of net.yacy.peers.Seed

        final serverObjects prop = new serverObjects();

        if (post != null) {
            if (post.containsKey("retrieve")) {
                final String peerhash = post.get("peer", null);
                final Seed seed = (peerhash == null) ? null : sb.peers.getConnected(peerhash);
                final RSSFeed feed = (seed == null) ? null : Protocol.queryRemoteCrawlURLs(sb.peers, seed, 20, 60000);
                if (feed != null) {
                    for (final Hit item: feed) {
                        //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate());
                       
View Full Code Here

Examples of net.yacy.peers.Seed

        return (url == null ? "null" : url.toNormalform(true, false));
    }
   
    private static void listHosts(final Switchboard sb, final serverObjects prop) {
        // list known hosts
        Seed seed;
        int hc = 0;
        if (sb.peers != null && sb.peers.sizeConnected() > 0) {
            final Iterator<Seed> e = PeerSelection.getProvidesRemoteCrawlURLs(sb.peers);
            while (e.hasNext()) {
                seed = e.next();
                if (seed != null) {
                    prop.put("hosts_" + hc + "_hosthash", seed.hash);
                    prop.putHTML("hosts_" + hc + "_hostname", seed.hash + " " + seed.get(Seed.NAME, "nameless") + " (" + seed.getLong(Seed.RCOUNT, 0) + ")");
                    hc++;
                }
            }
            prop.put("hosts", hc);
        } else {
View Full Code Here

Examples of net.yacy.peers.Seed

        if (seed.length() > Seed.maxsize) {
          Network.log.logInfo("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")");
            prop.put("message", "your seed is too long (" + seed.length() + ")");
            return prop;
        }
        Seed remoteSeed;
        try {
            remoteSeed = Seed.genRemoteSeed(seed, key, true, ias.getHostAddress());
        } catch (final IOException e) {
            Network.log.logInfo("hello/server: bad seed: " + e.getMessage() + ", time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "bad seed: " + e.getMessage());
            return prop;
        }

        if (remoteSeed == null || remoteSeed.hash == null) {
            Network.log.logInfo("hello/server: bad seed: null, time_dnsResolve=" + time_dnsResolve);
            prop.put("message", "cannot parse your seed");
            return prop;
        }

//      final String properTest = remoteSeed.isProper();
        // The remote peer might not know its IP yet, so don't abort if the IP check fails
//      if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }

        // we easily know the caller's IP:
        final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
        final String reportedip = remoteSeed.getIP();
        final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
        final float clientversion = remoteSeed.getVersion();

        if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
          // if we are a robinson cluster, answer only if this client is known by our network definition
            prop.put("message", "I am robinson, I do not answer");
            return prop;
        }

        long[] callback = new long[]{-1, -1};
        if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));

        // if the remote client has reported its own IP address and the client supports
        // the port forwarding feature (if client version >= 0.383) then we try to
        // connect to the reported IP address first
        time = System.currentTimeMillis();
        long time_backping = 0;
        String backping_method = "none";
        if (reportedip.length() > 0 &&
            !clientip.equals(reportedip) &&
            clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING &&
            magic != 0) {
            serverCore.checkInterruption();

            // try first the reportedip, since this may be a connect from a port-forwarding host
            prop.put("yourip", reportedip);
            remoteSeed.setIP(reportedip);
            callback = Protocol.queryUrlCount(remoteSeed);
            time_backping = System.currentTimeMillis() - time;
            backping_method = "reportedip=" + reportedip;
        } else {
            prop.put("yourip", ias.getHostAddress());
            remoteSeed.setIP(ias.getHostAddress());
        }

        // if the previous attempt (using the reported ip address) was not successful,
        // then try the ip where the request came from
        if (callback[0] < 0 || (magic != 0 && magic != callback[1])) {
            boolean isNotLocal = true;

            // we are only allowed to connect to the client IP address if it's not our own address
            if (serverCore.useStaticIP) {
                    isNotLocal = !ias.isSiteLocalAddress();
            }

            if (isNotLocal) {
                serverCore.checkInterruption();

                prop.put("yourip", clientip);
                remoteSeed.setIP(clientip);
                callback = Protocol.queryUrlCount(remoteSeed);
                time_backping = System.currentTimeMillis() - time;
                backping_method = "clientip=" + clientip;
            }
        }

//      System.out.println("YACYHELLO: YOUR IP=" + clientip);
        // set lastseen value (we have seen that peer, it contacted us!)
        remoteSeed.setLastSeenUTC();

        // assign status
        final int connectedBefore = sb.peers.sizeConnected();
        if (callback[0] >= 0) {
            if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
                remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
            } else if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL).equals(Seed.PEERTYPE_PRINCIPAL)) {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_PRINCIPAL);
            } else {
                prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
                remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
            }
            // connect the seed
            Network.log.logInfo("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        } else {
            prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR);
            remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
            Network.log.logInfo("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
            // no connection here, instead store junior in connection cache
            if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) {
                sb.peers.peerActions.peerPing(remoteSeed);
            }
        }
        final int connectedAfter = sb.peers.sizeConnected();

        // update event tracker
        EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);

        sb.peers.peerActions.setUserAgent(clientip, userAgent);
        if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) {
            Network.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() +
                                                           "' [" + reportedip +
                                             "] peerType from '" + reportedPeerType +
                                                        "' to '" + prop.get(Seed.YOURTYPE) + "'.");
        }

        serverCore.checkInterruption();
        final StringBuilder seeds = new StringBuilder(768);
        // attach some more seeds, as requested
        if (sb.peers.sizeConnected() > 0) {
            if (count > sb.peers.sizeConnected()) { count = sb.peers.sizeConnected(); }
            if (count > 100) { count = 100; }

            // latest seeds
            final Map<String, Seed> ySeeds = PeerSelection.seedsByAge(sb.peers, true, count); // peerhash/yacySeed relation

            // attach also my own seed
            seeds.append("seed0=").append(sb.peers.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
            count = 1;

            // attach other seeds
            if (ySeeds != null) {
                seeds.ensureCapacity((ySeeds.size() + 1) * 768);
                final Iterator<Seed> si = ySeeds.values().iterator();
                Seed s;
                String seedString;
                while (si.hasNext()) {
                  s = si.next();
                    if ((s != null) && (s.isProper(false) == null)) try {
                        seedString = s.genSeedStr(key);
                        if (seedString != null) {
                            seeds.append("seed").append(count).append('=').append(seedString).append(serverCore.CRLF_STRING);
                            count++;
                        }
                    } catch (final ConcurrentModificationException e) {
View Full Code Here

Examples of net.yacy.peers.Seed

        long   nwords  = sb.peers.countActiveRWI();
        final double nqpm    = sb.peers.countActiveQPM();
        long   nppm    = sb.peers.countActivePPM();
        double nqph    = 0;

        final Seed seed = sb.peers.mySeed();
        if (seed != null){
            name    = seed.get(Seed.NAME, "-").toUpperCase();
            links   = seed.getLinkCount();
            words   = seed.getWordCount();
            myppm   = seed.getPPM();
            myqph   = 60d * seed.getQPM();

            if (sb.peers.mySeed().isVirgin()) {
                type = "VIRGIN";
                nqph = Math.round(6000d * nqpm) / 100d;
            } else if(sb.peers.mySeed().isJunior()) {
View Full Code Here

Examples of net.yacy.peers.Seed

        // response values
        String result = "";
        String doublevalues = "0";

        final Seed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

        if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) {
            Network.log.logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash);
            result = "wrong_target";
        } else if ((!granted) || (sb.isRobinsonMode())) {
View Full Code Here

Examples of net.yacy.peers.Seed

        EventTracker.delete(EventTracker.EClass.SEARCH);
        final HandleSet abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : QueryParams.hashes2Set(abstracts);

        // store accessing peer
        Seed remoteSeed;
        try {
            remoteSeed = Seed.genRemoteSeed(oseed, key, false, client);
        } catch (final IOException e) {
            Network.log.logInfo("yacy.search: access with bad seed: " + e.getMessage());
            remoteSeed = null;
        }
        if (sb.peers == null) {
            Network.log.logSevere("yacy.search: seed cache not initialized");
        } else {
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        }

        // prepare search
        final HandleSet queryhashes = QueryParams.hashes2Set(query);
        final HandleSet excludehashes = (exclude.length() == 0) ? new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0) : QueryParams.hashes2Set(exclude);
        final long timestamp = System.currentTimeMillis();

      // prepare a search profile
        final RankingProfile rankingProfile = (profile.length() == 0) ? new RankingProfile(ContentDomain.contentdomParser(contentdom)) : new RankingProfile("", profile);

        // prepare an abstract result
        final StringBuilder indexabstract = new StringBuilder(6000);
        int indexabstractContainercount = 0;
        int joincount = 0;
        QueryParams theQuery = null;
        SearchEvent theSearch = null;
        ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
        if (query.length() == 0 && abstractSet != null) {
            // this is _not_ a normal search, only a request for index abstracts
            final Segment indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
            theQuery = new QueryParams(
                    null,
                    abstractSet,
                    new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0),
                    null,
                    snippetPattern,
                    null,
                    modifier,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.Searchdom.LOCAL,
                    -1,
                    null,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    indexSegment,
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");

            final long timer = System.currentTimeMillis();
            //final Map<byte[], ReferenceContainer<WordReference>>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls));
            final TreeMap<byte[], ReferenceContainer<WordReference>> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2Handles(urls));

            EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false);
            if (incc != null) {
                final Iterator<Map.Entry<byte[], ReferenceContainer<WordReference>>> ci = incc.entrySet().iterator();
                Map.Entry<byte[], ReferenceContainer<WordReference>> entry;
                byte[] wordhash;
                while (ci.hasNext()) {
                    entry = ci.next();
                    wordhash = entry.getKey();
                    final ReferenceContainer<WordReference> container = entry.getValue();
                    indexabstractContainercount += container.size();
                    indexabstract.append("indexabstract.");
                    indexabstract.append(ASCII.String(wordhash));
                    indexabstract.append("=");
                    indexabstract.append(WordReferenceFactory.compressIndex(container, null, 1000).toString());
                    indexabstract.append(serverCore.CRLF_STRING);
                }
            }

            prop.put("indexcount", "");
            prop.put("joincount", "0");
            prop.put("references", "");

        } else {
            // retrieve index containers from search request
            theQuery = new QueryParams(
                    null,
                    queryhashes,
                    excludehashes,
                    null,
                    snippetPattern,
                    null,
                    modifier,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.Searchdom.LOCAL,
                    -1,
                    constraint,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    sb.indexSegments.segment(Segments.Process.PUBLIC),
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
            EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));

            // make event
            theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));

            // set statistic details of search result and find best result index set
View Full Code Here

Examples of net.yacy.peers.Seed

            // open an editor page for the message
            // first ask if the other peer is online, and also what kind of document it accepts
            final Map<String, String> result = Protocol.permissionMessage(sb.peers, hash);
            //System.out.println("DEBUG: permission request result = " + result.toString());
            String peerName;
            Seed targetPeer = null;
            if (hash.equals(sb.peers.mySeed().hash)) {
                peerName = sb.peers.mySeed().get(Seed.NAME,"nameless");
            } else {
                targetPeer = sb.peers.getConnected(hash);
                if (targetPeer == null)
                    peerName = "nameless";
                else
                    peerName = targetPeer.get(Seed.NAME,"nameless");
            }

            prop.putXML("mode_permission_peerName", peerName);
            final String response = (result == null) ? null : result.get("response");
            if (response == null || result == null) {
View Full Code Here

Examples of net.yacy.peers.Seed

        final String col = post.get("col", "");
        final File listsPath = env.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);

        String otherPeerName = null;
        if (post.containsKey("iam")) {
            final Seed bla = sb.peers.get(post.get("iam", ""));
            if (bla != null) otherPeerName = bla.getName();
        }
        if (otherPeerName == null) otherPeerName = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP);

        if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) {
            // if we are a robinson cluster, answer only if this client is known by our network definition
View Full Code Here

Examples of net.yacy.peers.Seed

                prop.put("page_list_" + m + "_qph", qph);
               
                prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
                prop.putHTML("page_list_" + m + "_host", host);
                if (page == 5) {
                    final Seed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), true, true, true);
                    prop.putHTML("page_list_" + m + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
                }
                prop.putNum("page_list_" + m + "_count", handles.size());

                // next
                m++;
View Full Code Here

Examples of net.yacy.peers.Seed

            prop.put("loader-set", "0");
        } else {
            prop.put("loader-set", "1");
            boolean dark = true;
            final Request[] w = sb.crawlQueues.activeWorkerEntries();
            Seed initiator;
            int count = 0;
            for (int i = 0; i < w.length; i++)  {
                if (w[i] == null) continue;
               
                initiator = sb.peers.getConnected((w[i].initiator() == null) ? "" : ASCII.String(w[i].initiator()));
                prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
                prop.putHTML("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
                prop.put("loader-set_list_"+count+"_depth", w[i].depth());
                prop.put("loader-set_list_"+count+"_status", w[i].getStatus());
                prop.putHTML("loader-set_list_"+count+"_url", w[i].url().toNormalform(true, false));
                dark = !dark;
                count++;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.