Package net.yacy.peers

Examples of net.yacy.peers.Seed


        // the seedcount is the maximum number of wanted results
        if (seedDB == null) { return null; }
       
        // put in seeds according to dht
        final Map<String, Seed> regularSeeds = new HashMap<String, Seed>(); // dht position seeds
        Seed seed;
        Iterator<Seed> dhtEnum;        
        Iterator<byte[]> iter = wordhashes.iterator();
        while (iter.hasNext()) {
            selectDHTPositions(seedDB, iter.next(), redundancy, regularSeeds);
        }
        //int minimumseeds = Math.min(seedDB.scheme.verticalPartitions(), regularSeeds.size()); // that should be the minimum number of seeds that are returned
        //int maximumseeds = seedDB.scheme.verticalPartitions() * redundancy; // this is the maximum number of seeds according to dht and heuristics. It can be more using burst mode.
       
        // put in some seeds according to size of peer.
        // But not all, that would produce too much load on the largest peers
        dhtEnum = seedDB.seedsSortedConnected(false, Seed.ICOUNT);
        int c = Math.max(Math.min(5, seedDB.sizeConnected()), wordhashes.size() > 1 ? seedDB.sizeConnected() * burstMultiwordPercent / 100 : 0);
        while (dhtEnum.hasNext() && c-- > 0) {
            seed = dhtEnum.next();
            if (seed == null) continue;
            if (seed.isLastSeenTimeout(3600000)) continue;
            if (seed.getAge() < 1) { // the 'workshop feature'
                Log.logInfo("DHT", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge());
                regularSeeds.put(seed.hash, seed);
                continue;
            }
            if (Math.random() * 100 + (wordhashes.size() > 1 ? burstMultiwordPercent : 25) >= 50) {
                if (Log.isFine("DHT")) Log.logFine("DHT", "selectPeers/CountBurst: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.getWordCount());
                regularSeeds.put(seed.hash, seed);
                continue;
            }
        }

        // create a set that contains only robinson peers because these get a special handling
        dhtEnum = seedDB.seedsConnected(true, false, null, 0.50f);
        Set<Seed> robinson = new HashSet<Seed>();
        while (dhtEnum.hasNext()) {
            seed = dhtEnum.next();
            if (seed == null) continue;
            if (seed.getFlagAcceptRemoteIndex()) continue;
            if (seed.isLastSeenTimeout(3600000)) continue;
            robinson.add(seed);
        }

        // add robinson peers according to robinson burst rate
        dhtEnum = robinson.iterator();
        c = robinson.size() * burstRobinsonPercent / 100;
        while (dhtEnum.hasNext() && c-- > 0) {
            seed = dhtEnum.next();
            if (seed == null) continue;
            if (seed.isLastSeenTimeout(3600000)) continue;
            if (Math.random() * 100 + burstRobinsonPercent >= 100) {
                if (Log.isFine("DHT")) Log.logFine("DHT", "selectPeers/RobinsonBurst: " + seed.hash + ":" + seed.getName());
                regularSeeds.put(seed.hash, seed);
                continue;
            }
        }

        // put in seeds that are public robinson peers and where the peer tags match with query
        // or seeds that are newbies to ensure that private demonstrations always work
        dhtEnum = robinson.iterator();
        while (dhtEnum.hasNext()) {
            seed = dhtEnum.next();
            if (seed == null) continue;
            if (seed.isLastSeenTimeout(3600000)) continue;
            if (seed.matchPeerTags(wordhashes)) {
                // peer tags match
                String specialized = seed.getPeerTags().toString();
                if (specialized.equals("[*]")) {
                    Log.logInfo("DHT", "selectPeers/RobinsonTag: " + seed.hash + ":" + seed.getName() + " grants search for all");
                } else {
                    Log.logInfo("DHT", "selectPeers/RobinsonTag " + seed.hash + ":" + seed.getName() + " is specialized peer for " + specialized);
                }
                regularSeeds.put(seed.hash, seed);
            }
        }
       
View Full Code Here


            byte[] wordhash,
            int redundancy,
            Map<String, Seed> regularSeeds) {
        // this method is called from the search target computation
        final long[] dhtVerticalTargets = seedDB.scheme.dhtPositions(wordhash);
        Seed seed;
        for (long  dhtVerticalTarget : dhtVerticalTargets) {
            wordhash = FlatWordPartitionScheme.positionToHash(dhtVerticalTarget);
            Iterator<Seed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, wordhash, redundancy, false);
            int c = Math.min(seedDB.sizeConnected(), redundancy);
            int cc = 2; // select a maximum of 3, this is enough redundancy
            while (dhtEnum.hasNext() && c > 0 && cc-- > 0) {
                seed = dhtEnum.next();
                if (seed == null || seed.hash == null) continue;
                if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer
                if (Log.isFine("DHT")) Log.logFine("DHT", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/ score " + c);
                regularSeeds.put(seed.hash, seed);
                c--;
            }
        }
    }
View Full Code Here

       
        if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected();

        // fill a score object
        final ScoreMap<String> seedScore = new ConcurrentScoreMap<String>();
        Seed ys;
        long absage;
        final Iterator<Seed> s = seedDB.seedsSortedConnected(!up, Seed.LASTSEEN);
        int searchcount = 1000;
        if (searchcount > seedDB.sizeConnected()) searchcount = seedDB.sizeConnected();
        try {
            while ((s.hasNext()) && (searchcount-- > 0)) {
                ys = s.next();
                if ((ys != null) && (ys.get(Seed.LASTSEEN, "").length() > 10)) try {
                    absage = Math.abs(System.currentTimeMillis() + AbstractFormatter.dayMillis - ys.getLastSeenUTC()) / 1000 / 60;
                    if (absage > Integer.MAX_VALUE) absage = Integer.MAX_VALUE;
                    seedScore.inc(ys.hash, (int) absage); // the higher absage, the older is the peer
                } catch (final Exception e) {}
            }
           
View Full Code Here

            return nextSeed != null;
        }

        private Seed nextInternal() {
            if (this.remaining <= 0) return null;
            Seed s;
            try {
                while (se.hasNext()) {
                    s = se.next();
                    if (s == null) return null;
                    byte[] hashb = ASCII.getBytes(s.hash);
                    if (doublecheck.has(hashb)) return null;
                    try {
                        this.doublecheck.put(hashb);
                    } catch (RowSpaceExceededException e) {
                        Log.logException(e);
                        break;
                    }
                    if (s.getFlagAcceptRemoteIndex() ||
                        (alsoMyOwn && s.hash.equals(seedDB.mySeed().hash)) // Accept own peer regardless of FlagAcceptRemoteIndex
                       ) {
                        this.remaining--;
                        return s;
                    }
View Full Code Here

            }
            return null;
        }
       
        public Seed next() {
            final Seed next = nextSeed;
            nextSeed = nextInternal();
            return next;
        }
View Full Code Here

               ) {
                // take my own seed hash instead the enumeration result
                alsoMyOwn = false;
                return seedDB.mySeed();
            } else {
                final Seed next = nextSeed;
                nextSeed = nextInternal();
                return next;
            }
        }
View Full Code Here

        public boolean hasNext() {
            return nextSeed != null;
        }

        private Seed nextInternal() {
            Seed s;
            try {
                while (se.hasNext()) {
                    s = se.next();
                    if (s == null) return null;
                    if (s.getLong(Seed.RCOUNT, 0) > 0) return s;
                }
            } catch (final kelondroException e) {
                System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage());
                Network.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
                seedDB.resetActiveTable();
View Full Code Here

            }
            return null;
        }
       
        public Seed next() {
            final Seed next = nextSeed;
            nextSeed = nextInternal();
            return next;
        }
View Full Code Here

            return this.targets.size();
        }

        public boolean transmit() {
            if (this.targets.isEmpty()) return false;
            final Seed target = this.targets.remove(0);
            // transferring selected words to remote peer
            if (target == Transmission.this.seeds.mySeed() || target.hash.equals(Transmission.this.seeds.mySeed().hash)) {
              // target is my own peer. This is easy. Just restore the indexContainer
              restore();
              this.hit++;
              Transmission.this.log.logInfo("Transfer of chunk to myself-target");
              return true;
            }
            Transmission.this.log.logInfo("starting new index transmission request to " + ASCII.String(this.primaryTarget));
            final long start = System.currentTimeMillis();
            final String error = Protocol.transferIndex(target, this.containers, this.references, Transmission.this.gzipBody4Transfer, Transmission.this.timeout4Transfer);
            if (error == null) {
                // words successfully transfered
                final long transferTime = System.currentTimeMillis() - start;
                final Iterator<ReferenceContainer<WordReference>> i = this.containers.iterator();
                final ReferenceContainer<WordReference> firstContainer = (i == null) ? null : i.next();
                Transmission.this.log.logInfo("Index transfer of " + this.containers.size() +
                                 " words [" + ((firstContainer == null) ? null : ASCII.String(firstContainer.getTermHash())) + " .. " + ASCII.String(this.primaryTarget) + "]" +
                                 " and " + this.references.size() + " URLs" +
                                 " to peer " + target.getName() + ":" + target.hash +
                                 " in " + (transferTime / 1000) +
                                 " seconds successful ("  + (1000 * this.containers.size() / (transferTime + 1)) +
                                 " words/s)");
                Transmission.this.seeds.mySeed().incSI(this.containers.size());
                Transmission.this.seeds.mySeed().incSU(this.references.size());
                // if the peer has set a pause time and we are in flush mode (index transfer)
                // then we pause for a while now
                Transmission.this.log.logInfo("Transfer finished of chunk to target " + target.hash + "/" + target.getName());
                this.hit++;
                return true;
            }
            this.miss++;
            // write information that peer does not receive index transmissions
            Transmission.this.log.logInfo("Transfer failed of chunk to target " + target.hash + "/" + target.getName() + ": " + error);
            // get possibly newer target Info
            final Seed newTarget = Transmission.this.seeds.get(target.hash);
            if (newTarget != null) {
                final String oldAddress = target.getPublicAddress();
                if ((oldAddress != null) && (oldAddress.equals(newTarget.getPublicAddress()))) {
                    newTarget.setFlagAcceptRemoteIndex(false);
                    Transmission.this.seeds.update(newTarget.hash, newTarget);
                } else {
                    // we tried an old Address. Don't change anything
                }
            } else {
View Full Code Here

        final String host = urlcomps.url().getHost();
        if (host != null && host.endsWith(".yacyh")) {
            // translate host into current IP
            int p = host.indexOf('.');
            final String hash = Seed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6));
            final Seed seed = peers.getConnected(hash);
            final String filename = urlcomps.url().getFile();
            String address = null;
            if ((seed == null) || ((address = seed.getPublicAddress()) == null)) {
                // seed is not known from here
                try {
                    indexSegment.termIndex().remove(
                        Word.words2hashesHandles(Condenser.getWords(
                            ("yacyshare " +
                             filename.replace('?', ' ') +
                             " " +
                             urlcomps.dc_title()), null).keySet()),
                             urlentry.hash());
                } catch (IOException e) {
                    Log.logException(e);
                }
                indexSegment.urlMetadata().remove(urlentry.hash()); // clean up
                throw new RuntimeException("index void");
            }
            alternative_urlstring = "http://" + address + "/" + host.substring(0, p) + filename;
            alternative_urlname = "http://share." + seed.getName() + ".yacy" + filename;
            if ((p = alternative_urlname.indexOf('?')) > 0) alternative_urlname = alternative_urlname.substring(0, p);
        }
    }
View Full Code Here

TOP

Related Classes of net.yacy.peers.Seed

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.