Package net.yacy.kelondro.rwi

Examples of net.yacy.kelondro.rwi.Reference

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
                index = segment.termIndex().get(keyhash, null);
                // built urlCache
                final Iterator<WordReference> urlIter = index.entries();
                final TreeMap<byte[], URIMetadataRow> knownURLs = new TreeMap<byte[], URIMetadataRow>(Base64Order.enhancedCoder);
                final HandleSet unknownURLEntries = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, index.size());
                Reference iEntry;
                URIMetadataRow lurl;
                while (urlIter.hasNext()) {
                    iEntry = urlIter.next();
                    lurl = segment.urlMetadata().load(iEntry.urlhash());
                    if (lurl == null) {
                        try {
                            unknownURLEntries.put(iEntry.urlhash());
                        } catch (final RowSpaceExceededException e) {
                            Log.logException(e);
                        }
                        urlIter.remove();
                    } else {
                        knownURLs.put(iEntry.urlhash(), lurl);
                    }
                }

                // make an indexContainerCache
                final ReferenceContainerCache<WordReference> icc = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength);
View Full Code Here
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
                if (yacyCore.log.isInfo()) yacyCore.log.logInfo("remote search: rejected url '" + metadata.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
                continue; // reject url outside of our domain
            }

            // save the url entry
            final Reference entry = urlEntry.word();
            if (entry == null) {
                if (yacyCore.log.isWarning()) yacyCore.log.logWarning("remote search: no word attached from peer " + target.getName() + ", version " + target.getVersion());
                continue; // no word attached
            }

            // the search-result-url transports all the attributes of word indexes
            if (!Base64Order.enhancedCoder.equal(entry.urlhash(), urlEntry.hash())) {
                yacyCore.log.logInfo("remote search: url-hash " + ASCII.String(urlEntry.hash()) + " does not belong to word-attached-hash " + ASCII.String(entry.urlhash()) + "; url = " + metadata.url() + " from peer " + target.getName());
                continue; // spammed
            }

            // passed all checks, store url
            try {
View Full Code Here
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
            final boolean gzipBody,
            final int timeout) {

        // check if we got all necessary urls in the urlCache (only for debugging)
        Iterator<WordReference> eenum;
        Reference entry;
        for (final ReferenceContainer<WordReference> ic: indexes) {
            eenum = ic.entries();
            while (eenum.hasNext()) {
                entry = eenum.next();
                if (urlCache.get(entry.urlhash()) == null) {
                    if (yacyCore.log.isFine()) yacyCore.log.logFine("DEBUG transferIndex: to-send url hash '" + ASCII.String(entry.urlhash()) + "' is not contained in urlCache");
                }
            }
        }

        // transfer the RWI without the URLs
View Full Code Here
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
        }

        int indexcount = 0;
        final StringBuilder entrypost = new StringBuilder(indexes.size() * 73);
        Iterator<WordReference> eenum;
        Reference entry;
        for (final ReferenceContainer<WordReference> ic: indexes) {
            eenum = ic.entries();
            while (eenum.hasNext()) {
                entry = eenum.next();
                entrypost.append(ASCII.String(ic.getTermHash()))
                         .append(entry.toPropertyForm())
                         .append(serverCore.CRLF_STRING);
                indexcount++;
            }
        }

View Full Code Here
173
174
175
176
177
178
179
180
181
    }
    public float lon() {
        return urlentry.metadata().lon();
    }
    public WordReferenceVars word() {
        final Reference word = urlentry.word();
        assert word instanceof WordReferenceVars;
        return (WordReferenceVars) word;
    }
View Full Code Here
657
658
659
660
661
662
663
664
665
666
667
668
669
670
                    wordCounter++;
                    wordIdxContainer = indexContainerIterator.next();

                    // the combined container will fit, read the container
                    final Iterator<WordReference> wordIdxEntries = wordIdxContainer.entries();
                    Reference iEntry;
                    while (wordIdxEntries.hasNext()) {
                        iEntry = wordIdxEntries.next();
                        final byte[] urlHash = iEntry.urlhash();
                        if ((currentUrlDB.exists(urlHash)) && (!minimizedUrlDB.exists(urlHash))) try {
                            final URIMetadataRow urlEntry = currentUrlDB.load(urlHash);
                            urlCounter++;
                            minimizedUrlDB.store(urlEntry);
                            if (urlCounter % 500 == 0) {
View Full Code Here
666
667
668
669
670
671
672
673
674
675
676
677
678
679
                    wordCounter++;
                    wordIdxContainer = indexContainerIterator.next();

                    // the combined container will fit, read the container
                    final Iterator<WordReference> wordIdxEntries = wordIdxContainer.entries();
                    Reference iEntry;
                    while (wordIdxEntries.hasNext()) {
                        iEntry = wordIdxEntries.next();
                        final byte[] urlHash = iEntry.urlhash();
                        if ((currentUrlDB.exists(urlHash)) && (!minimizedUrlDB.exists(urlHash))) try {
                            final URIMetadataRow urlEntry = currentUrlDB.load(urlHash);
                            urlCounter++;
                            minimizedUrlDB.store(urlEntry);
                            if (urlCounter % 500 == 0) {
View Full Code Here
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
                index = segment.termIndex().get(keyhash, null);
                // built urlCache
                final Iterator<WordReference> urlIter = index.entries();
                final TreeMap<byte[], URIMetadataRow> knownURLs = new TreeMap<byte[], URIMetadataRow>(Base64Order.enhancedCoder);
                final HandleSet unknownURLEntries = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, index.size());
                Reference iEntry;
                URIMetadataRow lurl;
                while (urlIter.hasNext()) {
                    iEntry = urlIter.next();
                    lurl = segment.urlMetadata().load(iEntry.urlhash());
                    if (lurl == null) {
                        try {
                            unknownURLEntries.put(iEntry.urlhash());
                        } catch (final RowSpaceExceededException e) {
                            Log.logException(e);
                        }
                        urlIter.remove();
                    } else {
                        knownURLs.put(iEntry.urlhash(), lurl);
                    }
                }

                // make an indexContainerCache
                final ReferenceContainerCache<WordReference> icc = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength);
View Full Code Here
174
175
176
177
178
179
180
181
182
    }
    public float lon() {
        return urlentry.metadata().lon();
    }
    public WordReferenceVars word() {
        final Reference word = urlentry.word();
        assert word instanceof WordReferenceVars;
        return (WordReferenceVars) word;
    }
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
                if (Network.log.isInfo()) Network.log.logInfo("remote search: rejected url '" + metadata.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
                continue; // reject url outside of our domain
            }

            // save the url entry
            final Reference entry = urlEntry.word();
            if (entry == null) {
                if (Network.log.isWarning()) Network.log.logWarning("remote search: no word attached from peer " + target.getName() + ", version " + target.getVersion());
                continue; // no word attached
            }

            // the search-result-url transports all the attributes of word indexes
            if (!Base64Order.enhancedCoder.equal(entry.urlhash(), urlEntry.hash())) {
                Network.log.logInfo("remote search: url-hash " + ASCII.String(urlEntry.hash()) + " does not belong to word-attached-hash " + ASCII.String(entry.urlhash()) + "; url = " + metadata.url() + " from peer " + target.getName());
                continue; // spammed
            }

            // passed all checks, store url
            try {
View Full Code Here
TOP

Related Classes of net.yacy.kelondro.rwi.Reference

  • de.anomic.search.ResultEntry
  • de.anomic.yacy.yacyClient
  • IndexControlRWIs_p
  • net.yacy.peers.Protocol
  • net.yacy.search.snippet.ResultEntry
  • net.yacy.yacy
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.