Package net.yacy.kelondro.index

Examples of net.yacy.kelondro.index.HandleSet

105
106
107
108
109
110
111
112
        this.cachedUrlHashs = new HashMap<String, HandleSet>();

        for (final String blacklistType : BLACKLIST_TYPES) {
            this.hostpaths_matchable.put(blacklistType, new HashMap<String, List<String>>());
            this.hostpaths_notmatchable.put(blacklistType, new HashMap<String, List<String>>());
            this.cachedUrlHashs.put(blacklistType, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
        }
    }
View Full Code Here
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
        }

        if (url.getHost() == null) {
            return false;
        }
        final HandleSet urlHashCache = getCacheUrlHashsSet(blacklistType);
        if (!urlHashCache.has(url.hash())) {
            final boolean temp = isListed(blacklistType, url.getHost().toLowerCase(), url.getFile());
            if (temp) {
                try {
                    urlHashCache.put(url.hash());
                } catch (final RowSpaceExceededException e) {
                    Log.logException(e);
                }
            }
            return temp;
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        }
        return h;
    }

    public static final HandleSet words2hashesHandles(final Set<String> words) {
        final HandleSet hashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, words.size());
        for (final String word: words)
            try {
                hashes.put(word2hash(word));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
                return hashes;
            }
        return hashes;
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
            final RankingProfile ranking,
            final String userAgent) {

      if ((queryString.length() == 12) && (Base64Order.enhancedCoder.wellformed(UTF8.getBytes(queryString)))) {
            this.queryString = null;
            this.queryHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
            this.excludeHashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
            try {
                this.queryHashes.put(UTF8.getBytes(queryString));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
            }
        return hashes;
    }

    public static final HandleSet words2hashesHandles(final String[] words) {
        final HandleSet hashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, words.length);
        for (final String word: words)
            try {
                hashes.put(word2hash(word));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
                return hashes;
            }
        return hashes;
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
298
299
300
    public boolean isLocal() {
        return this.domType == Searchdom.LOCAL;
    }

    public static HandleSet hashes2Set(final String query) {
        final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
        if (query != null) {
            for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
                keyhashes.put(ASCII.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
        }
        return keyhashes;
View Full Code Here
299
300
301
302
303
304
305
306
307
308
309
310
311
312
        }
        return keyhashes;
    }

    public static HandleSet hashes2Handles(final String query) {
        final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
        if (query != null) {
            for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
                keyhashes.put(ASCII.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
        }
        return keyhashes;
View Full Code Here
347
348
349
350
351
352
353
354
355
356
357
     * @param text
     * @return true if the query matches with the given text
     */
    public final boolean matchesText(final String text) {
        boolean ret = false;
        final HandleSet wordhashes = Word.words2hashesHandles(Condenser.getWords(text, null).keySet());
        if (!SetTools.anymatch(wordhashes, this.excludeHashes)) {
            ret = SetTools.totalInclusion(this.queryHashes, wordhashes);
        }
        return ret;
    }
View Full Code Here
357
358
359
360
361
362
363
364
    }

    public static final boolean anymatch(final String text, final HandleSet keyhashes) {
      // returns true if any of the word hashes in keyhashes appear in the String text
      // to do this, all words in the string must be recognized and transcoded to word hashes
      final HandleSet wordhashes = Word.words2hashesHandles(Condenser.getWords(text, null).keySet());
      return SetTools.anymatch(wordhashes, keyhashes);
    }
View Full Code Here
431
432
433
434
435
436
437
438
    }

    public void filterOut(final SortedSet<String> blueList) {
        // filter out words that appear in this set
      // this is applied to the queryHashes
      final HandleSet blues = Word.words2hashesHandles(blueList);
      for (final byte[] b: blues) this.queryHashes.remove(b);
    }
View Full Code Here
TOP

Related Classes of net.yacy.kelondro.index.HandleSet

  • de.anomic.crawler.Balancer
  • de.anomic.crawler.NoticedURL
  • de.anomic.data.URLAnalysis
  • de.anomic.search.MediaSnippet
  • de.anomic.search.QueryParams
  • de.anomic.search.ResultFetcher
  • de.anomic.search.Segment$ReferenceCleaner
  • de.anomic.search.TextSnippet
  • de.anomic.yacy.dht.Dispatcher
  • de.anomic.yacy.dht.PeerSelection$acceptRemoteIndexSeedEnum
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.