Examples of net.yacy.kelondro.index.HandleSet

net.yacy.kelondro.index.HandleSet

        for (final Index i: this.tables.values()) m += i.mem();
        return m;
    }


    public final byte[] smallestKey() {
        final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
        for (final Index oi: this.tables.values()) try {
            keysort.put(oi.smallestKey());
        } catch (final RowSpaceExceededException e) {
            Log.logException(e);
        }
        return keysort.smallestKey();
    }

View Full Code Here

        }
        return keysort.smallestKey();
    }


    public final byte[] largestKey() {
        final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
        for (final Index oi: this.tables.values()) try {
            keysort.put(oi.largestKey());
        } catch (final RowSpaceExceededException e) {
            Log.logException(e);
        }
        return keysort.largestKey();
    }

View Full Code Here

        this.delayed = new TreeMap<Long, byte[]>();
        this.minimumLocalDelta = minimumLocalDelta;
        this.minimumGlobalDelta = minimumGlobalDelta;
        this.myAgentIDs = myAgentIDs;
        this.domStackInitSize = Integer.MAX_VALUE;
        this.ddc = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
        this.double_push_check = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);


        // create a stack for newly entered entries
        if (!(cachePath.exists())) cachePath.mkdir(); // make the path
        this.cacheStacksPath.mkdirs();
        final File f = new File(this.cacheStacksPath, stackname + indexSuffix);

View Full Code Here

        // removes all entries with a specific profile hash.
        // this may last some time
        // returns number of deletions


        // first find a list of url hashes that shall be deleted
        final HandleSet urlHashes = new HandleSet(this.urlFileIndex.row().primaryKeyLength, Base64Order.enhancedCoder, 100);
        final long terminate = (timeout > 0) ? System.currentTimeMillis() + timeout : Long.MAX_VALUE;
        synchronized (this) {
            final Iterator<Row.Entry> i = this.urlFileIndex.rows();
            Row.Entry rowEntry;
            Request crawlEntry;
            while (i.hasNext() && (System.currentTimeMillis() < terminate)) {
                rowEntry = i.next();
                crawlEntry = new Request(rowEntry);
                if (crawlEntry.profileHandle().equals(profileHandle)) {
                    urlHashes.put(crawlEntry.url().hash());
                }
            }
        }


        // then delete all these urls from the queues and the file index

View Full Code Here

            }
        }


        // iterate through the domain stacks
        final Iterator<Map.Entry<String, HandleSet>> q = this.domainStacks.entrySet().iterator();
        HandleSet stack;
        while (q.hasNext()) {
            stack = q.next().getValue();
            for (final byte[] handle: urlHashes) stack.remove(handle);
            if (stack.isEmpty()) q.remove();
        }


        return removedCounter;
    }

View Full Code Here

    }


    private void pushHashToDomainStacks(String host, final byte[] urlhash) throws RowSpaceExceededException {
        // extend domain stack
        if (host == null) host = localhost;
        HandleSet domainList = this.domainStacks.get(host);
        if (domainList == null) {
            // create new list
            domainList = new HandleSet(12, Base64Order.enhancedCoder, 1);
            domainList.put(urlhash);
            this.domainStacks.put(host, domainList);
        } else {
            // extend existent domain list
          domainList.put(urlhash);
        }
    }

View Full Code Here

    }


    private void removeHashFromDomainStacks(String host, final byte[] urlhash) {
        // reduce domain stack
        if (host == null) host = localhost;
        final HandleSet domainList = this.domainStacks.get(host);
        if (domainList == null) {
            this.domainStacks.remove(host);
            return;
        }
        domainList.remove(urlhash);
        if (domainList.isEmpty()) this.domainStacks.remove(host);
    }

View Full Code Here

    private void fillDomainStacks() throws IOException {
      if (!this.domainStacks.isEmpty() && System.currentTimeMillis() - this.lastDomainStackFill < 120000L) return;
      this.domainStacks.clear();
      this.top.clear();
      this.lastDomainStackFill = System.currentTimeMillis();
      final HandleSet handles = this.urlFileIndex.keysFromBuffer(objectIndexBufferSize / 2);
        final CloneableIterator<byte[]> i = handles.keys(true, null);
        byte[] handle;
        String host;
        Request request;
      while (i.hasNext()) {
          handle = i.next();

View Full Code Here

        this.asc = ascending;
        this.zero = null;
    }


    public HandleSet getHandleSet(final int keylength, final int space) {
        return new HandleSet(keylength, this, space);
    }

View Full Code Here

        this.order = order;
        this.remote_peerCount = 0;
        this.remote_resourceSize = 0;
        this.remote_indexCount = 0;
        this.local_indexCount = 0;
        this.urlhashes = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
        this.misses = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
        this.sortout = 0;
        this.flagcount = new int[32];
        for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;}
        this.hostNavigator = new ConcurrentScoreMap<String>();
        this.hostResolver = new ConcurrentHashMap<String, byte[]>();

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of net.yacy.kelondro.index.HandleSet

de.anomic.crawler.Balancer

de.anomic.crawler.NoticedURL

de.anomic.data.URLAnalysis

de.anomic.search.MediaSnippet

de.anomic.search.QueryParams

de.anomic.search.ResultFetcher

de.anomic.search.Segment$ReferenceCleaner

de.anomic.search.TextSnippet

de.anomic.yacy.dht.Dispatcher

de.anomic.yacy.dht.PeerSelection$acceptRemoteIndexSeedEnum

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.