Package net.yacy.kelondro.blob

Examples of net.yacy.kelondro.blob.HeapWriter


        assert this.cache != null;
        if (this.cache == null) return;
        Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's");
        if (heapFile.exists()) FileUtils.deletedelete(heapFile);
        final File tmpFile = new File(heapFile.getParentFile(), heapFile.getName() + ".prt");
        HeapWriter dump;
        try {
            dump = new HeapWriter(tmpFile, heapFile, this.termSize, this.termOrder, writeBuffer);
        } catch (final IOException e1) {
            Log.logException(e1);
            return;
        }
        final long startTime = System.currentTimeMillis();

        // sort the map
        final List<ReferenceContainer<ReferenceType>> cachecopy = sortedClone();

        // write wCache
        long wordcount = 0, urlcount = 0;
        byte[] term = null, lwh;
        for (final ReferenceContainer<ReferenceType> container: cachecopy) {
            // get entries
            lwh = term;
            term = container.getTermHash();

            // check consistency: entries must be ordered
            assert (lwh == null || this.termKeyOrdering().compare(term, lwh) > 0);

            // put entries on heap
            if (container != null && term.length == this.termSize) {
                //System.out.println("Dump: " + wordHash);
                try {
                    dump.add(term, container.exportCollection());
                } catch (final IOException e) {
                    Log.logException(e);
                } catch (final RowSpaceExceededException e) {
                    Log.logException(e);
                }
                if (destructive) container.clear(); // this memory is not needed any more
                urlcount += container.size();
            }
            wordcount++;
        }
        try {
            dump.close(true);
            Log.logInfo("indexContainerRAMHeap", "finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
        } catch (final IOException e) {
            Log.logSevere("indexContainerRAMHeap", "failed rwi heap dump: " + e.getMessage(), e);
        } finally {
            dump = null;
View Full Code Here


        assert this.cache != null;
        if (this.cache == null) return;
        Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's");
        if (heapFile.exists()) FileUtils.deletedelete(heapFile);
        final File tmpFile = new File(heapFile.getParentFile(), heapFile.getName() + ".prt");
        HeapWriter dump;
        try {
            dump = new HeapWriter(tmpFile, heapFile, this.termSize, this.termOrder, writeBuffer);
        } catch (final IOException e1) {
            Log.logException(e1);
            return;
        }
        final long startTime = System.currentTimeMillis();

        // sort the map
        final List<ReferenceContainer<ReferenceType>> cachecopy = sortedClone();

        // write wCache
        long wordcount = 0, urlcount = 0;
        byte[] term = null, lwh;
        assert this.termKeyOrdering() != null;
        for (final ReferenceContainer<ReferenceType> container: cachecopy) {
            // get entries
            lwh = term;
            term = container.getTermHash();
            if (term == null) continue;

            // check consistency: entries must be ordered
            assert (lwh == null || this.termKeyOrdering().compare(term, lwh) > 0);

            // put entries on heap
            if (container != null && term.length == this.termSize) {
                //System.out.println("Dump: " + wordHash);
                try {
                    dump.add(term, container.exportCollection());
                } catch (final IOException e) {
                    Log.logException(e);
                } catch (final RowSpaceExceededException e) {
                    Log.logException(e);
                }
                if (destructive) container.clear(); // this memory is not needed any more
                urlcount += container.size();
            }
            wordcount++;
        }
        try {
            dump.close(true);
            Log.logInfo("indexContainerRAMHeap", "finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
        } catch (final IOException e) {
            Log.logSevere("indexContainerRAMHeap", "failed rwi heap dump: " + e.getMessage(), e);
        } finally {
            dump = null;
View Full Code Here

TOP

Related Classes of net.yacy.kelondro.blob.HeapWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.