logger.debug("Selecting pages for the fetchlist");

        int pagesSeen = 0;

        int pagesAdded = 0;

        while (pagesAdded < fetchlistSize && pages.hasNext()) {

            pagesSeen++;

            // if it is a valid web page (not an image, etc), it is considered, otherwise it is discarded

            } else {

                // if the page should be fetched, it goes to the fetchlist, otherwise it is passed

                boolean added = false;

                if (shouldFetch(page)) {

                    try {

                        toFetch.addPage(page);

                        pagesAdded++;

                        added = true;

                    } catch (IOException e) {

                        logger.error("Trying to add a page to the IPageStore " + toFetch + ": " + e, e);

                    }

                }

                if (!added && null != toPass) {

                    try {

                        toPass.addPage(page);

                    } catch (IOException e) {

                        logger.error("Trying to add a page to the IPageStore " + toPass + ": " + e, e);

                    }

                }

            System.err.println ("\nMissing arguments.\n\nUsage:\n  "+mapperClass.getName()+" <nodeCount> <url>\n");

            System.exit(-1);

        }

        int nodes = Integer.parseInt(args[0]);

        APageMapper mapper = (APageMapper)mapperClass.getConstructor(new Class<?>[]{Config.class, Integer.TYPE}).newInstance(new Object[]{null,nodes});

        System.out.println("  "+args[1]+" -> "+mapper.mapPage(page));

    }

        private void advance () {

            doc = null;

            while (null == doc && pages.hasNext()) {

                if (cache.hasItem(url)) {

                    byte[] content = cache.getItem(url).getContent();

                    Map<String,String> header = new HashMap<String,String>(); // this info is lost, it should be stored in the cache along with the page contents.

                    boolean success = true;

                    boolean recoverable = true;

    @SuppressWarnings("unchecked")

    @Override

    protected void internalProcess(FetchDocument doc) {

        if (null == page) {

            logger.warn("Page is null. Ignoring this document.");

            return;

        }

        if (logger.isDebugEnabled()) {

            logger.debug("Doc has tags: "+doc.getTags().toString());

        }

        if (doc.hasTag(EMIT_DOC_TAG)) {

            addToIndex(doc);

        } else {

                deleteFromIndex(page);

            }

        }

    }

        // build xml doc

        org.dom4j.Document dom = DocumentHelper.createDocument();

        Element root = dom.addElement("documentAdd");

        String text = doc.getText();

        String host = getHost(url);

        String title = doc.getTitle(titleLengthLimit);

        String tokenizedHost = tokenizeHost(host);

        String anchorText = getAnchorText(page);

        float categoryBoost = calculateCategoryBoost(attributes);

        float pagerankBoost = calculatePagerankBoost(page);

        float spamrankBoost = calculateSpamrankBoost(page);

        float logBoost = calculateLogBoost(page);

        float freshnessBoost = calculateFreshnessBoost(page);

        // add overall score

        float f1 = factor("category",categoryBoost,categoryBoostDamp);

        float f2 = factor("pagerank",pagerankBoost,pagerankBoostDamp);

        float f3 = factor("spamrank",spamrankBoost,spamrankBoostDamp);

        float f4 = factor("log",logBoost,logBoostDamp);

        float f5 = factor("freshness",freshnessBoost,freshnessBoostDamp);

        float f6 = ((Double)attributes.get("boost")).floatValue(); // as calculated by the boost module, or 1.0 if no boost module is defined.

        float boost = f1 * f2 * f3 * f4 * f5 * f6;

        // System.out.println("BOOST url=["+url+"]  category="+f1+" ("+categoryBoost+":"+categoryBoostDamp+")  pagerank="+f2+" ("+pagerankBoost+":"+pagerankBoostDamp+")  log="+f3+" ("+logBoost+":"+logBoostDamp+")  freshness="+f4+" ("+freshnessBoost+":"+freshnessBoostDamp+") moduleBoost="+f5+"  Boost="+boost);

        if (boost < 1e-6f) {

            logger.warn("Boost too low! ("+boost+")  category="+f1+" ("+categoryBoost+":"+categoryBoostDamp+")  pagerank="+f2+" ("+pagerankBoost+":"+pagerankBoostDamp+")  spamrank="+f3+" ("+spamrankBoost+":"+spamrankBoostDamp+")  log="+f4+" ("+logBoost+":"+logBoostDamp+")  freshness="+f5+" ("+freshnessBoost+":"+freshnessBoostDamp+") moduleBoost="+f6);

            boost = 1e-6f;

        }

       

        if (null == title || "".equals(title)) {

            title = "Untitled";

        }

        root.addElement("boost").addText(String.valueOf(boost));

        root.addElement("documentId").addText(getDocumentId(page));

        Map<String,Double> boostMap = (Map<String,Double>)attributes.get("field_boost");

        // add the search fields

        addField(root, "url", url, true, true, true, boostMap);

        addField(root, "site", host, true, true, false, boostMap);

        addField(root, "tokenizedHost", tokenizedHost, false, true, true, boostMap);

        addField(root, "title", title, true, true, true, boostMap);

        addField(root, "text", text, true, true, true, boostMap);

        addField(root, "anchor", anchorText, false, true, true, boostMap);

        addField(root, "crawl", crawlName, false, true, true, boostMap);

        if (sendContent) {

            addBody(root,doc,content);

        }

        // for debugging only

        //addField(root, "boostinfo", "category="+f1+" ("+categoryBoost+":"+categoryBoostDamp+")  pagerank="+f2+" ("+pagerankBoost+":"+pagerankBoostDamp+")  log="+f3+" ("+logBoost+":"+logBoostDamp+")  freshness="+f4+" ("+freshnessBoost+":"+freshnessBoostDamp+") moduleBoost="+f5+"  Boost="+boost, true, false, false, null);

        addAdditionalFields(root, page, boostMap);

        // Adding metainfo from attributes

        Set<Entry<String,Object>> attributeSet = indexableAttributes.entrySet();

        for (Entry<String,Object> attribute : attributeSet) {

            addField(root, attribute.getKey(), attribute.getValue() == null ? "" : attribute.getValue().toString(), true, true, true, boostMap);

        }

        StringBuffer assignedCategories = new StringBuffer();

        if (null != categories) {

            // iterate through the classes the page belongs to add each category and its score

            for (Iterator<String> iter = categories.iterator(); iter.hasNext();) {

                assignedCategories.append(iter.next());

                assignedCategories.append(" ");

                // repeat the field times proportional to the score (this is a way to boost the document by category);

                //for (int rep = 0; rep < score*10; rep++) {

                //    addField(root, "categoryBoost", categ, false, true, false);

                //}

            }

            addField(root, "categories", assignedCategories.toString().trim(), true, true, true, boostMap);

        }

        if (logger.isDebugEnabled()) {

            logger.debug("Indexing dom: " + DomUtil.domToString(dom));

        }

        // Send the document to the indexer. If the queue is full, wait and retry.

        try {

          int i = pageMapper.mapPage(page);

            while (indexers[i].index(dom) == IndexerReturnCode.RETRY_QUEUE_FULL) {

                try {

                    Thread.sleep(indexerBusyRetryTime*1000);

                } catch (InterruptedException e) {

                    logger.debug("Sleep interrupted: " + e, e);

                }

            }

        } catch (Exception e) {

            logger.error(e,e);

        }

    }

                logger.error(e,e);

            }

        } else if ("delete".equals(command.toString())) {

            FetchDocument doc = ((CommandWithDoc)command).getDoc();

            deleteFromIndex(page);

        } else if ("startCycle".equals(command.toString())) {

            PageDB pagedb = ((CommandWithPageDB)command).getPageDB();

            scoreThreshold = new float[11];

            for (int i = 0; i < 11; i++) {

    }

   

    @Override

    protected void internalProcess (FetchDocument doc) {   

        if (null == page) {

            logger.warn("Fetchdata does not have a page");

            return;

        }

        String text = getDocumentText(doc);

        config.set("keep.original.url.on.redirect", "true");

        TestUtils.writeFile(tmpDir+"/web/test.html", "<html><head><title>title</title></head><body>"+TestUtils.randomText(25,25)+"</body></html>");

        in = PageTest.randomPage();

        in.setUrl(url);

        WebServer server = null;

        Crawler crawler = null;

        try {

            server = new WebServer(8085);

            server.addResourceHandler("/", tmpDir+"/web");

            server.start();

            PageDB db = new PageDB(tmpDir+"/testdb");

            db.open(PageDB.WRITE);

            db.addPage(in);

            db.close();

            crawler = new Crawler();

            int tries = 0;

            int maxTries = 10;

            do {

                tries++;

                crawler.crawl(1);

                db.open(PageDB.READ);

                Iterator<Page> pages = db.iterator();

                assertTrue("The crawler lost or discarded the test page", pages.hasNext());

                out = pages.next();

                assertFalse("The crawler has more than the test page", pages.hasNext());

                db.close();

        } finally {

            if (null != crawler) {

                crawler.cleanup();

            }

            server.requestStop();

            while (! server.isStopped()) {

                Execute.sleep(20);

            }

        }

        String[] anchorsIn = in.getAnchors();

        Arrays.sort(anchorsIn);

        Arrays.sort(anchorsOut);

        assertTrue("Test page anchors changed (in=["+Arrays.toString(anchorsIn)+"] out=["+Arrays.toString(anchorsOut)+"])", Arrays.equals(anchorsIn, anchorsOut));

    }

       

        WebServer server = null;

        Crawler crawler = null;

        in = PageTest.randomPage();

       

        try {

            server = new WebServer(8087);

            server.addResourceHandler("/", tmpDir+"/web");

            server.start();

    }

    @TestInfo(testType = TestInfo.TestType.UNIT)

    public void testHasValue() throws Exception {

        assertTrue(cbv.hasValue(doc));

    }