Package net.vidageek.crawler

Examples of net.vidageek.crawler.Page


    public void run() {
        try {

            log.info("crawling url: " + urlToCrawl.link());

            Page page = downloader.get(urlToCrawl.link());
            if (page.getStatusCode() != Status.OK) {
                visitor.onError(urlToCrawl, page.getStatusCode());
            } else {
                visitor.visit(page);
            }

            for (String l : page.getLinks()) {
                String link = normalizer.normalize(l);
                final Url url = new Url(link, urlToCrawl.depth() + 1);
                if (visitor.followUrl(url)) {
                    executor.execute(new PageCrawlerExecutor(url, executor, counter, downloader, normalizer, visitor));
                }
View Full Code Here

TOP

Related Classes of net.vidageek.crawler.Page

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.