Package net.matuschek.examples

Source Code of net.matuschek.examples.StopDownload$DownloadStopper

package net.matuschek.examples;

/*********************************************
    Copyright (c) 2001 by Daniel Matuschek
*********************************************/

import java.net.URL;

import net.matuschek.spider.WebRobot;
import net.matuschek.spider.WebRobotCallback;

import org.apache.log4j.BasicConfigurator;

/**
* This example program shows how it is possible
* to use the WebRobotCallback to stop after a given number
* of documents has been retrieved.
*
* @author Daniel Matuschek
* @version $Revision: 1.3 $
*/
public class StopDownload {

  class DownloadStopper implements WebRobotCallback {
   
    /** maximal number of documents to retrieve */
    int max = 0;

    /** current number of retrieved documents */
    int count = 0;
   
    /** WebRobot to control */
    WebRobot robot = null;


    public DownloadStopper(int max, WebRobot robot) {
      this.max = max;
      this.robot = robot;
    }
   
    /**
     * Increases the number of retrieved documents and stops
     * the robot, if the number has reached the maximum
     */
    public void webRobotRetrievedDoc(String url, int size) {
      count ++;
      if (count >= max) {
  robot.stopRobot();
      }
    }

    // ignore these methods
    public void webRobotDone() {};
    public void webRobotSleeping(boolean sleeping) {}
    public void webRobotUpdateQueueStatus(int length) {}
  }



  public StopDownload() {
  }



  public void run() throws Exception {
    WebRobot robby = new WebRobot();
    robby.setStartURL(new URL("http://www.matuschek.net"));
    robby.setMaxDepth(1);
    robby.setSleepTime(0);

    // download only the first 5 documents
    DownloadStopper stopit = new DownloadStopper(5,robby);
    robby.setWebRobotCallback(stopit);

    robby.run();
  }


  public static void main(String[] args)
    throws Exception
  {
    BasicConfigurator.configure();
    StopDownload stopper = new StopDownload();
    stopper.run();
  }
}
 
TOP

Related Classes of net.matuschek.examples.StopDownload$DownloadStopper

TOP
Copyright © 2015 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.