Package org.apache.droids.robot.crawler

Examples of org.apache.droids.robot.crawler.CrawlingDroid

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    Queue<Link> queue = new LinkedList<Link>();
   
    CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
//    MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
   
    CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue, taskMaster );
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURL );
    helloCrawler.setInitialLocations(initialLocations);
   
    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();
   
    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    Queue<Link> queue = new LinkedList<Link>();
   
    CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
//    MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
   
    CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue, taskMaster );
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURL );
    helloCrawler.setInitialLocations(initialLocations);
   
    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();
   
    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
  }
View Full Code Here
TOP

Related Classes of org.apache.droids.robot.crawler.CrawlingDroid

  • org.apache.droids.DroidsFactory
  • org.apache.droids.examples.SimpleRuntime
  • org.apache.droids.LinkTask
  • java.net.URI
  • org.apache.droids.exception.InvalidTaskException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.