System.out.println("User name: "+resultItems.get("username"));
return RequestMatcher.MatchOther.YES;
}
};
CompositePageProcessor pageProcessor = new CompositePageProcessor(Site.me().setDomain("github.com").setRetryTimes(3));
CompositePipeline pipeline = new CompositePipeline();
pageProcessor.setSubPageProcessors(githubRepoProcessor, githubUserProcessor);
pipeline.setSubPipeline(githubRepoProcessor, githubUserProcessor);
Spider.create(pageProcessor).addUrl("https://github.com/code4craft").thread(5).addPipeline(pipeline).runAsync();
}