page.setUrl(url);
page.setFetchTime(crawldatum.getFetchTime());
if (response == null) {
LogUtils.getLogger().info("failed " + url);
HandlerUtils.sendMessage(handler, new Message(Fetcher.FETCH_FAILED, page), true);
continue;
}
page.setResponse(response);
LogUtils.getLogger().info("fetch " + url);
String contentType = response.getContentType();
if (parsing) {
try {
Parser parser = parserFactory.createParser(url, contentType);
if (parser != null) {
ParseResult parseresult = parser.getParse(page);
page.setParseResult(parseresult);
}
} catch (Exception ex) {
LogUtils.getLogger().info("Exception", ex);
}
}
if (needUpdateDb) {
try {
dbUpdater.getSegmentWriter().wrtieFetch(crawldatum);
if (isContentStored) {
Content content = new Content();
content.setUrl(url);
if (response.getContent() != null) {
content.setContent(response.getContent());
} else {
content.setContent(new byte[0]);
}
content.setContentType(contentType);
dbUpdater.getSegmentWriter().wrtieContent(content);
}
if (parsing && page.getParseResult() != null) {
dbUpdater.getSegmentWriter().wrtieParse(page.getParseResult());
}
} catch (Exception ex) {
LogUtils.getLogger().info("Exception", ex);
}
}
HandlerUtils.sendMessage(handler, new Message(Fetcher.FETCH_SUCCESS, page), true);
} catch (Exception ex) {
LogUtils.getLogger().info("Exception", ex);
}
}