// Look for the 'robots' meta-tag
if("robots".equalsIgnoreCase(name) && content != null ) {
curi.getData().put(A_META_ROBOTS, content);
RobotsPolicy policy = metadata.getRobotsPolicy();
String contentLower = content.toLowerCase();
if (policy.obeyMetaRobotsNofollow()
&& (contentLower.indexOf("nofollow") >= 0
|| contentLower.indexOf("none") >= 0)) {
// if 'nofollow' or 'none' is specified and the
// honoring policy is not IGNORE or CUSTOM, end html extraction
logger.fine("HTML extraction skipped due to robots meta-tag for: "