}
// test against robots.txt if available
if (cs.isValidRobots()) {
String ua = metadata.getUserAgent();
RobotsPolicy robots = metadata.getRobotsPolicy();
if(!robots.allows(ua, curi, cs.getRobotstxt())) {
if(getCalculateRobotsOnly()) {
// annotate URI as excluded, but continue to process normally
curi.getAnnotations().add("robotExcluded");
return false;
}