new AndFilter (
new NodeClassFilter (MetaTag.class),
new HasAttributeFilter ("name", "robots")), true);
if (0 != robots.size ())
{
robot = (MetaTag)robots.elementAt (0);
content = robot.getAttribute ("content").toLowerCase ();
if ((-1 != content.indexOf ("none")) || (-1 != content.indexOf ("nofollow")))
// reset mPages
for (int i = bookmark; i < mPages.size (); i++)
mPages.remove (i);