Examples of org.apache.nutch.util.NodeWalker

org.apache.nutch.util.NodeWalker

A utility class that allows the walking of any DOM tree using a stack instead of recursion. As the node tree is walked the next node is popped off of the stack and all of its children are automatically added to the stack to be called in tree order.

Currently this class is not thread safe. It is assumed that only one thread will be accessing the NodeWalker at any given time.

  // anchors
  private boolean getTextHelper(StringBuffer sb, Node node, 
                                             boolean abortOnNestedAnchors,
                                             int anchorDepth) {
    boolean abort = false;
    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
    
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      if ("script".equalsIgnoreCase(nodeName)) {
        walker.skipChildren();
      }
      if ("style".equalsIgnoreCase(nodeName)) {
        walker.skipChildren();
      }
      if (abortOnNestedAnchors && "a".equalsIgnoreCase(nodeName)) {
        anchorDepth++;
        if (anchorDepth > 1) {
          abort = true;
          break;
        }        
      }
      if (nodeType == Node.COMMENT_NODE) {
        walker.skipChildren();
      }
      if (nodeType == Node.TEXT_NODE) {
        // cleanup and trim the value
        String text = currentNode.getNodeValue();
        text = text.replaceAll("\\s+", " ");

View Full Code Here

   *
   * @return true if a title node was found, false otherwise
   */
  public boolean getTitle(StringBuffer sb, Node node) {
    
    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
  
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      if ("body".equalsIgnoreCase(nodeName)) { // stop after HEAD
        return false;

View Full Code Here

  }


  /** If Node contains a BASE tag then it's HREF is returned. */
  public URL getBase(Node node) {


    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
  
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      // is this node a BASE tag?
      if (nodeType == Node.ELEMENT_NODE) {

View Full Code Here

   * nekohtml).
   */
  public void getOutlinks(URL base, ArrayList outlinks, 
                                       Node node) {
    
    NodeWalker walker = new NodeWalker(node);
    while (walker.hasNext()) {
      
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();      
      NodeList children = currentNode.getChildNodes();
      int childLen = (children != null) ? children.getLength() : 0;

View Full Code Here

  // anchors
  private boolean getTextHelper(StringBuffer sb, Node node, 
                                             boolean abortOnNestedAnchors,
                                             int anchorDepth) {
    boolean abort = false;
    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
    
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      if ("script".equalsIgnoreCase(nodeName)) {
        walker.skipChildren();
      }
      if ("style".equalsIgnoreCase(nodeName)) {
        walker.skipChildren();
      }
      if (abortOnNestedAnchors && "a".equalsIgnoreCase(nodeName)) {
        anchorDepth++;
        if (anchorDepth > 1) {
          abort = true;
          break;
        }        
      }
      if (nodeType == Node.COMMENT_NODE) {
        walker.skipChildren();
      }
      if (nodeType == Node.TEXT_NODE) {
        // cleanup and trim the value
        String text = currentNode.getNodeValue();
        text = text.replaceAll("\\s+", " ");

View Full Code Here

   *
   * @return true if a title node was found, false otherwise
   */
  public boolean getTitle(StringBuffer sb, Node node) {
    
    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
  
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      if ("body".equalsIgnoreCase(nodeName)) { // stop after HEAD
        return false;

View Full Code Here

  }


  /** If Node contains a BASE tag then it's HREF is returned. */
  URL getBase(Node node) {


    NodeWalker walker = new NodeWalker(node);
    
    while (walker.hasNext()) {
  
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();
      
      // is this node a BASE tag?
      if (nodeType == Node.ELEMENT_NODE) {

View Full Code Here

   * nekohtml).
   */
  public void getOutlinks(URL base, ArrayList<Outlink> outlinks, 
                                       Node node) {
    
    NodeWalker walker = new NodeWalker(node);
    while (walker.hasNext()) {
      
      Node currentNode = walker.nextNode();
      String nodeName = currentNode.getNodeName();
      short nodeType = currentNode.getNodeType();      
      NodeList children = currentNode.getChildNodes();
      int childLen = (children != null) ? children.getLength() : 0;

View Full Code Here

  /**
   * Finds the specified element and returns its value
   */
  protected List<String> getElement(DocumentFragment doc, String element) {
    List<String> headings = new ArrayList<String>();
    NodeWalker walker = new NodeWalker(doc);


    while (walker.hasNext()) {
      Node currentNode = walker.nextNode();


      if (currentNode.getNodeType() == Node.ELEMENT_NODE) {
        if (element.equalsIgnoreCase(currentNode.getNodeName())) {
          headings.add(getNodeValue(currentNode));

View Full Code Here

      return language;
    }
    
    void parse(Node node) {


      NodeWalker walker = new NodeWalker(node);
      while (walker.hasNext()) {
        
        Node currentNode = walker.nextNode();
        String nodeName = currentNode.getNodeName();
        short nodeType = currentNode.getNodeType();
        
        String lang = null;

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.nutch.util.NodeWalker

org.apache.nutch.analysis.lang.HTMLLanguageParser$LanguageParser

org.apache.nutch.parse.headings.HeadingsParseFilter

org.apache.nutch.parse.html.DOMContentUtils

org.apache.nutch.parse.tika.DOMContentUtils

org.w3c.dom.Node

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.