Package org.vietspider.html

Examples of org.vietspider.html.HTMLDocument

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    saveData(name);
  }

  public void saveData(String name) throws Exception {
    if(data == null || data.length < 1) return;
    HTMLDocument doc = new HTMLParser2().createDocument(data, "utf-8");
    if(this.addPaths == null) return;
    doc = htmlExtractor.extract(doc, this.addPaths);
    if(removePaths != null && removePaths.length > 0){
      htmlExtractor.remove(doc.getRoot(), removePaths);
    }
    data = doc.getTextValue().getBytes("utf-8");
    if(data.length < 1) return;
    File file = new File(name);
    FileOutputStream output = new FileOutputStream(file);
    output.write(data);
    output.flush();
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
    if(!childrenThread[0].isComplete()) return ;
    if(childrenThread == null) return;
    if(childrenThread[0] == null) return;
    byte [] data = childrenThread[0].getData();
    if(data == null || data.length < 0) return;
    HTMLDocument document =  new HTMLParser2().createDocument(data, "utf-8");
    if(homePath != null) {
      document = htmlExtractor.extract(document, new NodePath[]{homePath});
    }
    linkUtil.createFullNormalLink(document.getRoot(), new URL(url));
    links  = linkUtil.scanSiteLink(document.getRoot());
    idx = 0;
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    URL url = new URL(address);
    String ref = url.getRef();
    if(ref != null && (ref = ref.trim()).isEmpty())  ref = null;
    if(ref != null) address = address.substring(0, address.indexOf('#'));
   
    HTMLDocument document = null;
    if(address.startsWith("file")){
      file = new File(url.toURI());     
      document = detector.loadDocument(file);
    } else {
      byte[] obj = loadContent(refer, address);      
      if( obj == null || obj.length < 1) return null;
      document = detector.createDocument(obj);
      chars = document.getTextValue().toCharArray();
      if(cache) cacheResponse(address, chars);
    }
   
    return document == null || ref == null ?
        document : new HTMLAnchorUtil().searchDocument(document, ref);
View Full Code Here
13
14
15
16
17
18
19
20
21
22
23
24
25
*/
public class JavascriptExtractor {

  public static void main(String[] args) throws Exception {
    URL url = new URL("http://java.sun.com/");
    HTMLDocument document = new HTMLParser2().createDocument(url.openStream(), "utf-8");
   
    NodeIterator iterator =  document.getRoot().iterator();
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(node.isNode(Name.SCRIPT)) {
        if(node.hasChildren() && node.getChildren().size() > 0) {
          System.out.println("===================================================");
View Full Code Here
135
136
137
138
139
140
141
142
143
    builder.append("<h4>Địa chỉ gửi link liên kết đến bài viết này</h4>");
   
    List<String> jsScripts = new ArrayList<String>();
    jsScripts.add("show_postcontent");
   
    HTMLDocument document = new HTMLParser2().createDocument(builder.toString());
    JsHandler.updateDocument(document, jsScripts);
    System.out.println(document.getTextValue());
  }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
  void setAfterLogin(List<NodeImpl> tokens) throws Exception {
    afters = build(tokens);
  }
 
  private String [] build(List<NodeImpl> tokens) throws Exception {
    HTMLDocument doc = new HTMLParser2().createDocument(tokens);
    NodePathParser pathParser = new NodePathParser();
    HTMLExtractor extractor  = new HTMLExtractor();
   
    NodePath nodePath  = pathParser.toPath("BODY");
    HTMLNode body = extractor.lookNode(doc.getRoot(), nodePath);
   
    TextRenderer renderer = new TextRenderer(body, TextRenderer.HANDLER);
    String value = renderer.getTextValue().toString();
    return value.trim().split("\n");
  }
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
  public HTMLDocument extract(HTMLDocument document, NodePath... nodePaths) {
    HTMLNode root = document.getRoot();
//    CharsToken tokens = document.getTokens();
   
    HTMLNode newRoot = HTMLParser2.clone(root);
    HTMLDocument newDocument  = new HTMLDocument();
//    CharsToken newTokens = new CharsToken(newDocument);
//    newTokens.push((NodeImpl)newRoot);

    LookupNode lookupNode = new LookupNode();
   
    for(int i = 0; i < nodePaths.length; i++) {
      List<HTMLNode> htmlNodes = lookupNode.lookupNodes(root, nodePaths[i]);
      if(htmlNodes == null ) continue;
      for(HTMLNode htmlNode : htmlNodes) {
        if(htmlNode == null) continue;
//        lookupNode.extractTokens(tokens, newTokens, htmlNode);
        htmlNode.clone(newRoot);
      }
    }
   
    newDocument.setRoot(newRoot);
    return newDocument;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
          nextNodes.get(i).clone(newRoot);
        } catch (Exception e) {
          continue;
        }
      }
      newDocuments[i] = new HTMLDocument(newRoot);
    }
    return newDocuments;
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
  public HTMLDocument extract(HTMLDocument document, NodePath... nodePaths) {
    HTMLNode root = document.getRoot();
//    CharsToken tokens = document.getTokens();
   
    HTMLNode newRoot = HTMLParser2.clone(root);
    HTMLDocument newDocument  = new HTMLDocument();
//    CharsToken newTokens = new CharsToken(newDocument);
//    newTokens.push((NodeImpl)newRoot);

    for(int i = 0; i < nodePaths.length; i++) {
      List<HTMLNode> htmlNodes = matchNodes(root, nodePaths[i]);
      if(htmlNodes == null ) continue;
      for(HTMLNode htmlNode : htmlNodes) {
        if(htmlNode == null) continue;
//        extractTokens(tokens, newTokens, htmlNode);
        newRoot.addChild(htmlNode);
//        htmlNode.setParent(newRoot);
      }
    }
   
    newDocument.setRoot(newRoot);
    return newDocument;
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
//          newHtmlValues.get(i).setParent(html);
        } catch (Exception e) {
          continue;
        }
      }
      newDocuments[i] = new HTMLDocument(html);
    }
    return newDocuments;
  }
View Full Code Here
TOP

Related Classes of org.vietspider.html.HTMLDocument

  • AttributeExplorer
  • CrawlerService
  • DecodeHTMLRef
  • ExtractForum
  • HASTCIndexEx
  • HTMLNodeExplorer
  • JavascriptExtractor
  • LinkUtilExample
  • NewsComExtractor
  • org.vietspider.browser.LoginUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.