Package org.vietspider.html.util

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

80
81
82
83
84
85
86
87
88
89
90
    if(level == depth) return ;
    System.out.println("\nstart download "+ address +" level "+level +" depth "+depth +" ...");
    byte [] bytes = download(parent.toString(), address);
    if(bytes == null || bytes.length < 0) return;
    HTMLParserDetector parser = new HTMLParserDetector();
    String charset = parser.detectCharset(bytes);
    char [] chars = CharsDecoder.decode(charset, bytes, 0, bytes.length);

    List<NodeImpl> tokens = parser.createTokens(chars);

    List<Resource> resources = new ArrayList<Resource>();
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
      message = "Error: Not html data!";
      return ERROR;
    }
    if(charset == null) {
      HTMLParserDetector detector = new HTMLParserDetector();
      charset = detector.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    return post(referer, formName, url, chars, abort);
  }
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
  }
 
  private boolean checkTimeout(byte [] data) throws Exception {
    if(data == null) return true;
    HTMLParserDetector parser = new HTMLParserDetector();
    if(charset == null) charset = parser.detectCharset(data);
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    List<NodeImpl> tokens  = parser.createTokens(chars);
    if(tokens == null) return true;
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
      message = "Not html data!";
      return ERROR;
    }
    if(charset == null) {
      HTMLParserDetector detector = new HTMLParserDetector();
      charset = detector.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    return post(referer, url, chars);
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
    byte [] data = responseReader.readBody(response);
   
    HTMLParserDetector htmlParser2 = new HTMLParserDetector();
    if(data == null) return false;
    if(charset == null) {
      charset = htmlParser2.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    List<NodeImpl> tokens  = htmlParser2.createTokens(chars);
    if(tokens == null) return false;
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    if(level == depth) return ;
    System.out.println("\nstart download "+ address +" level "+level +" depth "+depth +" ...");
    byte [] bytes = download(parent.toString(), address);
    if(bytes == null || bytes.length < 0) return;
    HTMLParserDetector parser = new HTMLParserDetector();
    String charset = parser.detectCharset(bytes);
    char [] chars = CharsDecoder.decode(charset, bytes, 0, bytes.length);

    List<NodeImpl> tokens = parser.createTokens(chars);

    List<Resource> resources = new ArrayList<Resource>();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.