Package net.yacy.document.parser.html

Examples of net.yacy.document.parser.html.ScraperInputStream

954
955
956
957
958
959
960
961
962
963
964
965
                            // detect charset of html-files
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
        }
       
        // nothing found: try to find a meta-tag
        if (charset == null) {
            try {
                final ScraperInputStream htmlFilter = new ScraperInputStream(sourceStream,documentCharset,location,null,false);
                sourceStream = htmlFilter;
                charset = htmlFilter.detectCharset();
            } catch (IOException e1) {
                throw new Parser.Failure("Charset error:" + e1.getMessage(), location);
            }
        }
View Full Code Here
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
                            // detect charset of html-files
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
171
172
        }

        // nothing found: try to find a meta-tag
        if (charset == null) {
            try {
                final ScraperInputStream htmlFilter = new ScraperInputStream(sourceStream,documentCharset,location,null,false);
                sourceStream = htmlFilter;
                charset = htmlFilter.detectCharset();
            } catch (final IOException e1) {
                throw new Parser.Failure("Charset error:" + e1.getMessage(), location);
            }
        }
View Full Code Here
TOP

Related Classes of net.yacy.document.parser.html.ScraperInputStream

  • de.anomic.http.server.HTTPDFileHandler
  • net.yacy.document.parser.htmlParser
  • java.io.InputStreamReader
  • java.io.BufferedInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.