Package org.apache.tika.sax

Examples of org.apache.tika.sax.LinkContentHandler

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
            "in the US is <a href=\"http://antoniospizzas.com/\">Antonio's Pizza</a>.</p>" +
            "<p>It is located in Amherst, MA.</p></body></html>";
    //<start id="tika-html"/>
    InputStream input = new ByteArrayInputStream(html.getBytes(Charset.forName("UTF-8")));
    ContentHandler text = new BodyContentHandler();//<co id="html.text.co"/>
    LinkContentHandler links = new LinkContentHandler();//<co id="html.link.co"/>
    ContentHandler handler = new TeeContentHandler(links, text);//<co id="html.merge"/>
    Metadata metadata = new Metadata();//<co id="html.store"/>
    Parser parser = new HtmlParser();//<co id="html.parser"/>
    ParseContext context = new ParseContext();
    parser.parse(input, handler, metadata, context);//<co id="html.parse"/>
    System.out.println("Title: " + metadata.get(Metadata.TITLE));
    System.out.println("Body: " + text.toString());
    System.out.println("Links: " + links.getLinks());
    /*
    <calloutlist>
        <callout arearefs="html.text.co"><para>Construct a ContentHandler that will just extract between the body tags.</para></callout>
        <callout arearefs="html.link.co"><para>Construct ContentHandler that knows about HTML links</para></callout>
        <callout arearefs="html.merge"><para>Wrap up our ContentHandlers into one</para></callout>
View Full Code Here
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
    public void testCustomHtmlSchema() throws Exception {
        // Default schema does not allow tables inside anchors
        String test = "<html><body><a><table><tr><td>text</tr></tr></table></a></body></html>";

        Metadata metadata = new Metadata();
        LinkContentHandler linkContentHandler = new LinkContentHandler();

        new HtmlParser().parse (
                new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
                linkContentHandler, metadata, new ParseContext());

        // Expect no anchor text
        assertEquals("", linkContentHandler.getLinks().get(0).getText());

        // We'll change the schema to allow tables inside anchors!
        Schema schema = new HTMLSchema();
        schema.elementType("a", HTMLSchema.M_ANY, 65535, 0);

        ParseContext parseContext = new ParseContext();
        parseContext.set(Schema.class, schema);
        linkContentHandler = new LinkContentHandler();
        new HtmlParser().parse (
                new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
                linkContentHandler, metadata, parseContext);

        // Expect anchor text
        assertEquals("\ttext\n\n", linkContentHandler.getLinks().get(0).getText());
    }
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
        while (scanner.hasNext())
          sb.append(scanner.nextLine());

        HtmlParser parser = new HtmlParser();
        Metadata met = new Metadata();
        LinkContentHandler handler = new LinkContentHandler();

        parser.parse(new ByteArrayInputStream(sb.toString().getBytes()),
            handler, met);
        List<Link> links = handler.getLinks();
        children = new LinkedList<ProtocolFile>();
        for (Link link : links) {
          String href = link.getUri();
          String linkName = link.getTitle();
          String curPath = this.pwd().getProtocolPath().getPathString();
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
        ByteArrayInputStream bais = new ByteArrayInputStream(content);
        Metadata md = new Metadata();

        String text = null;

        LinkContentHandler linkHandler = new LinkContentHandler();
        ContentHandler textHandler = new BodyContentHandler();
        TeeContentHandler teeHandler = new TeeContentHandler(linkHandler,
                textHandler);
        ParseContext parseContext = new ParseContext();
        // parse
        try {
            tika.getParser().parse(bais, teeHandler, md, parseContext);
            text = textHandler.toString();
        } catch (Exception e) {
            LOG.error("Exception while parsing " + url, e.getMessage());
            eventMeters.scope(
                    "error_content_parsing_" + e.getClass().getSimpleName())
                    .mark();
            collector.fail(tuple);
            eventMeters.scope("tuple_fail").mark();
            return;
        } finally {
            try {
                bais.close();
            } catch (IOException e) {
                LOG.error("Exception while closing stream", e);
            }
        }

        long duration = System.currentTimeMillis() - start;

        LOG.info("Parsed " + url + " in " + duration + " msec");

        // get the outlinks and convert them to strings (for now)
        String fromHost;
        URL url_;
        try {
            url_ = new URL(url);
            fromHost = url_.getHost().toLowerCase();
        } catch (MalformedURLException e1) {
            // we would have known by now as previous
            // components check whether the URL is valid
            LOG.error("MalformedURLException on " + url);
            eventMeters.scope(
                    "error_outlinks_parsing_" + e1.getClass().getSimpleName())
                    .mark();
            collector.fail(tuple);
            eventMeters.scope("tuple_fail").mark();
            return;
        }

        List<Link> links = linkHandler.getLinks();
        Set<String> slinks = new HashSet<String>(links.size());
        for (Link l : links) {
            if (StringUtils.isBlank(l.getUri()))
                continue;
            String urlOL = null;
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    xmlHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
    xmlHandler.setResult(new StreamResult(dataBuffer));
   
    BoilerpipeContentHandler mainContentHandler = new BoilerpipeContentHandler(mainContentBuffer);
    BodyContentHandler bodyHandler = new BodyContentHandler(bodyBuffer);
    LinkContentHandler linkHandler = new LinkContentHandler();
   
    TeeContentHandler parallelHandler = new TeeContentHandler(xmlHandler, mainContentHandler, bodyHandler, linkHandler );

    InputStream instream = entity.obtainContent();
    try {
      parser.parse(instream, parallelHandler, metadata, new ParseContext());
     
      ArrayList<Link> extractedTasks = new ArrayList<Link>();
      int depth = task.getDepth() + 1;
      if (task instanceof LinkTask) {
        for(org.apache.tika.sax.Link tikaLink : linkHandler.getLinks()) {
          try {
            URI uri = new URI(tikaLink.getUri());
            // Test to see if the scheme is empty
            // This would indicate a relative URL, so resolve it against the task URI
            if(uri.getScheme() == null) {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    xmlHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
    xmlHandler.setResult(new StreamResult(dataBuffer));
   
    BoilerpipeContentHandler mainContentHandler = new BoilerpipeContentHandler(mainContentBuffer);
    BodyContentHandler bodyHandler = new BodyContentHandler(bodyBuffer);
    LinkContentHandler linkHandler = new LinkContentHandler();
   
    TeeContentHandler parallelHandler = new TeeContentHandler(xmlHandler, mainContentHandler, bodyHandler, linkHandler );

    InputStream instream = entity.obtainContent();
    try {
      parser.parse(instream, parallelHandler, metadata, new ParseContext());
     
      ArrayList<Link> extractedTasks = new ArrayList<Link>();
      if (task instanceof Link) {
        int depth = task.getDepth() + 1;
        for(org.apache.tika.sax.Link tikaLink : linkHandler.getLinks()) {
          try {
            URI uri = new URI(tikaLink.getUri());
            // Test to see if the scheme is empty
            // This would indicate a relative URL, so resolve it against the task URI
            if(uri.getScheme() == null) {
View Full Code Here
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
    public void testCustomHtmlSchema() throws Exception {
        // Default schema does not allow tables inside anchors
        String test = "<html><body><a><table><tr><td>text</tr></tr></table></a></body></html>";

        Metadata metadata = new Metadata();
        LinkContentHandler linkContentHandler = new LinkContentHandler();

        new HtmlParser().parse (
                new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
                linkContentHandler, metadata, new ParseContext());

        // Expect no anchor text
        assertEquals("", linkContentHandler.getLinks().get(0).getText());

        // We'll change the schema to allow tables inside anchors!
        Schema schema = new HTMLSchema();
        schema.elementType("a", HTMLSchema.M_ANY, 65535, 0);

        ParseContext parseContext = new ParseContext();
        parseContext.set(Schema.class, schema);
        linkContentHandler = new LinkContentHandler();
        new HtmlParser().parse (
                new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
                linkContentHandler, metadata, parseContext);

        // Expect anchor text
        assertEquals("\ttext\n\n", linkContentHandler.getLinks().get(0).getText());
    }
View Full Code Here
TOP

Related Classes of org.apache.tika.sax.LinkContentHandler

  • com.digitalpebble.storm.crawler.bolt.ParserBolt
  • com.tamingtext.tika.TikaTest
  • org.apache.droids.tika.TikaDocumentParser
  • org.apache.droids.tika.TikaHtmlParser
  • org.apache.oodt.cas.pushpull.protocol.http.HttpClient
  • org.apache.tika.parser.html.HtmlParserTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.