Package net.nutch.parse

Examples of net.nutch.parse.Parser

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  public void testId3v2() throws ProtocolException, ParseException {

    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    urlString = "file:" + sampleDir + fileSeparator + id3v2;
    protocol = ProtocolFactory.getProtocol(urlString);
    content = protocol.getContent(urlString);

    parser = ParserFactory.getParser(content.getContentType(), urlString);
    parse = parser.getParse(content);
    Properties metadata = parse.getData().getMetadata();
    assertEquals("postgresql comment id3v2", metadata.getProperty("COMM-Text"));
    assertEquals("postgresql composer id3v2", metadata.getProperty("TCOM-Text"));
    assertEquals("02", metadata.getProperty("TRCK-Text"));
    assertEquals("http://localhost/", metadata.getProperty("WCOP-URL Link"));
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  public void testId3v1() throws ProtocolException, ParseException {

    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    urlString = "file:" + sampleDir + fileSeparator + id3v1;
    protocol = ProtocolFactory.getProtocol(urlString);
    content = protocol.getContent(urlString);
    parser = ParserFactory.getParser(content.getContentType(), urlString);
    parse = parser.getParse(content);

    Properties metadata = parse.getData().getMetadata();
    assertEquals("postgresql comment id3v1", metadata.getProperty("COMM-Text"));
    assertEquals("postgresql artist id3v1", metadata.getProperty("TPE1-Text"));
    assertEquals("(28)", metadata.getProperty("TCON-Text"));
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  public void testNone() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    urlString = "file:" + sampleDir + fileSeparator + none;
    protocol = ProtocolFactory.getProtocol(urlString);
    content = protocol.getContent(urlString);
    parser = ParserFactory.getParser(content.getContentType(), urlString);
    try {
      parse = parser.getParse(content);
      Properties metadata = parse.getData().getMetadata();
    } catch (ParseException e) {
      return;
    }
    fail("Expected ParseException");
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
237
    Protocol protocol;
    try {
      protocol = ProtocolFactory.getProtocol(url);
      Content content = protocol.getContent(url);
      String contentType = content.getContentType();
      Parser parser = ParserFactory.getParser(contentType, url);
      Parse parse = parser.getParse(content);
      System.out.println("text:" + parse.getText());
      return parse.getText();

    } catch (ProtocolNotFound e) {
      e.printStackTrace();
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    for (int i=0; i<sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = ProtocolFactory.getProtocol(urlString);
      content = protocol.getContent(urlString);

      parser = ParserFactory.getParser(content.getContentType(), urlString);
      parse = parser.getParse(content);

      assertTrue(parse.getText().startsWith(expectedText));
    }
  }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
  public void testIt() throws ProtocolException, ParseException {

    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    urlString = "file:" + sampleDir + fileSeparator + rtfFile;
    protocol = ProtocolFactory.getProtocol(urlString);
    content = protocol.getContent(urlString);

    parser = ParserFactory.getParser(content.getContentType(), urlString);
    parse = parser.getParse(content);
    String text = parse.getText();
    assertEquals("The quick brown fox jumps over the lazy dog", text.trim());

    String title = parse.getData().getTitle();
    Properties meta = parse.getData().getMetadata();
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {

        Content content = getContent(docs[t]);
        Parser parser = ParserFactory.getParser("text/html", URL);
        Parse parse = parser.getParse(content);

        assertEquals(metalanguages[t], (String) parse.getData().get(
            HTMLLanguageParser.META_LANG_NAME));

      }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parser parser;
    Parse parse;

    for (int i=0; i<sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = ProtocolFactory.getProtocol(urlString);
      content = protocol.getContent(urlString);

      parser = ParserFactory.getParser(content.getContentType(), urlString);
      parse = parser.getParse(content);

      int index = parse.getText().indexOf(expectedText);
      assertTrue(index > 0);
    }
  }
View Full Code Here
TOP

Related Classes of net.nutch.parse.Parser

  • net.nutch.analysis.lang.LanguageIdentifier
  • net.nutch.analysis.lang.TestHTMLLanguageParser
  • net.nutch.parse.mp3.TestMP3Parser
  • net.nutch.parse.msword.TestMSWordParser
  • net.nutch.parse.pdf.TestPdfParser
  • net.nutch.parse.rtf.TestRTFParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.