Package org.apache.nutch.protocol

Examples of org.apache.nutch.protocol.Protocol

67
68
69
70
71
72
73
74
75
76
  protected void tearDown() {}

  public String getTextContent(String fileName) throws ProtocolException, ParseException {
    String urlString = "file:" + sampleDir + fileSeparator + fileName;
    Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString);
    Content content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
    Parse parse = new ParseUtil(conf).parseByExtensionId("parse-tika", content).get(content.getUrl());
    return parse.getText();
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  }

  public void testIt() throws ProtocolException, ParseException {

    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    urlString = "file:" + sampleDir + fileSeparator + rtfFile;
    protocol = new ProtocolFactory(conf).getProtocol(urlString);
    content = protocol.getProtocolOutput(new Text(urlString),
        new CrawlDatum()).getContent();
    parse = new ParseUtil(conf).parseByExtensionId("parse-tika", content)
        .get(content.getUrl());
    String text = parse.getText();
    assertEquals("The quick brown fox jumps over the lazy dog", text.trim());
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  protected void tearDown() {}

  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      Configuration conf = NutchConfiguration.create();
      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-tika", content).get(content.getUrl());

      int index = parse.getText().indexOf(expectedText);
      assertTrue(index > 0);
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
   * file</li>
   * </ul>
   */
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
          new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-tika",
          content).get(content.getUrl());

      // check that there are 2 outlinks:
View Full Code Here
334
335
336
337
338
339
340
341
342
343
344
345
346
347
  /**
   * @param url
   * @return contents of url
   */
  private static String getUrlContent(String url, Configuration conf) {
    Protocol protocol;
    try {
      protocol = new ProtocolFactory(conf).getProtocol(url);
      Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();
      Parse parse = new ParseUtil(conf).parse(content).get(content.getUrl());
      System.out.println("text:" + parse.getText());
      return parse.getText();

    } catch (ProtocolNotFound e) {
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
    if (LOG.isInfoEnabled()) { LOG.info("fetching: "+url); }

    Configuration conf = NutchConfiguration.create();
    ProtocolFactory factory = new ProtocolFactory(conf);
    Protocol protocol = factory.getProtocol(url);
    Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();

    if (force) {
      content.setContentType(contentType);
    } else {
      contentType = content.getContentType();
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
   * @throws ParseException
   *           If the {@link Parser}Layer cannot be loaded.
   */
  public void testParseFetchChannel() throws ProtocolNotFound, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    ParseResult parseResult;

    Configuration conf = NutchConfiguration.create();
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
      urlString = urlString.replace('\\', '/');

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
          new CrawlDatum()).getContent();

      parseResult = new ParseUtil(conf).parseByExtensionId("feed", content);

      assertEquals(3, parseResult.size());
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
   * file</li>
   * </ul>
   */
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
          new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-tika",
          content).get(content.getUrl());

      // check that there are 2 outlinks:
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    }
       
    IndexingFilters indexers = new IndexingFilters(conf);
   
    ProtocolFactory factory = new ProtocolFactory(conf);
    Protocol protocol = factory.getProtocol(url);
    CrawlDatum datum = new CrawlDatum();
   
    Content content = protocol.getProtocolOutput(new Text(url), datum)
        .getContent();
   
    if (content == null) {
      System.out.println("No content for " + url);
      return 0;
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
    if (LOG.isInfoEnabled()) {
      LOG.info("fetching: " + url);
    }

    ProtocolFactory factory = new ProtocolFactory(conf);
    Protocol protocol = factory.getProtocol(url);
    Content content = protocol.getProtocolOutput(new Text(url),
        new CrawlDatum()).getContent();

    if (content == null) {
      System.err.println("Can't fetch URL successfully");
      return (-1);
View Full Code Here
TOP

Related Classes of org.apache.nutch.protocol.Protocol

  • org.apache.nutch.analysis.lang.LanguageIdentifier
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
  • org.apache.nutch.indexer.IndexingFiltersChecker
  • org.apache.nutch.parse.ext.TestExtParser
  • org.apache.nutch.parse.feed.TestFeedParser
  • org.apache.nutch.parse.html.TestMetatagParser
  • org.apache.nutch.parse.metatags.TestMetatagParser
  • org.apache.nutch.parse.mp3.TestMP3Parser
  • org.apache.nutch.parse.msexcel.TestMSExcelParser
  • org.apache.nutch.parse.msword.TestMSWordParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.