Package org.apache.nutch.storage

Examples of org.apache.nutch.storage.WebPage

102
103
104
105
106
107
108
109
110
111
112
113
114
115
  private List<String> readDb() throws Exception {
    List<URLWebPage> pages = CrawlTestUtil.readContents(webPageStore, null, fields);
    ArrayList<String> read = new ArrayList<String>();
    for (URLWebPage up : pages) {
      WebPage page = up.getDatum();
      String representation = up.getUrl();
      representation += "\tnutch.score=" + (int)page.getScore();
      ByteBuffer bb = page.getFromMetadata(new Utf8("custom.attribute"));
      if (bb != null) {
        representation += "\tcustom.attribute=" + ByteUtils.toString(bb.array());
      }
      read.add(representation);
    }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
   
    private void advance() throws IOException {
      hasNext = res.next();
      if (hasNext && batchId != null) {
        do {
          WebPage page = res.get();
          Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page);
          if (NutchJob.shouldProcess(mark, batchId)) {
            return;
          } else {
            if (LOG.isDebugEnabled()) {
View Full Code Here
30
31
32
33
34
35
36
implements Writable {

  private WebPage webPage;

  public WebPageWritable() {
    this(null, new WebPage());
  }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
          return redirectedPage;
        }
        if (newUrl == null || newUrl.equals(url)) {
          String reprUrl = URLUtil.chooseRepr(url, newUrl,
              refreshTime < FetcherJob.PERM_REFRESH_TIME);
          WebPage newWebPage = new WebPage();
          if (reprUrl == null) {
            LOG.warn("reprUrl==null for " + url);
            return redirectedPage;
          } else {
            page.setReprUrl(new Utf8(reprUrl));
View Full Code Here
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
      if (newUrl == null || newUrl.equals(url)) {
        return;
      }
      reprUrl = URLUtil.chooseRepr(reprUrl, newUrl, temp);
      final String reversedNewUrl = TableUtil.reverseUrl(newUrl);
      WebPage newWebPage = new WebPage();
      if (!reprUrl.equals(url)) {
        newWebPage.setReprUrl(new Utf8(reprUrl));
      }
      newWebPage.putToMetadata(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL);
      context.write(reversedNewUrl, newWebPage);
      if (LOG.isDebugEnabled()) {
        LOG.debug(" - " + redirType + " redirect to " +
            reprUrl + " (fetching later)");
      }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
    try {
      ParseUtil parser = new ParseUtil(NutchConfiguration.create());
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {
        WebPage page = getPage(docs[t]);
        parser.parse(URL.toString(), page);
        ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE));
        String lang = null;
        if (blang != null)
          lang = Bytes.toString(blang.array());
        assertEquals(metalanguages[t], lang);
      }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
      fail(e.toString());
    }
  }

  private WebPage getPage(String text) {
    WebPage page = new WebPage();
    page.setBaseUrl(BASE);
    page.setContent(ByteBuffer.wrap(text.getBytes()));
    page.setContentType(new Utf8("text/html"));
    page
        .putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/html"));
    return page;
  }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
   *             When an error occurs or test case fails.
   */
  private void fetchPage(String page, int expectedCode) throws Exception {
    URL url = new URL("http", "127.0.0.1", port, page);
    Response response = null;
    response = http.getResponse(url, new WebPage(), true);

    int code = response.getCode();
    assertEquals("HTTP Status Code for " + url, expectedCode, code);
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
   *
   */
  public void setContentType(String testTextFile) throws ProtocolNotFound {
    String urlString = "file:" + sampleDir + fileSeparator + testTextFile;
    assertNotNull(urlString);
    WebPage datum = new WebPage();
    Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString);
    ProtocolOutput output = protocol.getProtocolOutput(urlString,datum);
    assertNotNull(output);

    assertEquals("Status code: [" + output.getStatus().getCode()
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
     MoreIndexingFilter filter = new MoreIndexingFilter();
     filter.setConf(conf);
     assertNotNull(filter);
     NutchDocument doc = new NutchDocument();
     try{
       filter.filter(doc, "http://nutch.apache.org/index.html", new WebPage());
     }
     catch(Exception e){
       e.printStackTrace();
       fail(e.getMessage());
     }
View Full Code Here
TOP

Related Classes of org.apache.nutch.storage.WebPage

  • org.apache.gora.persistency.impl.StateManagerImpl
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
  • org.apache.nutch.api.DbReader$DbIterator
  • org.apache.nutch.crawl.DbUpdateReducer
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.TestGenerator
  • org.apache.nutch.crawl.TestInjector
  • org.apache.nutch.crawl.TestURLPartitioner
  • org.apache.nutch.crawl.WebTableReader
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.