Package org.apache.nutch.storage

Examples of org.apache.nutch.storage.WebPage

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
      byte[] bytes = new byte[(int) file.length()];
      DataInputStream in = new DataInputStream(new FileInputStream(file));
      in.readFully(bytes);
      in.close();

      WebPage page = new WebPage();
      page.setBaseUrl(new Utf8(urlString));
      page.setContent(ByteBuffer.wrap(bytes));
      String mtype = mimeutil.getMimeType(file);
      page.setContentType(new Utf8(mtype));

      parse = new ParseUtil(conf).parse(urlString, page);

      int index = parse.getText().indexOf(expectedText);
      assertTrue(index > 0);
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
      byte[] bytes = new byte[(int) file.length()];
      DataInputStream in = new DataInputStream(new FileInputStream(file));
      in.readFully(bytes);
      in.close();

      WebPage page = new WebPage();
      page.setBaseUrl(new Utf8(urlString));
      page.setContent(ByteBuffer.wrap(bytes));
      String mtype = mimeutil.getMimeType(file);
      page.setContentType(new Utf8(mtype));

      parse = new ParseUtil(conf).parse(urlString, page);

      // check that there are 2 outlinks:
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
    DataInputStream in = new DataInputStream(new FileInputStream(file));
    in.readFully(bytes);
    Configuration conf = NutchConfiguration.create();
    // TikaParser parser = new TikaParser();
    // parser.setConf(conf);
    WebPage page = new WebPage();
    page.setBaseUrl(new Utf8(url));
    page.setContent(ByteBuffer.wrap(bytes));
    MimeUtil mimeutil = new MimeUtil(conf);
    String mtype = mimeutil.getMimeType(file);
    page.setContentType(new Utf8(mtype));
    // Parse parse = parser.getParse(url, page);

    Parse parse = new ParseUtil(conf).parse(url, page);

    System.out.println("content type: " + mtype);
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
      ftp.setTimeout(timeout);

    if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength
      ftp.setMaxContentLength(maxContentLength);

    Content content = ftp.getProtocolOutput(urlString, new WebPage())
        .getContent();

    System.err.println("Content-Type: " + content.getContentType());
    System.err.println("Content-Length: "
        + content.getMetadata().get(Response.CONTENT_LENGTH));
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    }
    in.close();
    byte[] bytes = out.toByteArray();
    Configuration conf = NutchConfiguration.create();

    WebPage page = new WebPage();
    page.setBaseUrl(new Utf8(url));
    page.setContent(ByteBuffer.wrap(bytes));
    MimeUtil mimeutil = new MimeUtil(conf);
    String mtype = mimeutil.getMimeType(file);
    page.setContentType(new Utf8(mtype));

    new ParseUtil(conf).parse(url, page);

    ByteBuffer bb = page.getFromMetadata(new Utf8("License-Url"));
    assertEquals(license, new String(bb.array()));
    bb = page.getFromMetadata(new Utf8("License-Location"));
    assertEquals(location, new String(bb.array()));
    bb = page.getFromMetadata(new Utf8("Work-Type"));
    if (bb == null)
      assertEquals(type, null);
    else
      assertEquals(type, new String(bb.array()));
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
      if (LOG.isTraceEnabled())
        LOG.trace("cache miss " + url);

      try {
        String robotsUrl = new URL(url, "/robots.txt").toString();       
        ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, new WebPage());
        int statusCode = output.getStatus().getCode();

        if (statusCode == ProtocolStatusCodes.SUCCESS) {
          robotRules =  parseRules(url.toString(), output.getContent().getContent(),
                                  CONTENT_TYPE, agentNames);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  byte[] bytes = new byte[(int) file.length()];
  DataInputStream in = new DataInputStream(new FileInputStream(file));
  in.readFully(bytes);
  in.close();
  Parse parse;
  WebPage page = new WebPage();
  page.setBaseUrl(new Utf8("file:"+urlString));
  page.setContent(ByteBuffer.wrap(bytes));
  // set the content type?
  MimeUtil mimeutil = new MimeUtil(conf);
  String mtype = mimeutil.getMimeType(file);
  page.setContentType(new Utf8(mtype));
   
  parse = new ParseUtil(conf).parse("file:"+urlString, page);
  return parse.getText();
    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
      byte[] bytes = new byte[(int) file.length()];
      DataInputStream in = new DataInputStream(new FileInputStream(file));
      in.readFully(bytes);
      in.close();

      WebPage page = new WebPage();
      page.setBaseUrl(new Utf8(urlString));
      page.setContent(ByteBuffer.wrap(bytes));
      String mtype = mimeutil.getMimeType(file);
      page.setContentType(new Utf8(mtype));

      parse = new ParseUtil(conf).parse(urlString, page);

      String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    byte[] bytes = new byte[(int) file.length()];
    DataInputStream dip = new DataInputStream(new FileInputStream(file));
    dip.readFully(bytes);
    dip.close();
   
    WebPage page = new WebPage();
    page.setBaseUrl(new Utf8(urlString));
    page.setContent(ByteBuffer.wrap(bytes));
    MimeUtil mutil = new MimeUtil(conf);
    String mime = mutil.getMimeType(file);
    page.setContentType(new Utf8(mime));
 
    parse = new ParseUtil(conf).parse(urlString, page);
    return parse.getOutlinks();
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
    try {
      ParseUtil parser = new ParseUtil(NutchConfiguration.create());
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {
        WebPage page = getPage(docs[t]);
        parser.parse(URL.toString(), page);
        ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE));
        String lang = null;
        if (blang != null)
          lang = Bytes.toString(blang.array());
        assertEquals(metalanguages[t], lang);
      }
View Full Code Here
TOP

Related Classes of org.apache.nutch.storage.WebPage

  • org.apache.gora.persistency.impl.StateManagerImpl
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
  • org.apache.nutch.api.DbReader$DbIterator
  • org.apache.nutch.crawl.DbUpdateReducer
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.TestGenerator
  • org.apache.nutch.crawl.TestInjector
  • org.apache.nutch.crawl.TestURLPartitioner
  • org.apache.nutch.crawl.WebTableReader
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.