Package org.apache.nutch.protocol

Examples of org.apache.nutch.protocol.Content

304
305
306
307
308
309
310
311
312
313
314
    in.readFully(bytes);
    Configuration conf = NutchConfiguration.create();
    HtmlParser parser = new HtmlParser();
    parser.setConf(conf);
    Parse parse = parser.getParse(
            new Content(url, url, bytes, "text/html", new Metadata(), conf)).get(url);
    System.out.println("data: "+parse.getData());

    System.out.println("text: "+parse.getText());
   
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  @Test
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;
    Configuration conf = NutchConfiguration.create();

    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();

      parse = new ParseUtil(conf).parse(content).get(content.getUrl());

      String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
      Assert.assertTrue(sampleTexts[i].equals(text));
    }
  }
View Full Code Here
102
103
104
105
106
107
108
109
  }

  protected Parse parse(byte[] contentBytes) {
    String dummyUrl = "http://dummy.url/";
    return parser.getParse(
        new Content(dummyUrl, dummyUrl, contentBytes, "text/html", new Metadata(),
            conf)).get(dummyUrl);
  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
  @Test
  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-zip",content).get(content.getUrl());
      Assert.assertTrue(parse.getText().equals(expectedText));
    }
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    if (!output.getStatus().isSuccess()) {
      System.out.println("Fetch failed with protocol status: " + output.getStatus());
      return 0;
    }
        
    Content content = output.getContent();

    if (content == null) {
      System.out.println("No content for " + url);
      return 0;
    }

    contentType = content.getContentType();

    if (contentType == null) {
      return -1;
    }
View Full Code Here
372
373
374
375
376
377
378
379
380
381
382
  public void reduce(Text key, Iterator<MetaWrapper> values,
      OutputCollector<Text, MetaWrapper> output, Reporter reporter) throws IOException {
    CrawlDatum lastG = null;
    CrawlDatum lastF = null;
    CrawlDatum lastSig = null;
    Content lastC = null;
    ParseData lastPD = null;
    ParseText lastPT = null;
    String lastGname = null;
    String lastFname = null;
    String lastSigname = null;
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
          }
          addMyHeader(res, "ProtocolStatus", ps.toString());
        } else {
          res.setStatus(HttpServletResponse.SC_OK);         
        }
        Content c = seg.getContent(url);
        if (c == null) { // missing content
          req.setHandled(true);
          res.addHeader("X-Handled-By", getClass().getSimpleName());
          return;
        }
        byte[] data = c.getContent();
        LOG.debug("-data len=" + data.length);
        Metadata meta = c.getMetadata();
        String[] names = meta.names();
        LOG.debug("- " + names.length + " meta");
        for (int i = 0; i < names.length; i++) {
          boolean my = true;
          char ch = names[i].charAt(0);
View Full Code Here
114
115
116
117
118
119
120
    public Content getContent(Text url) throws IOException {
      synchronized (cLock) {
        if (content == null)
          content = getReaders(Content.DIR_NAME);
      }
      return (Content)getEntry(content, url, new Content());
    }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
      file.setMaxContentLength(maxContentLength);

    // set log level
    //LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));

    Content content = file.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();

    System.err.println("Content-Type: " + content.getContentType());
    System.err.println("Content-Length: " +
                       content.getMetadata().get(Response.CONTENT_LENGTH));
    System.err.println("Last-Modified: " +
                       content.getMetadata().get(Response.LAST_MODIFIED));
    if (dumpContent) {
      System.out.print(new String(content.getContent()));
    }

    file = null;
  }
View Full Code Here
87
88
89
90
91
92
93
94
  public byte[] getContent() {
    return content;
  }

  public Content toContent() {
    return new Content(orig, base, (content != null ? content : EMPTY_CONTENT),
        getHeader(Response.CONTENT_TYPE), headers, this.conf);
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.protocol.Content

  • com.atlantbh.nutch.filter.xpath.XPathHtmlParserFilterTest
  • com.flaptor.hounder.crawler.Nutch9Fetcher$NutchSegment$SegmentIterator
  • nise.ExtractNutch
  • org.apache.commons.cli.Options
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.util.GenericOptionsParser
  • org.apache.nutch.analysis.lang.LanguageIdentifier
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.