Package org.apache.nutch.parse

Examples of org.apache.nutch.parse.ParseImpl

75
76
77
78
79
80
81
82
83
84
85
      title = "";
    }

    String text = delegate.getText();

    return new ParseImpl(text,
                         new ParseData(ParseStatus.STATUS_SUCCESS,
                                       title,
                                       OutlinkExtractor
        .                              getOutlinks(text, this.conf),
                                       content.getMetadata(),
View Full Code Here
103
104
105
106
107
108
109
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
                                        metadataCollector.getTitle(),
                                        metadataCollector.getOutlinks(),
                                        contentMeta,
                                        metadataCollector.getData());
    return new ParseImpl(metadataCollector.getText(), parseData);
  }
View Full Code Here
131
132
133
134
135
136
137
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
                                        metadataCollector.getTitle(),
                                        metadataCollector.getOutlinks(),
                                        contentMeta,
                                        metadataCollector.getData());
    return new ParseImpl(metadataCollector.getText(), parseData);
  }
View Full Code Here
131
132
133
134
135
136
137
    Outlink[] outlinks = OutlinkExtractor.getOutlinks(text, getConf());

    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title,
                                        outlinks, content.getMetadata());
    parseData.setConf(this.conf);
    return new ParseImpl(text, parseData);
  }
View Full Code Here
100
101
102
103
104
105
106
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title,
                                        outlinks, content.getMetadata(),
                                        metadata);
    parseData.setConf(this.conf);
    return new ParseImpl(text, parseData);
  }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
      while ((line = br.readLine()) != null) {
        if (text.length() > 0) text.append("\n");
        text.append(line);
      }
      br.close();
      byte[] signature = sig.calculate(null, new ParseImpl(text.toString(), null));
      res.put(files[i].toString(), signature);
    }
    Iterator it = res.keySet().iterator();
    while (it.hasNext()) {
      String name = (String)it.next();
View Full Code Here
40
41
42
43
44
45
46
47
48
    String class1 = "NonExistingFilter";
    String class2 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);

    IndexingFilters filters = new IndexingFilters(conf);
    filters.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
        new ParseStatus(), "title", new Outlink[0], new Metadata())), new Text(
        "http://www.example.com/"), new CrawlDatum(), new Inlinks());
  }
View Full Code Here
100
101
102
103
104
105
106
    final ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
                                              resultTitle, outlinks,
                                              content.getMetadata());

    if (LOG.isTraceEnabled()) { LOG.trace("Zip file parsed sucessfully !!"); }
    return ParseResult.createParseResult(content.getUrl(), new ParseImpl(resultText, parseData));
  }
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
          Integer.toString(metaTags.getRefreshTime()) });
    }
    ParseData parseData = new ParseData(status, title, outlinks, content
        .getMetadata(), nutchMetadata);
    ParseResult parseResult = ParseResult.createParseResult(content
        .getUrl(), new ParseImpl(text, parseData));

    // run filters on parse
    ParseResult filteredParse = this.htmlParseFilters.filter(content,
        parseResult, metaTags, root);
    if (metaTags.getNoCache()) { // not okay to cache
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
    doc.add("segment", metadata.get(Nutch.SEGMENT_NAME_KEY));

    // add digest, used by dedup
    doc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));

    final Parse parse = new ParseImpl(parseText, parseData);
    try {
      // extract information from dbDatum and pass it to
      // fetchDatum so that indexing filters can use it
      final Text url = (Text) dbDatum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY);
      if (url != null) {
View Full Code Here
TOP

Related Classes of org.apache.nutch.parse.ParseImpl

  • org.apache.nutch.crawl.TextProfileSignature
  • org.apache.nutch.indexer.anchor.TestAnchorIndexingFilter
  • org.apache.nutch.indexer.basic.TestBasicIndexingFilter
  • org.apache.nutch.indexer.field.BasicFields$Extractor
  • org.apache.nutch.indexer.IndexerMapReduce
  • org.apache.nutch.indexer.more.TestMoreIndexingFilter
  • org.apache.nutch.indexer.staticfield.TestStaticFieldIndexerTest
  • org.apache.nutch.indexer.TestIndexingFilters
  • org.apache.nutch.parse.ext.ExtParser
  • org.apache.nutch.parse.ext.WaxExtParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.