Package org.apache.nutch.parse

Examples of org.apache.nutch.parse.ParseImpl

157
158
159
160
161
162
163
    // collect meta data
    Properties metaData = new Properties();
    metaData.putAll(content.getMetadata()); // copy through

    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
    return new ParseImpl(text, parseData);
  }
View Full Code Here
218
219
220
221
222
223
224
        LOG.fine("nutch:parse-rss:getParse:found " + outlinks.length + " outlinks");
        // LOG.info("Outlinks: "+outlinks);

        ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
                contentTitle.toString(), outlinks, content.getMetadata());
        return new ParseImpl(indexText.toString(), parseData);
    }
View Full Code Here
119
120
121
122
123
124
125
126
127
    // collect outlink
    Outlink[] outlinks = OutlinkExtractor.getOutlinks(text);

    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metadata);
    return new ParseImpl(text, parseData);
    // any filter?
    //return HtmlParseFilters.filter(content, parse, root);
  }
View Full Code Here
167
168
169
170
171
172
173
174
175
    // collect meta data
    Properties metadata = new Properties();
    metadata.putAll(content.getMetadata()); // copy through

    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metadata);
    return new ParseImpl(text, parseData);
    // any filter?
    //return HtmlParseFilters.filter(content, parse, root);
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
    doc.add("segment", metadata.get(Nutch.SEGMENT_NAME_KEY));

    // add digest, used by dedup
    doc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));

    final Parse parse = new ParseImpl(parseText, parseData);
    try {
      // extract information from dbDatum and pass it to
      // fetchDatum so that indexing filters can use it
      final Text url = (Text) dbDatum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY);
      if (url != null) {
View Full Code Here
102
103
104
105
106
107
108
109
110
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
        metadataCollector.getTitle(), metadataCollector.getOutlinks(),
        contentMeta, metadataCollector.getData());
    ParseResult parseResult = ParseResult.createParseResult(content
        .getUrl(),
        new ParseImpl(metadataCollector.getText(), parseData));

    return parseResult;
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
        metadataCollector.getTitle(), metadataCollector.getOutlinks(),
        contentMeta, metadataCollector.getData());
    ParseResult parseResult = ParseResult.createParseResult(content
        .getUrl(),
        new ParseImpl(metadataCollector.getText(), parseData));

    return parseResult;
  }
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
    doc.add("segment", metadata.get(Nutch.SEGMENT_NAME_KEY));

    // add digest, used by dedup
    doc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));

    final Parse parse = new ParseImpl(parseText, parseData);
    try {
      // extract information from dbDatum and pass it to
      // fetchDatum so that indexing filters can use it
      final Text url = (Text) dbDatum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY);
      if (url != null) {
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  Text url;
  StaticFieldIndexer filter;

  protected void setUp() throws Exception {
    conf = NutchConfiguration.create();
    parse = new ParseImpl();
    url = new Text("http://nutch.apache.org/index.html");
    crawlDatum = new CrawlDatum();
    inlinks = new Inlinks();
    filter = new StaticFieldIndexer();
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
    doc.add("segment", metadata.get(Nutch.SEGMENT_NAME_KEY));

    // add digest, used by dedup
    doc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));

    final Parse parse = new ParseImpl(parseText, parseData);
    try {
      // extract information from dbDatum and pass it to
      // fetchDatum so that indexing filters can use it
      final Text url = (Text) dbDatum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY);
      if (url != null) {
View Full Code Here
TOP

Related Classes of org.apache.nutch.parse.ParseImpl

  • org.apache.nutch.crawl.TextProfileSignature
  • org.apache.nutch.indexer.anchor.TestAnchorIndexingFilter
  • org.apache.nutch.indexer.basic.TestBasicIndexingFilter
  • org.apache.nutch.indexer.field.BasicFields$Extractor
  • org.apache.nutch.indexer.IndexerMapReduce
  • org.apache.nutch.indexer.more.TestMoreIndexingFilter
  • org.apache.nutch.indexer.staticfield.TestStaticFieldIndexerTest
  • org.apache.nutch.indexer.TestIndexingFilters
  • org.apache.nutch.parse.ext.ExtParser
  • org.apache.nutch.parse.ext.WaxExtParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.