Package org.apache.nutch.parse

Examples of org.apache.nutch.parse.ParseImpl

76
77
78
79
80
81
82
83
84
85
86
      title = "";
    }

    String text = delegate.getText();

    return new ParseImpl(text,
                         new ParseData(ParseStatus.STATUS_SUCCESS,
                                       title,
                                       OutlinkExtractor
        .                              getOutlinks(text, this.conf),
                                       content.getMetadata(),
View Full Code Here
82
83
84
85
86
87
88
89
90
      Outlink[] newlinks = (Outlink[])outlinks.toArray(new Outlink[outlinks.size()]);
      ParseData parseData = new ParseData(status, title, newlinks,
                                          parse.getData().getContentMeta(),
                                          parse.getData().getParseMeta());
      parseData.setConf(this.conf);
      parse = new ParseImpl(text, parseData);
    }
    return parse;
  }
View Full Code Here
160
161
162
163
164
165
166
167
      title = script.substring(0, idx);
    }
    ParseData pd = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks,
                                 c.getMetadata());
    pd.setConf(this.conf);
    Parse parse = new ParseImpl(script, pd);
    return parse;
  }
View Full Code Here
152
153
154
155
156
157
158
159
160
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title,
                                        outlinks, content.getMetadata(),
                                        metadata);
    parseData.setConf(this.conf);
    return new ParseImpl(text, parseData);
    // any filter?
    //return HtmlParseFilters.filter(content, parse, root);
  }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
      while ((line = br.readLine()) != null) {
        if (text.length() > 0) text.append("\n");
        text.append(line);
      }
      br.close();
      byte[] signature = sig.calculate(null, new ParseImpl(text.toString(), null));
      res.put(files[i].toString(), signature);
    }
    Iterator it = res.keySet().iterator();
    while (it.hasNext()) {
      String name = (String)it.next();
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  conf.setBoolean("moreIndexingFilter.indexMimeTypeParts", false);
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);
  assertNotNull(filter);
  NutchDocument doc = new NutchDocument();
        ParseImpl parse = new ParseImpl("foo bar", new ParseData());
       
  try{
      filter.filter(doc, parse, new Text("http://nutch.apache.org/index.html"), new CrawlDatum(), new Inlinks());
  }
        catch(Exception e){
View Full Code Here
81
82
83
84
85
86
87
88
89
90
  private void assertContentType(Configuration conf, String source, String expected) throws IndexingException {
    Metadata metadata = new Metadata();
    metadata.add(Response.CONTENT_TYPE, source);
    MoreIndexingFilter filter = new MoreIndexingFilter();
    filter.setConf(conf);
    NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
        new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
        "http://www.example.com/"), new CrawlDatum(), new Inlinks());
    assertEquals("mime type not detected", expected, doc.getFieldValue("type"));
  }
View Full Code Here
197
198
199
200
201
202
203
        //   LOG.info("Outlinks: "+outlinks);
        // }

        ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS,
                contentTitle.toString(), outlinks, content.getMetadata());
        return ParseResult.createParseResult(content.getUrl(), new ParseImpl(indexText.toString(), parseData));
    }
View Full Code Here
61
62
63
64
65
66
67
68
69
      List list = Arrays.asList(old);
      outlinks.addAll(list);
      ParseStatus status = parse.getData().getStatus();
      String text = parse.getText();
      Outlink[] newlinks = (Outlink[])outlinks.toArray(new Outlink[outlinks.size()]);
      parse = new ParseImpl(text, new ParseData(status, title, newlinks, metadata));
    }
    return parse;
  }
View Full Code Here
138
139
140
141
142
143
144
145
    }
    Properties metadata = new Properties();
    metadata.putAll(c.getMetadata());
    ParseData pd = new ParseData(ParseStatus.STATUS_SUCCESS, title,
            outlinks, metadata);
    Parse parse = new ParseImpl(script, pd);
    return parse;
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.parse.ParseImpl

  • org.apache.nutch.crawl.TextProfileSignature
  • org.apache.nutch.indexer.anchor.TestAnchorIndexingFilter
  • org.apache.nutch.indexer.basic.TestBasicIndexingFilter
  • org.apache.nutch.indexer.field.BasicFields$Extractor
  • org.apache.nutch.indexer.IndexerMapReduce
  • org.apache.nutch.indexer.more.TestMoreIndexingFilter
  • org.apache.nutch.indexer.staticfield.TestStaticFieldIndexerTest
  • org.apache.nutch.indexer.TestIndexingFilters
  • org.apache.nutch.parse.ext.ExtParser
  • org.apache.nutch.parse.ext.WaxExtParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.