Package org.apache.nutch.protocol

Examples of org.apache.nutch.protocol.Content

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
    }

    public void map(Writable key, Content value,
        OutputCollector<BytesWritable, BytesWritable> output, Reporter reporter)
        throws IOException {
        Content content = (Content) value;

        if (content.getContentType().toLowerCase().startsWith("image/jpeg")) {
            try {
                //MessageDigest md = MessageDigest.getInstance("SHA-1");
                output.collect(new BytesWritable(content.getUrl().getBytes()),
                            new BytesWritable(content.getContent()));
            } catch (Exception e) {
            }
        }
    }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    if (LOG.isInfoEnabled()) { LOG.info("fetching: "+url); }

    Configuration conf = NutchConfiguration.create();
    ProtocolFactory factory = new ProtocolFactory(conf);
    Protocol protocol = factory.getProtocol(url);
    Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();

    if (force) {
      content.setContentType(contentType);
    } else {
      contentType = content.getContentType();
    }

    if (contentType == null) {
      System.err.println("");
      System.exit(-1);
View Full Code Here
122
123
124
125
126
127
128
129
130
131
    String file = args[0];
    byte[] raw = getRawBytes(new File(file));

    Metadata meta = new Metadata();
    meta.set(Response.CONTENT_LENGTH, "" + raw.length);
    Content content = new Content(file, file, raw, mime, meta,
                                  NutchConfiguration.create());

    System.out.println(parser.getParse(content).getText());
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  protected void tearDown() {}

  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    for (int i=0; i<sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
          String contentType = MIME.getMimeType(fname).getName();
          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content);
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
           
            for(int count = 0; count < theOutlinks.length; count++) {
View Full Code Here
215
216
217
218
219
220
221
222
223
224
    String url = args[0];
    Configuration conf = NutchConfiguration.create();
    RSSParser parser = new RSSParser();
    parser.setConf(conf);
    Protocol protocol = new ProtocolFactory(conf).getProtocol(url);
    Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();
    Parse parse = parser.getParse(content);
    System.out.println("data: "+ parse.getData());
    System.out.println("text: "+parse.getText());
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  protected void tearDown() {}

  public void testIt() throws ProtocolException, ParseException {
    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
    ArrayList<String> handledurls=new ArrayList<String>();
   
    READ:
      do {
      Text key=new Text();
      Content value=new Content();
      if(!reader.next(key, value)) break READ;
      String contentString=new String(value.getContent());
      if(contentString.indexOf("Nutch fetcher test page")!=-1) {
        handledurls.add(key.toString());
      }
    } while(true);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
  public void testIt() throws ProtocolException, ParseException {

    String urlString;
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    ParseUtil parser = new ParseUtil(conf);
    ProtocolFactory factory = new ProtocolFactory(conf);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
     * </ul>
     */
    public void testIt() throws ProtocolException, ParseException {
        String urlString;
        Protocol protocol;
        Content content;
        Parse parse;

        Configuration conf = NutchConfiguration.create();
        for (int i = 0; i < sampleFiles.length; i++) {
            urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
View Full Code Here
TOP

Related Classes of org.apache.nutch.protocol.Content

  • com.atlantbh.nutch.filter.xpath.XPathHtmlParserFilterTest
  • com.flaptor.hounder.crawler.Nutch9Fetcher$NutchSegment$SegmentIterator
  • nise.ExtractNutch
  • org.apache.commons.cli.Options
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.util.GenericOptionsParser
  • org.apache.nutch.analysis.lang.LanguageIdentifier
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.