Examples of WebUrl

  • edu.uci.ics.crawler4j.url.WebURL
    @author Yasser Ganjisaffar
  • org.apache.manifoldcf.crawler.connectors.rss.WebURL
    Replacement class for java.net.URI, which is broken in many ways.
  • org.apache.manifoldcf.crawler.connectors.webcrawler.WebURL
    Replacement class for java.net.URI, which is broken in many ways.

    • Examples of edu.uci.ics.crawler4j.url.WebURL

    341
    342
    343
    344
    345
    346
    347
    348
    349
    350
    351
    352
    353
    354
          } catch (Exception e) {
            logger.error("Could not add seed: " + e.getMessage());
          }
        }

        WebURL webUrl = new WebURL();
        webUrl.setURL(canonicalUrl);
        webUrl.setDocid(docId);
        webUrl.setDepth((short) 0);
        if (robotstxtServer.allowedIn(webUrl) == null) {
          logger.info("Robots.txt does not allow this seed: " + pageUrl);
        } else {
          frontier.schedule(webUrl);
        }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.rss.WebURL

    27
    28
    29
    30
    31
    32
    33
    34
    35
      @Test
      public void absolutePath()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com");
        WebURL resolved = parent.resolve("http://bar.com");
        assertEquals(resolved.toASCIIString(),"http://bar.com");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.rss.WebURL

    36
    37
    38
    39
    40
    41
    42
    43
    44
      @Test
      public void relativePath()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com/abc/def.html");
        WebURL resolved = parent.resolve("/def/ghi.html");
        assertEquals(resolved.toASCIIString(),"http://foo.com/def/ghi.html");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.rss.WebURL

    45
    46
    47
    48
    49
    50
    51
    52
    53
      @Test
      public void noSlashDocument()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com");
        WebURL resolved = parent.resolve("hello.pdf");
        assertEquals(resolved.toASCIIString(),"http://foo.com/hello.pdf");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.rss.WebURL

    54
    55
    56
    57
    58
    59
    60
    61
    62
      @Test
      public void relativeQuery()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com/abc/def/ghi.asmx?q=foo");
        WebURL resolved = parent.resolve("?q=bar");
        assertEquals(resolved.toASCIIString(),"http://foo.com/abc/def/ghi.asmx?q=bar");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.rss.WebURL

    63
    64
    65
    66
    67
    68
    69
    70
    71
      @Test
      public void queryEscaping()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com/abc/def/ghi.asmx?q=foo%3Dbar");
        WebURL resolved = parent.resolve("?q=bar%3Dfoo");
        assertEquals(resolved.toASCIIString(),"http://foo.com/abc/def/ghi.asmx?q=bar%3Dfoo");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.webcrawler.WebURL

    27
    28
    29
    30
    31
    32
    33
    34
    35
      @Test
      public void absolutePath()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com");
        WebURL resolved = parent.resolve("http://bar.com");
        assertEquals(resolved.toASCIIString(),"http://bar.com");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.webcrawler.WebURL

    36
    37
    38
    39
    40
    41
    42
    43
    44
      @Test
      public void relativePath()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com/abc/def.html");
        WebURL resolved = parent.resolve("/def/ghi.html");
        assertEquals(resolved.toASCIIString(),"http://foo.com/def/ghi.html");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.webcrawler.WebURL

    45
    46
    47
    48
    49
    50
    51
    52
    53
      @Test
      public void noSlashDocument()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com");
        WebURL resolved = parent.resolve("hello.pdf");
        assertEquals(resolved.toASCIIString(),"http://foo.com/hello.pdf");
      }
    View Full Code Here

    Examples of org.apache.manifoldcf.crawler.connectors.webcrawler.WebURL

    54
    55
    56
    57
    58
    59
    60
    61
    62
      @Test
      public void relativeQuery()
        throws Exception
      {
        WebURL parent = new WebURL("http://foo.com/abc/def/ghi.asmx?q=foo");
        WebURL resolved = parent.resolve("?q=bar");
        assertEquals(resolved.toASCIIString(),"http://foo.com/abc/def/ghi.asmx?q=bar");
      }
    View Full Code Here
    TOP
    Copyright © 2018 www.massapi.com. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.