Package org.apache.nutch.crawl

Examples of org.apache.nutch.crawl.UrlWithScore


*/
public class TestUrlWithScore extends TestCase {

  public void testSerialization() throws IOException {
    // create a key and test basic functionality
    UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
    assertEquals("http://example.org/", keyOut.getUrl().toString());
    assertEquals(1f, keyOut.getScore().get(), 0.001);
   
    // write to out
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(bos);
    keyOut.write(out);

    // read from in
    UrlWithScore keyIn = new UrlWithScore();
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream in = new DataInputStream(bis);
    keyIn.readFields(in);
    assertEquals(keyOut.getUrl().toString(), keyIn.getUrl().toString());
    assertEquals(keyOut.getScore().get(), keyIn.getScore().get(), 0.001);

    in.close();
    out.close();
  }
View Full Code Here


  }
 
  public void testPartitioner() throws IOException {
    UrlOnlyPartitioner part = new UrlOnlyPartitioner();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    int numReduces = 7;
   
    // keys 1 and 2 should be partitioned together
    int partForKey1 = part.getPartition(k1, null, numReduces);
View Full Code Here

  }
 
  public void testUrlOnlySorting() throws IOException {
    UrlOnlyComparator comp = new UrlOnlyComparator();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    // k1 should be equal to k2
    assertEquals(0, compareBothRegularAndRaw(comp, k1, k2));
    // test symmetry
    assertEquals(0, compareBothRegularAndRaw(comp, k2, k1));
View Full Code Here

  }
 
  public void testUrlScoreSorting() throws IOException {
    UrlScoreComparator comp = new UrlScoreComparator();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    // k1 is after k2, because score is lower
    assertEquals(1, comp.compare(k1, k2));
    // test symmetry
    assertEquals(-1, comp.compare(k2, k1));
View Full Code Here

public class TestUrlWithScore {

  @Test
  public void testSerialization() throws IOException {
    // create a key and test basic functionality
    UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
    assertEquals("http://example.org/", keyOut.getUrl().toString());
    assertEquals(1f, keyOut.getScore().get(), 0.001);
   
    // write to out
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(bos);
    keyOut.write(out);

    // read from in
    UrlWithScore keyIn = new UrlWithScore();
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream in = new DataInputStream(bis);
    keyIn.readFields(in);
    assertEquals(keyOut.getUrl().toString(), keyIn.getUrl().toString());
    assertEquals(keyOut.getScore().get(), keyIn.getScore().get(), 0.001);

    in.close();
    out.close();
  }
View Full Code Here

 
  @Test
  public void testPartitioner() throws IOException {
    UrlOnlyPartitioner part = new UrlOnlyPartitioner();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    int numReduces = 7;
   
    // keys 1 and 2 should be partitioned together
    int partForKey1 = part.getPartition(k1, null, numReduces);
View Full Code Here

 
  @Test
  public void testUrlOnlySorting() throws IOException {
    UrlOnlyComparator comp = new UrlOnlyComparator();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    // k1 should be equal to k2
    assertEquals(0, compareBothRegularAndRaw(comp, k1, k2));
    // test symmetry
    assertEquals(0, compareBothRegularAndRaw(comp, k2, k1));
View Full Code Here

 
  @Test
  public void testUrlScoreSorting() throws IOException {
    UrlScoreComparator comp = new UrlScoreComparator();
   
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
   
    // k1 is after k2, because score is lower
    assertEquals(1, comp.compare(k1, k2));
    // test symmetry
    assertEquals(-1, comp.compare(k2, k1));
View Full Code Here

TOP

Related Classes of org.apache.nutch.crawl.UrlWithScore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.