Examples of org.apache.nutch.crawl.UrlWithScore

org.apache.nutch.crawl.UrlWithScore
A writable comparable container for an url with score. Provides a {@link Partitioner} and {@link RawComparator} implementations for easyintegration with jobs.

 */
public class TestUrlWithScore extends TestCase {


  public void testSerialization() throws IOException {
    // create a key and test basic functionality
    UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
    assertEquals("http://example.org/", keyOut.getUrl().toString());
    assertEquals(1f, keyOut.getScore().get(), 0.001);
    
    // write to out
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(bos);
    keyOut.write(out);


    // read from in
    UrlWithScore keyIn = new UrlWithScore();
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream in = new DataInputStream(bis);
    keyIn.readFields(in);
    assertEquals(keyOut.getUrl().toString(), keyIn.getUrl().toString());
    assertEquals(keyOut.getScore().get(), keyIn.getScore().get(), 0.001);


    in.close();
    out.close();
  }

View Full Code Here

  }
  
  public void testPartitioner() throws IOException {
    UrlOnlyPartitioner part = new UrlOnlyPartitioner();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    int numReduces = 7;
    
    // keys 1 and 2 should be partitioned together
    int partForKey1 = part.getPartition(k1, null, numReduces);

View Full Code Here

  }
  
  public void testUrlOnlySorting() throws IOException {
    UrlOnlyComparator comp = new UrlOnlyComparator();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    // k1 should be equal to k2
    assertEquals(0, compareBothRegularAndRaw(comp, k1, k2));
    // test symmetry
    assertEquals(0, compareBothRegularAndRaw(comp, k2, k1));

View Full Code Here

  }
  
  public void testUrlScoreSorting() throws IOException {
    UrlScoreComparator comp = new UrlScoreComparator();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    // k1 is after k2, because score is lower
    assertEquals(1, comp.compare(k1, k2));
    // test symmetry
    assertEquals(-1, comp.compare(k2, k1));

View Full Code Here

public class TestUrlWithScore {


  @Test
  public void testSerialization() throws IOException {
    // create a key and test basic functionality
    UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
    assertEquals("http://example.org/", keyOut.getUrl().toString());
    assertEquals(1f, keyOut.getScore().get(), 0.001);
    
    // write to out
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(bos);
    keyOut.write(out);


    // read from in
    UrlWithScore keyIn = new UrlWithScore();
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream in = new DataInputStream(bis);
    keyIn.readFields(in);
    assertEquals(keyOut.getUrl().toString(), keyIn.getUrl().toString());
    assertEquals(keyOut.getScore().get(), keyIn.getScore().get(), 0.001);


    in.close();
    out.close();
  }

View Full Code Here

  
  @Test
  public void testPartitioner() throws IOException {
    UrlOnlyPartitioner part = new UrlOnlyPartitioner();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    int numReduces = 7;
    
    // keys 1 and 2 should be partitioned together
    int partForKey1 = part.getPartition(k1, null, numReduces);

View Full Code Here

  
  @Test
  public void testUrlOnlySorting() throws IOException {
    UrlOnlyComparator comp = new UrlOnlyComparator();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    // k1 should be equal to k2
    assertEquals(0, compareBothRegularAndRaw(comp, k1, k2));
    // test symmetry
    assertEquals(0, compareBothRegularAndRaw(comp, k2, k1));

View Full Code Here

  
  @Test
  public void testUrlScoreSorting() throws IOException {
    UrlScoreComparator comp = new UrlScoreComparator();
    
    UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
    UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
    UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
    UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
    UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
    
    // k1 is after k2, because score is lower
    assertEquals(1, comp.compare(k1, k2));
    // test symmetry
    assertEquals(-1, comp.compare(k2, k1));

View Full Code Here

TOP

Related Classes of org.apache.nutch.crawl.UrlWithScore

org.apache.hadoop.io.FloatWritable

org.apache.hadoop.io.Text

org.apache.nutch.crawl.TestUrlWithScore

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.