Examples of org.apache.nutch.crawl.MapWritable

org.apache.nutch.crawl.MapWritable
A writable map, with a similar behavior as java.util.HashMap. In addition to the size of key and value writable tuple two additional bytes are stored to identify the Writable classes. This means that a maximum of 255 different class types can be used for key and value objects. A binary-id to class mapping is defined in a static block of this class. However it is possible to use custom implementations of Writable. For these custom Writables we write the byte id - utf class name tuple into the header of each MapWritable that uses these types. @author Stefan Groschupf @deprecated Use org.apache.hadoop.io.MapWritable instead.

    assertFalse(map.containsKey(new UTF8("" + 1)));


  }


  public void testWritable() throws Exception {
    MapWritable datum1 = new MapWritable();
    for (int i = 0; i < 100; i++) {
      datum1.put(new LongWritable(i), new UTF8("" + 1));
    }
    assertEquals(100, datum1.size());
    testWritable(datum1);


    MapWritable datum2 = new MapWritable();
    for (int i = 0; i < 100; i++) {
      datum2.put(new DummyWritable(i), new DummyWritable(i));
    }
    assertEquals(100, datum2.size());
    testWritable(datum2);


    CrawlDatum c = new CrawlDatum(CrawlDatum.STATUS_DB_FETCHED, 1f);
    c.setMetaData(new MapWritable());
    for (int i = 0; i < 100; i++) {
      c.getMetaData().put(new LongWritable(i), new UTF8("" + 1));
    }
    testWritable(c);
  }

View Full Code Here

    }
    testWritable(c);
  }
  
  public void testEquals() {
    MapWritable map1 = new MapWritable();
    MapWritable map2 = new MapWritable();
    map1.put(new UTF8("key1"), new UTF8("val1"));
    map1.put(new UTF8("key2"), new UTF8("val2"));
    map2.put(new UTF8("key2"), new UTF8("val2"));
    map2.put(new UTF8("key1"), new UTF8("val1"));
    assertTrue(map1.equals(map2));
  }

View Full Code Here

        fs, file, IntWritable.class, MapWritable.class);
    // write map
    System.out.println("start writing map's");
    long start = System.currentTimeMillis();
    IntWritable key = new IntWritable();
    MapWritable map = new MapWritable();
    LongWritable mapValue = new LongWritable();
    for (int i = 0; i < 1000000; i++) {
      key.set(i);
      mapValue.set(i);
      map.put(key, mapValue);
      writer.append(key, map);
    }
    long needed = System.currentTimeMillis() - start;
    writer.close();
    System.out.println("needed time for writing map's: " + needed);

View Full Code Here

0 1

TOP

Related Classes of org.apache.nutch.crawl.MapWritable

org.apache.hadoop.io.DataInputBuffer

org.apache.hadoop.io.DataOutputBuffer

org.apache.hadoop.io.Writable

org.apache.nutch.crawl.TestMapWritable

org.apache.nutch.tools.compat.CrawlDbConverter

org.archive.access.nutch.indexer.WaxIndexingFilter

org.archive.access.nutch.jobs.ImportArcs

org.archive.access.nutch.jobs.ImportArcs$WaxFetcherOutputFormat

java.util.LinkedList

java.util.Iterator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.