Examples of org.terrier.structures.seralization.FixedSizeTextFactory

Package org.terrier.structures.seralization

Examples of org.terrier.structures.seralization.FixedSizeTextFactory

org.terrier.structures.seralization.FixedSizeTextFactory
Makes a {@link org.apache.hadoop.io.Text} object that has a fixed size when written using {@link org.apache.hadoop.io.Writable} methods. @author Craig Macdonald @since 3.0 @see org.apache.hadoop.io.Text @see FixedSizeWriteableFactory @see org.apache.hadoop.io.Writable

    for(i=0;i<forwardKeys.length;i++)
    {
      forwardWriters[i] = new MultiFSOMapWriter(
        _index.getPath() + "/" + _index.getPrefix() + "."+structureName+"-"+i+FSOrderedMapFile.USUAL_EXTENSION, 
        REVERSE_KEY_LOOKUP_WRITING_BUFFER_SIZE, 
        keyFactories[i] = new FixedSizeTextFactory(valueLensChars[forwardKeys[i]]), 
        new FixedSizeIntWritableFactory()
        );
      forwardKeyValuesSorted[i] = true;
    }

View Full Code Here

    /* open a MapFileWriter for the specified key. This will automatically promoted to the index folder when the job is finished */
    protected MapFileWriter openMapFileWriter(String keyName) throws IOException
    {
      final int metaKeyIndex = key2reverseOffset.get(keyName);
      final int valueLength = key2valuelength.get(keyName);
      keyFactory = new FixedSizeTextFactory(valueLength);
      //logger.info("Opening MapFileWriter for key "+ keyName + " - index " + metaKeyIndex);
      return FSOrderedMapFile.mapFileWrite(reduceTaskFileDestinations.toString() /*index.getPath()*/ 
            + "/" + index.getPrefix() + "."
            + jc.get("MetaIndexInputStreamRecordReader.structureName")
            + "-"+metaKeyIndex+FSOrderedMapFile.USUAL_EXTENSION

View Full Code Here

    {
      if (keyName.trim().equals(""))
        continue;
      key2forwardOffset.put(keyName, 1+i);
      logger.debug("Forward key "+ keyName +", length="+ key2bytelength.get(keyName));
      keyFactories[i] = new FixedSizeTextFactory(key2stringlength.get(keyName));
      String filename = path+ApplicationSetup.FILE_SEPARATOR+prefix+"."+structureName+"-"+i+FSOrderedMapFile.USUAL_EXTENSION;
      String loadFormat = index.getIndexProperty("index."+structureName+".reverse."+keyName+".in-mem", "false");
      if (loadFormat.equals("hashmap"))
      {
        //logger.info("Structure "+ structureName + " reading reverse map for key "+ keyName + " into memory as hashmap");

View Full Code Here

  @Override
  protected void setUp() throws Exception {
    super.setUp();
    
    MapFileWriter w = FSOrderedMapFile.mapFileWrite(file);
    FixedSizeWriteableFactory<Text> keyFactory = new FixedSizeTextFactory(20);
    int offset = 0;
    for(String key : testKeys)
    {
      Text wkey = keyFactory.newInstance();
      IntWritable wvalue = new IntWritable();
      wkey.set(key);
      wvalue.set(offset);


      w.write(wkey, wvalue);

View Full Code Here

    assertEquals(testKeys.length, entryIndex);
  }
  
  @Test public void testStream() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    FSOrderedMapFile.EntryIterator<Text, IntWritable> inputStream = new FSOrderedMapFile.EntryIterator<Text, IntWritable>(
        file, keyFactory, new FixedSizeIntWritableFactory());
    readStream(inputStream);
    inputStream.close();
  }

View Full Code Here

    inputStream.close();
  }
  
  @Test public void testStreamSkip() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    FSOrderedMapFile.EntryIterator<Text, IntWritable> inputStream = new FSOrderedMapFile.EntryIterator<Text, IntWritable>(
        file, keyFactory, new FixedSizeIntWritableFactory());
    readStreamSkip(inputStream, testKeys.length);
    inputStream.close();
  }

View Full Code Here

    inputStream.close();
  }
  
  @Test public void testOnDisk() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    FSOrderedMapFile<Text, IntWritable> mapfile = new FSOrderedMapFile<Text, IntWritable>(file, false, keyFactory, new FixedSizeIntWritableFactory());
    checkKeys(keyFactory, mapfile);
  }

View Full Code Here

    checkKeys(keyFactory, mapfile);
  }
  
  @Test public void testInMemory() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    Map<Text, IntWritable> mapfileInMem = new FSOrderedMapFile.MapFileInMemory<Text, IntWritable>(file, keyFactory, new FixedSizeIntWritableFactory());
    checkKeys(keyFactory, mapfileInMem);
  }

View Full Code Here

    checkKeys(keyFactory, mapfileInMem);
  }
  
  @Test public void testInMemoryRandomDataInputMemory() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    Map<Text, IntWritable> mapfileInMem = new FSOrderedMapFile<Text, IntWritable>(
        new RandomDataInputMemory(file), file,
        keyFactory, new FixedSizeIntWritableFactory());
    checkKeys(keyFactory, mapfileInMem);
  }

View Full Code Here

  }
  
  @Test public void testVariousStrings() throws Exception
  {
    int length = 20;
    FixedSizeWriteableFactory<Text> factory = new FixedSizeTextFactory(length);
    
    String[] testStrings = {
        "", "a", "abat", 
        "1234567890", "123456789001234567890",
        "\u0290\u0290",

View Full Code Here

0 1

TOP

Related Classes of org.terrier.structures.seralization.FixedSizeTextFactory

org.terrier.structures.collections.TestFSOrderedMapFile

org.terrier.structures.CompressingMetaIndex

org.terrier.structures.indexing.CompressingMetaIndexBuilder

org.terrier.structures.indexing.CompressingMetaIndexBuilder$MapperReducer

org.terrier.structures.serialization.TestFixedSizeTextFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.