Package org.terrier.utility.io

Examples of org.terrier.utility.io.RandomDataInputMemory$Seekable


      throw new IOException("ArrayFileInMem too big: > Integer.MAX_VALUE");
    byte b[] = new byte[(int)len];
    DataInputStream dis = new DataInputStream(Files.openFileStream(filename));
    dis.readFully(b);
    dis.close();
    this.dataFile = new RandomDataInputMemory(b);
    this.entrySize = factory.getSize();
    this.numberOfEntries = (int)(len / (long)entrySize);
    //System.err.println("document index: "+ this.numberOfEntries + " entries of size "+ entrySize);
    value = factory.newInstance();
  }
View Full Code Here


   * @param filename
   * @throws IOException
   */
  public BitFileInMemoryLarge(String filename) throws IOException
  {
    this(new RandomDataInputMemory(filename))
  }
View Full Code Here

    {
      //logger.info("Structure "+ structureName + " loading data file into memory");
      try{
        logger.debug("Caching metadata file "+ dataFilename + " to memory");
        final DataInputStream di = new DataInputStream(Files.openFileStream(dataFilename));
        _dataSource = new RandomDataInputAccessor(new RandomDataInputMemory(di, dataFileLength));
      } catch (OutOfMemoryError oome) {
        //logger.warn("OutOfMemoryError: Structure "+ structureName + " reading data file directly from disk");
        //logger.debug("Metadata will be read directly from disk");
        RandomDataInput rfi = Files.openFileRandom(dataFilename);
        _dataSource = (rfi instanceof RandomAccessFile)
View Full Code Here

      }
      else
      {
        try{
          DataInputStream dis = new DataInputStream(Files.openFileStream(indexFilename));
          offsetLookup = new OnDiskDocid2OffsetLookup(new RandomDataInputAccessor(new RandomDataInputMemory(dis, indexFileLength)),length, dataFileLength);
          dis.close();
        }
        catch (OutOfMemoryError oome) {
          //logger.warn("OutOfMemoryError: Structure "+ structureName + " reading lookup file directly from disk");
          //logger.debug("Metadata lookup will be read directly from disk: "+ length +" entries, size "+ dataFileLength + " bytes");
          RandomDataInput rfi = Files.openFileRandom(indexFilename);
          offsetLookup = new OnDiskDocid2OffsetLookup(
            rfi instanceof RandomAccessFile
              ? new ChannelByteAccessor((RandomAccessFile)rfi)
              : new RandomDataInputAccessor(rfi),
            length, dataFileLength
            );
        }
      } 
    } else {
      //logger.warn("Structure "+ structureName + " reading lookup file directly from disk (SLOW)");
      //logger.debug("Metadata lookup will be read directly from disk: "+ length +" entries, size "+ dataFileLength + " bytes");
      RandomDataInput rfi = Files.openFileRandom(indexFilename);
      offsetLookup = new OnDiskDocid2OffsetLookup(
        rfi instanceof RandomAccessFile
          ? new ChannelByteAccessor((RandomAccessFile)rfi)
          : new RandomDataInputAccessor(rfi),
        length, dataFileLength
        );
    }
    //debug log lookups using a wrapper class
    if (logger.isDebugEnabled())
      offsetLookup = new LoggingDocid2OffsetLookup(offsetLookup);
     
   
   

    //now build the keyname and lengths into 2 maps:
    // keyname -> length & keyname -> offsets
    keyCount = keyNames.length;
    key2bytelength = new TObjectIntHashMap<String>(keyCount);
    TObjectIntHashMap<String> key2stringlength = new TObjectIntHashMap<String>(keyCount);
    key2byteoffset = new TObjectIntHashMap<String>(keyCount);
    valueByteOffsets = new int[keyCount];
    int cumulativeOffset = 0;
    for(i=0;i<keyCount;i++)
    {
      key2stringlength.put(keyNames[i], valueCharLengths[i]);
      key2bytelength.put(keyNames[i], valueByteLengths[i]);
      key2byteoffset.put(keyNames[i], cumulativeOffset);
      valueByteOffsets[i] = cumulativeOffset;
      cumulativeOffset += valueByteLengths[i];
    }
   
    key2forwardOffset = new TObjectIntHashMap<String>(2);
    final String[] forwardKeys = index.getIndexProperty("index."+structureName+".reverse-key-names", "").split("\\s*,\\s*");
    forwardMetaMaps = (Map<Text,IntWritable>[])new Map[forwardKeys.length];
    keyFactories = (FixedSizeWriteableFactory<Text>[])new FixedSizeWriteableFactory[forwardKeys.length];
    i=0;
    final FixedSizeIntWritableFactory valueFactory = new FixedSizeIntWritableFactory();
    for(String keyName : forwardKeys)
    {
      if (keyName.trim().equals(""))
        continue;
      key2forwardOffset.put(keyName, 1+i);
      logger.debug("Forward key "+ keyName +", length="+ key2bytelength.get(keyName));
      keyFactories[i] = new FixedSizeTextFactory(key2stringlength.get(keyName));
      String filename = path+ApplicationSetup.FILE_SEPARATOR+prefix+"."+structureName+"-"+i+FSOrderedMapFile.USUAL_EXTENSION;
      String loadFormat = index.getIndexProperty("index."+structureName+".reverse."+keyName+".in-mem", "false");
      if (loadFormat.equals("hashmap"))
      {
        //logger.info("Structure "+ structureName + " reading reverse map for key "+ keyName + " into memory as hashmap");
        forwardMetaMaps[i] = new FSOrderedMapFile.MapFileInMemory<Text, IntWritable>(
            filename,
            keyFactories[i],
            valueFactory);
      }
      else if (loadFormat.equals("mapfileinmem"))
      {
       
        final long revDataFileLength = Files.length(filename);
        //if (revDataFileLength > Integer.MAX_VALUE)
        //{
        //  loadFormat = "false";
        //  //logger.info("Structure "+ structureName + " reading reverse map for key "+ keyName + " - too big for memory as bytearray");
        //}
        //else
        //{ 
          //logger.info("Structure "+ structureName + " reading reverse map for key "+ keyName + " into memory as bytearray");
          DataInputStream dis = new DataInputStream(Files.openFileStream(filename));
          //final byte[] bytes = new byte[(int)revDataFileLength];
          //dis.readFully(bytes);
          //dis.close();       
          forwardMetaMaps[i] = new FSOrderedMapFile<Text, IntWritable>(
              new RandomDataInputMemory(dis, revDataFileLength),
              filename,
              keyFactories[i],
              valueFactory);
        //}
      }
View Full Code Here

  static FSOrderedMapFile<Text,LexiconEntry> loadMapFile(String filename, FixedSizeWriteableFactory<Text> keyFactory,
        FixedSizeWriteableFactory<LexiconEntry> valueFactory, String dataSource) throws IOException
    {
      if (dataSource.equals("fileinmem"))
        return new FSOrderedMapFile<Text,LexiconEntry>(
          new RandomDataInputMemory(filename),
          filename,
          keyFactory,
                    valueFactory);
      if (dataSource.equals("file"))
        return new FSOrderedMapFile<Text,LexiconEntry>(
View Full Code Here

      testBitIn(new BitInputStream(new ByteArrayInputStream(bytes)));
    }
   
    @Test public void testBitFileBuffered() throws IOException
    {
      testBitIn(new BitFileBuffered(new RandomDataInputMemory(bytes)).readReset(0l, (byte)0));
    }
View Full Code Here

    public TestCompressedBitFiles_BitFile_RandomDataInputMemory(){}
       
    @SuppressWarnings("deprecation")
    protected BitIn getBitIn() throws Exception
    {
      return new BitFile(new RandomDataInputMemory(filename)).readReset((long)0, (byte)0, new File(filename).length()-1, (byte)7);
    }
View Full Code Here

  {
    public TestCompressedBitFiles_BitFileBuffered_RandomDataInputMemory(){}
       
    protected BitIn getBitIn() throws Exception
    {
      return new BitFileBuffered(new RandomDataInputMemory(filename)).readReset((long)0, (byte)0, new File(filename).length()-1, (byte)7);
    }
View Full Code Here

      testBitInSkip(2, new BitInputStream(new ByteArrayInputStream(bytes)));
    }
   
    @Test public void testBitFileBuffered() throws IOException
    {
      testBitIn(new BitFileBuffered(new RandomDataInputMemory(bytes)).readReset(0l, (byte)0));
      testBitInSkip(1, new BitFileBuffered(new RandomDataInputMemory(bytes)).readReset(0l, (byte)0));
      testBitInSkip(2, new BitFileBuffered(new RandomDataInputMemory(bytes)).readReset(0l, (byte)0));
    }
View Full Code Here

 
  @Test public void testInMemoryRandomDataInputMemory() throws Exception
  {
    FixedSizeTextFactory keyFactory = new FixedSizeTextFactory(20);
    Map<Text, IntWritable> mapfileInMem = new FSOrderedMapFile<Text, IntWritable>(
        new RandomDataInputMemory(file), file,
        keyFactory, new FixedSizeIntWritableFactory());
    checkKeys(keyFactory, mapfileInMem);
  }
View Full Code Here

TOP

Related Classes of org.terrier.utility.io.RandomDataInputMemory$Seekable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.