Examples of org.terrier.compression.BitInputStream

org.terrier.compression.BitInputStream
This class reads from a file or an InputStream integers that can be coded with different encoding algorithms. It does not use any internal buffering, and operates with bytes. @author Roi Blanco

    * @param runNo the number of this run
    * @param _postingInRunClass the class that all postings in this class have
    */  
  public FileRunIterator(String filename, String termsFile, int runNo, Class<? extends PostingInRun> _postingInRunClass, int fieldCount) throws Exception{
    super(_postingInRunClass, runNo, fieldCount);
    mbis = new BitInputStream(filename);
    stringDIS = new DataInputStream( Files.openFileStream(termsFile) );
    if (Files.length(filename) > 0)
    {
      maxSize = mbis.readGamma();
      size = mbis.readGamma();

View Full Code Here

          
            //use a PostingInRun to decompress the postings stored in memory
            final PostingInRun pir = getPostingReader();
            pir.setDf(p.getDocF());
            pir.setTF(p.getTF());
            pir.setPostingSource(new BitInputStream(new ByteArrayInputStream(
              Docs.getMOS().getBuffer())));
            //System.err.println("temp compressed buffer size="+Docs.getMOS().getPos() + " length="+Docs.getMOS().getBuffer().length);
            //decompress the memory postings and write out to the direct file
            pir.append(bos, -1);
          }

View Full Code Here

  public PostingInRun next() {
    try{
      /** Current Posting List */
      final MapEmittedPostingList post = postingIterator.next();
      posting.setTerm(term);
      posting.setPostingSource(new BitInputStream(new ByteArrayInputStream(post.getArray())));
      posting.setDf(post.getDocumentFreq());
      posting.setTF(post.getTermFreq());
      mapNo = post.getMap();
      flushNo = post.getFlushNo();
      splitNo = post.getSplitNo();

View Full Code Here

  BitPostingIndexInputStream(String _filename, byte _fileCount, 
      Iterator<? extends BitIndexPointer> _pointerList, 
      Class<? extends IterablePosting> _postingIteratorClass, int _fieldCount) throws IOException
  {
    fileCount = 0;
    file = new BitInputStream(_filename);
    //file = new org.terrier.compression.BitFileBuffered(_filename).readReset(0l, (byte)0);
    if (DEBUG)
      file = new DebuggingBitIn(file);
    pointerList = _pointerList;
    postingIteratorClass = _postingIteratorClass;

View Full Code Here

  {
    this.index = _index;
    this.doi = _index.getDocumentIndex();
    this.structureName = _structureName;
    fileCount = Byte.parseByte(_index.getIndexProperty("index."+structureName+".data-files", "1"));
    file = new BitInputStream(getFilename(_index, structureName, fileCount, (byte)0));
    if (DEBUG)
      file = new DebuggingBitIn(file);
    pointerList = _pointerList;
    postingIteratorClass = _postingIteratorClass;
    fieldCount = _index.getIntIndexProperty("index."+structureName+".fields.count", currentFile = 0);

View Full Code Here

    //check to see if file id has changed
    if (pointer.getFileNumber() > currentFile)
    {
      //file id changed: close current file, open specified file
      file.close();
      file = new BitInputStream(getFilename(index, structureName, fileCount, currentFile = pointer.getFileNumber()));
      if (DEBUG)
        file = new DebuggingBitIn(file);
    }
    if (file.getByteOffset() != pointer.getOffset())
    {

View Full Code Here

  {
    List<Posting> postings = new ArrayList<Posting>();
    postings.add(new BasicPostingImpl(1,1));
    List<BitIndexPointer> pointerList = new ArrayList<BitIndexPointer>();
    String filename = PostingTestUtils.writePostingsToFile(new Iterator[]{postings.iterator()}, pointerList);
    BitInputStream bitIn = new BitInputStream(filename);
    IterablePosting ip = new BasicIterablePosting(bitIn, postings.size(), null);
    PostingTestUtils.comparePostings(postings, ip);
  }

View Full Code Here

    postings.add(new BasicPostingImpl(2,1));
    postings.add(new BasicPostingImpl(10,1));
    postings.add(new BasicPostingImpl(100,1));
    List<BitIndexPointer> pointerList = new ArrayList<BitIndexPointer>();
    String filename = PostingTestUtils.writePostingsToFile(new Iterator[]{postings.iterator()}, pointerList);
    BitInputStream bitIn = new BitInputStream(filename);
    IterablePosting ip = new BasicIterablePosting(bitIn, postings.size(), null);
    PostingTestUtils.comparePostings(postings, ip);
  }

View Full Code Here

TOP

Related Classes of org.terrier.compression.BitInputStream

org.terrier.structures.BitPostingIndexInputStream

org.terrier.structures.indexing.singlepass.FileRunIterator

org.terrier.structures.indexing.singlepass.hadoop.HadoopRunPostingIterator

org.terrier.structures.indexing.singlepass.Inverted2DirectIndexBuilder

org.terrier.structures.TestPostingStructures

java.io.DataInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.