Package org.terrier.structures

Examples of org.terrier.structures.BitIndexPointer


   
      if (! srcIndex1.getIndexProperty("index.meta.key-names", "docno").equals(srcIndex2.getIndexProperty("index.meta.key-names", "docno")))
      {
        throw new Error("Meta fields in source indices must match");
      }
      final BitIndexPointer emptyPointer = new SimpleBitIndexPointer();
     
       
      final int srcFieldCount1 = srcIndex1.getIntIndexProperty("index.direct.fields.count", 0);
      final int srcFieldCount2 = srcIndex1.getIntIndexProperty("index.direct.fields.count", 0);
      if (srcFieldCount1 != srcFieldCount2)
      {
        throw new Error("FieldCounts in source indices must match");
      }
     
      final int fieldCount = srcFieldCount1;
     
     
      for(String property : new String[] {"index.direct.fields.names","index.direct.fields.count" } )
      {
        destIndex.setIndexProperty(property, srcIndex1.getIndexProperty(property, null));
      }
     
      DirectInvertedOutputStream dfOutput = null;
      try{
        dfOutput =
          (fieldCount > 0 ? fieldDirectFileOutputStreamClass : directFileOutputStreamClass)
          .getConstructor(String.class)
          .newInstance(destIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + 
                destIndex.getPrefix() + ".direct" + BitIn.USUAL_EXTENSION);
      } catch (Exception e) {
        logger.error("Couldn't create specified DirectInvertedOutputStream", e);
        return;
      }
     
     
      final Iterator<DocumentIndexEntry> docidInput1 = (Iterator<DocumentIndexEntry>)srcIndex1.getIndexStructureInputStream("document");
      final PostingIndexInputStream dfInput1 = (PostingIndexInputStream)srcIndex1.getIndexStructureInputStream("direct");
      final MetaIndex metaInput1 = srcIndex1.getMetaIndex();
     
      int sourceDocid = 0;
      //traversing the direct index, without any change
      while(docidInput1.hasNext())
      {
        BitIndexPointer pointerDF = emptyPointer;
        DocumentIndexEntry die = docidInput1.next();
        if (die.getDocumentLength() > 0)
        {
          pointerDF = dfOutput.writePostings(dfInput1.next());
        }
        die.setBitIndexPointer(pointerDF);
        docidOutput.addEntryToBuffer(die);
        metaBuilder.writeDocumentEntry(metaInput1.getAllItems(sourceDocid));
        sourceDocid++;
      }
      dfInput1.close();
      metaInput1.close();
      IndexUtil.close(docidInput1);
      final Iterator<DocumentIndexEntry> docidInput2 = (Iterator<DocumentIndexEntry>)srcIndex2.getIndexStructureInputStream("document");
      final PostingIndexInputStream dfInput2 = (PostingIndexInputStream)srcIndex2.getIndexStructureInputStream("direct");
      final MetaIndex metaInput2 = srcIndex2.getMetaIndex();
     
      sourceDocid = 0;
      while (docidInput2.hasNext())
      {
        DocumentIndexEntry die = docidInput2.next();
     
        BitIndexPointer pointerDF = emptyPointer;
        if (die.getDocumentLength() > 0)
        {
          final IterablePosting postings = dfInput2.next();
         
          List<Posting> postingList = new ArrayList<Posting>();
View Full Code Here


        IterablePosting ip = new ArrayOfIdsIterablePosting(_tmp);
        while(ip.next() != IterablePosting.EOL)
        {
          postingList.add(ip.asWritablePosting());
        }
        BitIndexPointer diosPointer = dios.writePostings(postingList.iterator());
       
        IDS.add(_tmp);
        ids.clear();
        int previous = -1;
        for(int i : _tmp)
        {
          bo.writeGamma(i - previous);
          previous = i;
        }
       
        assertEquals(byteOffset, diosPointer.getOffset());
        assertEquals(bitOffset, diosPointer.getOffsetBits());
        assertEquals(_tmp.length, diosPointer.getNumberOfEntries());
       
        //System.err.println("startoffset="+ bp.toString() + " dios_pointer="+ diosPointer.toString());
       
        docid++;
      }   
View Full Code Here

    Index index = Index.createNewIndex(path, prefix);
    DirectInvertedDocidOnlyOuptutStream dios = new DirectInvertedDocidOnlyOuptutStream(path + '/'+ prefix + ".direct.bf");
    //FSArrayFile<BitIndexPointer>
    DocumentIndexBuilder dib = new DocumentIndexBuilder(index, "document");
    BitIndexPointer p;
    for(int[] list : postings)
    {
      final int doclen = StaTools.sum(list);
      p = dios.writePostings(new ArrayOfIdsIterablePosting(list));
      DocumentIndexEntry die = new BasicDocumentIndexEntry(doclen, p);
View Full Code Here

TOP

Related Classes of org.terrier.structures.BitIndexPointer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.