Package it.unimi.dsi.io

Examples of it.unimi.dsi.io.InputBitStream

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    final String basename = jsapResult.getString( "basename" );

    // Just to check that the index is of the right type
    final BitStreamHPIndex index = (BitStreamHPIndex)Index.getInstance( basename, false, false );
   
    final InputBitStream ibs = new InputBitStream( basename + DiskBasedIndex.INDEX_EXTENSION );
    final InputBitStream offsets = new InputBitStream( basename + DiskBasedIndex.OFFSETS_EXTENSION );
    final OutputBitStream numBitsPos = new OutputBitStream( basename + DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION );
    long pos = 0, positionOffset = 0, o;
    for( int i = 0; i < index.numberOfTerms; i++ ) {
      ibs.position( pos += offsets.readLongGamma() );
      // Read offset into position file
      o = ibs.readLongDelta();

      if ( i > 0 ) numBitsPos.writeLongGamma( o - positionOffset );
      positionOffset = o;
    }

    // This is necessarily imprecise
    numBitsPos.writeLongGamma( new File( basename + DiskBasedIndex.POSITIONS_EXTENSION ).length() * 8 - positionOffset );

    numBitsPos.close();
    offsets.close();
    ibs.close(); 
  }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
      occ[ i ] = index[ i ].maxCount > 0 ? new int[ index[ i ].maxCount ] : IntArrays.EMPTY_ARRAY;
      wordInPos[ i ] = new int[ Math.max( 0, index[ i ].properties.getInt( Index.PropertyKeys.MAXDOCSIZE ) ) ];
      indexReader[ i ] = index[ i ].getReader();
     
      if ( new File( basenameField + DiskBasedIndex.FREQUENCIES_EXTENSION ).exists() ) frequencies[ i ] = new InputBitStream( basenameField + DiskBasedIndex.FREQUENCIES_EXTENSION );
      termsInDoc[ i ] = new Int2IntOpenHashMap();
    }


    int currDoc = 0,
View Full Code Here
47
48
49
50
51
52
53
54
    }
    int size = (int)( streamer.writtenBits() / 8 ) + ( ( streamer.writtenBits() % 8 ) == 0 ? 0 : 1 );
    byte[] smaller = new byte[ size ];
    System.arraycopy( array, 0, smaller, 0, size );

    return new InputBitStream( smaller );

  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    assertEquals( 2, index.documents( 0 ).frequency() );     
    assertEquals( 2, index.documents( 1 ).frequency() );
    assertEquals( 1, index.documents( 2 ).frequency() );
    assertEquals( 1, index.documents( 3 ).frequency() );
   
    SemiExternalGammaList frequencies = new SemiExternalGammaList( new InputBitStream( basename + "-mo" + DiskBasedIndex.FREQUENCIES_EXTENSION ), 1, 4 );

    assertEquals( 2, frequencies.getLong( 0 ) );     
    assertEquals( 2, frequencies.getLong( 1 ) );     
    assertEquals( 1, frequencies.getLong( 2 ) );
    assertEquals( 1, frequencies.getLong( 3 ) );
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
    assertScoresMultiIndex( "u:X & (u:A | t:C)", indexMap, indexBody, scorer, new double[] { .3053 * .75 + .3053 * .75, .4021 * .75 + .3053 *.25 } );
  }
 
 
  public void testBM25FScorer() throws FileNotFoundException, IOException, QueryParserException, QueryBuilderVisitorException {
    Scorer scorer = new BM25FScorer( .95, new Reference2DoubleOpenHashMap<Index>( new Index[] { indexFBody, indexFTitle }, new double[] { 0.5, 0.3 } ), immutableExternalPrefixMap, new SemiExternalGammaList( new InputBitStream( basenameFComb + ".frequencies" ) ) );
    scorer.setWeights( new Reference2DoubleOpenHashMap<Index>( new Index[] { indexFTitle, indexFBody }, new double[] { .85, .15 } ) );
    // b(ody):.3 t(itle):1.7
    Object2ReferenceMap<String,Index> indexMap = new Object2ReferenceOpenHashMap<String,Index>( new String[] { "b", "t" }, new Index[] { indexFBody, indexFTitle } );
   
    assertScoresMultiIndex( "b:A | t:A", indexMap, indexFBody, scorer, new double[] { 0.953, 0.202 } );
View Full Code Here
18
19
20
21
22
23
24
25
26
    final FastByteArrayOutputStream fbos = new FastByteArrayOutputStream();
    final OutputBitStream obs = new OutputBitStream( fbos );
    Object o = payload.get();
    payload.write( obs );
    obs.flush();
    final InputBitStream ibs = new InputBitStream( fbos.array );
    payload.read( ibs );
    assertEquals( o, payload.get() );
  }
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
187
188
189
          basename, field, DiskBasedIndex.COUNTS_EXTENSION));
      LOGGER.info("Loading term frequencies from file " +
          frequenciesFile);
      list = new LongBigArrayBigList(index.numberOfTerms);

      final InputBitStream in = new InputBitStream(frequenciesFile);
      for (long i = 0; i < list.size64(); i++)
        list.set(i, in.readLongGamma());
      in.close();
      termfrequencies = new SoftReference<>(list);
     
      LOGGER.info("Completed.");
    }
    return list;
View Full Code Here
TOP

Related Classes of it.unimi.dsi.io.InputBitStream

  • it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollection
  • it.unimi.dsi.mg4j.index.DiskBasedIndex
  • it.unimi.dsi.mg4j.index.DiskBasedIndex$GammaCodedIterableList
  • it.unimi.dsi.mg4j.index.FileHPIndex
  • it.unimi.dsi.mg4j.index.FileIndex
  • it.unimi.dsi.mg4j.index.InMemoryHPIndex
  • it.unimi.dsi.mg4j.index.InMemoryIndex
  • it.unimi.dsi.mg4j.index.MemoryMappedHPIndex
  • it.unimi.dsi.mg4j.index.MemoryMappedIndex
  • it.unimi.dsi.mg4j.index.payload.PayloadTestCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.