Package it.unimi.dsi.util

Examples of it.unimi.dsi.util.Properties

71
72
73
74
75
76
77
78
79
80
81
82
83
    final double lowGlobFreq = globalFrequency * ( 1 - error / 100.0 );
    final double highGlobFreq = globalFrequency * ( 1 + error / 100.0 );
    final int lowQbl= (int)Math.round(quantumBitLength * ( 1 - error / 100.0 ));
    final int highQbl = (int)Math.round( quantumBitLength* ( 1 + error / 100.0 ) );

    final Properties properties = new Properties( basename + DiskBasedIndex.PROPERTIES_EXTENSION );
    final int numberOfTerms = properties.getInt( Index.PropertyKeys.TERMS );
    final long numberOfoccurrences = properties.getLong( Index.PropertyKeys.OCCURRENCES );
   
    final InputBitStream globCounts = new InputBitStream( basename + DiskBasedIndex.GLOBCOUNTS_EXTENSION );
    long gc[] = new long[ numberOfTerms ];
    for( int t = 0; t < numberOfTerms; t++ ) gc[ t ] = globCounts.readLongGamma();
    globCounts.close();
View Full Code Here
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
  public long writtenBits() {
    return bitsForFrequencies + bitsForPointers + bitsForPayloads + bitsForCounts + bitsForPositions;
  }

  public Properties properties() {
    Properties result = new Properties();
    result.setProperty( Index.PropertyKeys.DOCUMENTS, numberOfDocuments );
    result.setProperty( Index.PropertyKeys.TERMS, currentTerm + 1 );
    result.setProperty( Index.PropertyKeys.POSTINGS, numberOfPostings );
    result.setProperty( Index.PropertyKeys.MAXCOUNT, maxCount );
    result.setProperty( Index.PropertyKeys.INDEXCLASS, FileIndex.class.getName() );
    // We save all flags, except for the PAYLOAD component, which is just used internally.
    for( Map.Entry<Component,Coding> e: flags.entrySet() )
      if ( e.getKey() != Component.PAYLOADS ) result.addProperty( Index.PropertyKeys.CODING, new MutableString().append( e.getKey() ).append( ':' ).append( e.getValue() ) );
    return result;
  }
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
   * @param property an array of strings specifying properties.
   * @return a metadata map.
   */
  public Reference2ObjectMap<Enum<?>,Object> parseProperties( final String[] property ) throws ConfigurationException {
    final Reference2ObjectArrayMap<Enum<?>,Object> metadata = new Reference2ObjectArrayMap<Enum<?>,Object>();
    Properties properties;
    int pos;
    for( int i = 0; i < property.length; i++ ) {
      if ( ( pos = property[ i ].indexOf( '=' ) ) != - 1 ) {
        properties = new Properties();
        properties.addProperty( property[ i ].substring( 0, pos ), property[ i ].substring( pos + 1 ) );
      }
      else properties =  new Properties( property[ i ] );
       
      metadata.putAll( parseProperties( properties ) );;
    }
   
    return metadata;
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
180
181
      File f1 = new File( basename1 + ext );
      assertEquals( ext, f0.exists(), f1.exists() );
      if ( ext != SIZES_EXTENSION && f0.exists() ) assertTrue( ext, IOUtils.contentEquals( new FileInputStream( f0 ), new FileInputStream( f1 ) ) );
    }

    Properties properties0 = new Properties( basename0 + PROPERTIES_EXTENSION );
    Properties properties1 = new Properties( basename1 + PROPERTIES_EXTENSION );
    for ( String p : excludedProperty ) {
      properties0.setProperty( p, null );
      properties1.setProperty( p, null );
    }

    assertEquals( properties0, properties1 );
  }
View Full Code Here
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
        interleaved, quantum != 0, Math.abs( quantum ), height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    new PartitionDocumentally( basename + "-virtual", basename + "-virtual-part", DocumentalStrategies.uniform( 3, NUMBER_OF_DOCUMENTS ), basename + "-strategy", 0, 1024, flags,
        interleaved, quantum != 0, Math.abs( quantum ), height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();

    // For the text part, we need term maps to call sameIndex()
    String[] localIndex = new Properties( basename + "-text-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    for ( String index : localIndex ) BinIO.storeObject( createMap(index + TERMS_EXTENSION ), index + TERMMAP_EXTENSION );

    sameContent( basename + "-text", basename + "-text-part", new FileLinesCollection( basename + "-text" + TERMS_EXTENSION, "UTF-8" ).iterator() );

    sameContent( basename + "-int", basename + "-int-part" );
    sameContent( basename + "-date", basename + "-date-part" );

    localIndex = new Properties( basename + "-virtual-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    for ( String index : localIndex )
      BinIO.storeObject( createMap( index + TERMS_EXTENSION ), index + TERMMAP_EXTENSION );

    sameContent( basename + "-virtual", basename + "-virtual-part", new FileLinesCollection( basename + "-virtual" + TERMS_EXTENSION, "UTF-8" ).iterator() );

    localIndex = new Properties( basename + "-text-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Concatenate( basename + "-text-merged", localIndex, false, 1024, flags, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( quantum >= 0 ) sameIndex( basename + "-text", basename + "-text-merged", "batches", flags.containsKey( Component.COUNTS ) ? "" : "occurrences" );
    sameContent( basename + "-text", basename + "-text-merged" );

    localIndex = new Properties( basename + "-int-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Concatenate( basename + "-int-merged", localIndex, false, 1024, DEFAULT_PAYLOAD_INDEX, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( quantum >= 0 ) sameIndex( basename + "-text", basename + "-text-merged", "batches", flags.containsKey( Component.COUNTS ) ? "" : "occurrences" );
    sameContent( basename + "-text", basename + "-text-merged" );

    localIndex = new Properties( basename + "-date-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Concatenate( basename + "-date-merged", localIndex, false, 1024, DEFAULT_PAYLOAD_INDEX, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( quantum >= 0 ) sameIndex( basename + "-text", basename + "-text-merged", "batches", flags.containsKey( Component.COUNTS ) ? "" : "occurrences" );
    sameContent( basename + "-text", basename + "-text-merged" );

    localIndex = new Properties( basename + "-virtual-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Concatenate( basename + "-virtual-merged", localIndex, false, 1024, flags, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( quantum >= 0 ) sameIndex( basename + "-text", basename + "-text-merged", "batches", flags.containsKey( Component.COUNTS ) ? "" : "occurrences" );
    sameContent( basename + "-text", basename + "-text-merged" );
  }
View Full Code Here
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
    new PartitionDocumentally( basename + "-date", basename + "-date-part", modulo3, basename + "-strategy", 0, 1024, DEFAULT_PAYLOAD_INDEX, interleaved, quantum != 0, Math.abs( quantum ), height,
        1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    new PartitionDocumentally( basename + "-virtual", basename + "-virtual-part", modulo3, basename + "-strategy", 0, 1024, flags, interleaved, quantum != 0, Math.abs( quantum ), height,
        1024 * 1024, DEFAULT_LOG_INTERVAL ).run();

    String[] localIndex = new Properties( basename + "-text-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    for ( String index : localIndex )
      BinIO.storeObject( createMap( index + TERMS_EXTENSION ), index + TERMMAP_EXTENSION );
    sameContent( basename + "-text", basename + "-text-part", new FileLinesCollection( basename + "-text" + TERMS_EXTENSION, "UTF-8" ).iterator() );

    sameContent( basename + "-int", basename + "-int-part" );
    sameContent( basename + "-date", basename + "-date-part" );

    localIndex = new Properties( basename + "-virtual-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    for ( String index : localIndex )
      BinIO.storeObject( createMap( index + TERMS_EXTENSION ), index + TERMMAP_EXTENSION );
    sameContent( basename + "-virtual", basename + "-virtual-part", new FileLinesCollection( basename + "-virtual" + TERMS_EXTENSION, "UTF-8" ).iterator() );

    localIndex = new Properties( basename + "-text-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );

    new Merge( basename + "-text-merged", localIndex, false, 1024, flags, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( ! interleaved && quantum >= 0 ) sameIndex( basename + "-text", basename + "-text-merged", "batches" );
    else sameContent( basename + "-text", basename + "-text-merged" );
    localIndex = new Properties( basename + "-int-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Merge( basename + "-int-merged", localIndex, false, 1024, DEFAULT_PAYLOAD_INDEX, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( ! interleaved && quantum >= 0 ) sameIndex( basename + "-int", basename + "-int-merged", "batches" );
    else sameContent( basename + "-int", basename + "-int-merged" );
    localIndex = new Properties( basename + "-date-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Merge( basename + "-date-merged", localIndex, false, 1024, DEFAULT_PAYLOAD_INDEX, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( ! interleaved && quantum >= 0 ) sameIndex( basename + "-date", basename + "-date-merged", "batches" );
    else sameContent( basename + "-date", basename + "-date-merged" );
    localIndex = new Properties( basename + "-virtual-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    new Merge( basename + "-virtual-merged", localIndex, false, 1024, flags, interleaved, quantum != 0, quantum, height, 1024 * 1024, DEFAULT_LOG_INTERVAL ).run();
    if ( ! interleaved && quantum >= 0 ) sameIndex( basename + "-virtual", basename + "-virtual-merged", "batches" );
    else sameContent( basename + "-virtual", basename + "-virtual-merged" );
  }
View Full Code Here
889
890
891
892
893
894
895
896
897
898
899
    BinIO.storeObject( uniform, basename + "-strategy" );

    new PartitionLexically( basename + "-text", basename + "-text-part", uniform, basename + "-strategy", 1024, DEFAULT_LOG_INTERVAL ).run();
    new PartitionLexically( basename + "-virtual", basename + "-virtual-part", uniform, basename + "-strategy", 1024, DEFAULT_LOG_INTERVAL ).run();

    String[] localIndex = new Properties( basename + "-text-part" + PROPERTIES_EXTENSION ).getStringArray( IndexCluster.PropertyKeys.LOCALINDEX );
    for ( String index : localIndex )
      BinIO.storeObject( createMap( index + TERMS_EXTENSION ), index + TERMMAP_EXTENSION );
    sameContent( basename + "-text", basename + "-text-part", new FileLinesCollection( basename + "-text" + TERMS_EXTENSION, "UTF-8" ).iterator() );
    sameContent( basename + "-virtual", basename + "-virtual-part" );
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
        String docType = conf.format;
        File metadataFile = new File(output.getAbsolutePath() + ".metadata");

        switch (docType) {
            case "trec":
                Properties properties = new Properties();
                properties.setProperty(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8");
                final TRECDocumentFactory documentFactory = new TRECDocumentFactory(properties);

                collection = new TRECDocumentCollection(files,
                            documentFactory, SegmentedDocumentCollection.DEFAULT_BUFFER_SIZE, compression, metadataFile);
                break;
View Full Code Here
TOP

Related Classes of it.unimi.dsi.util.Properties

  • it.unimi.dsi.mg4j.document.DispatchingDocumentFactory
  • it.unimi.dsi.mg4j.document.PropertyBasedDocumentFactory
  • it.unimi.dsi.mg4j.index.BitStreamHPIndexWriter
  • it.unimi.dsi.mg4j.index.BitStreamIndexWriter
  • it.unimi.dsi.mg4j.index.cluster.ContiguousDocumentalStrategy
  • it.unimi.dsi.mg4j.index.cluster.ContiguousLexicalStrategy
  • it.unimi.dsi.mg4j.index.cluster.IndexCluster
  • it.unimi.dsi.mg4j.index.DiskBasedIndex
  • it.unimi.dsi.mg4j.index.Index
  • it.unimi.dsi.mg4j.index.SkipBitStreamIndexWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.