Examples of it.unimi.dsi.mg4j.tool.IndexBuilder

it.unimi.dsi.mg4j.tool.IndexBuilder
An index builder.
An instance of this class exposes a {@link #run()} methodthat will index the {@link DocumentSequence} provided at construction timeby calling {@link Scan} and {@link Combine} in sequence.
Additionally, a main method provides easy access to index construction.
All indexing parameters are available either as chainable setters that can be called optionally before invoking {@link #run()}, or as public mutable collections and maps. For instance,
```
 new IndexBuilder( "foo", sequence ).skips( true ).run(); 
```
will build an index with basename foo using skips. If instead we want to index just the first field of the sequence, and use a {@link ShiftAddXorSignedStringMap}as a term map, we can use the following code:
```
 new IndexBuilder( "foo", sequence ) .termMapClass( ShiftAddXorSignedMinimalPerfectHash.class ) .indexedFields( 0 ).run(); 
```
More sophisticated modifications can be applied using public maps:
```
 IndexBuilder indexBuilder = new IndexBuilder( "foo", sequence ); indexBuilder.virtualDocumentGaps.put( 0, 30 ); indexBuilder.virtualDocumentResolver.put( 0, someVirtualDocumentResolver ); indexBuilder.run(); 
```

  private String basename;


  @Override
  public void setUp() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a", "b", "c" ) ).run();
    index = DiskBasedIndex.getInstance( basename + "-text", true, true );
  }

View Full Code Here

  private Object2ReferenceMap<String, Index> indexMap;


  @Override
  public void setUp() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename + "0", new StringArrayDocumentCollection( "a b", "b c", "c" ) ).run();
    new IndexBuilder( basename + "1", new StringArrayDocumentCollection( "a b", "b c", "c" ) ).run();
    index0 = DiskBasedIndex.getInstance( basename + "0-text", true, true );
    index1 = DiskBasedIndex.getInstance( basename + "1-text", true, true );
    index2 = DiskBasedIndex.getInstance( basename + "0-text", true, true );
    index3 = DiskBasedIndex.getInstance( basename + "1-text", true, true );
    indexMap = new Object2ReferenceOpenHashMap<String, Index>( new String[] { "index0", "index1", "index2", "index3" }, new Index[] { index0, index1, index2, index3 } );

View Full Code Here

  private BitStreamIndex index;
  private SimpleParser simpleParser;
  public void setUp() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {


    String basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a", "b", "c" ) ).run();
    index = DiskBasedIndex.getInstance( basename + "-text", true, true );
    simpleParser = new SimpleParser( index.termProcessor );
  }

View Full Code Here

  }
  
  // Contributed by Fabien Campagne
  public void testMG4JMultiTermPositionIssue() throws IllegalAccessException, NoSuchMethodException, ConfigurationException, IOException, InvocationTargetException, InstantiationException, ClassNotFoundException, URISyntaxException {
    String basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection(
                "A B C D E F F G G",
                "G A T H S K L J W L",
                "E S K D L J F K L S J D L S J D",
                "E B"
    ) ).run();

View Full Code Here

public class ScanTest extends TestCase {


  public void testEverywhereTerms() throws IOException, ConfigurationException, SecurityException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    
    String basename = File.createTempFile( getClass().getSimpleName(), "everywhereTerms" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a a" ) ).keepBatches( true ).run();
    IndexIterator indexIterator = Index.getInstance( basename + "-text@0" ).documents( 0 );
    indexIterator.next();
    assertEquals( 2, indexIterator.count() );
    int[] position = new int[ 2 ];
    indexIterator.positions( position );

View Full Code Here


  public void testSkipBug() throws QueryParserException, QueryBuilderVisitorException, IOException, ConfigurationException, SecurityException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    BitStreamIndex index;
    SimpleParser simpleParser;
    String basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a a b c d e" ) ).run();
    index = DiskBasedIndex.getInstance( basename + "-text", true, true );
    simpleParser = new SimpleParser( index.termProcessor );


    Query query = simpleParser.parse( "(a|b|d)<(a|b|d)" );
    DocumentIteratorBuilderVisitor documentIteratorBuilderVisitor = new DocumentIteratorBuilderVisitor( null, index, Integer.MAX_VALUE );

View Full Code Here

  private String basename;


  @Override
  public void setUp() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a", "b", "c" ) ).run();
    index = DiskBasedIndex.getInstance( basename + "-text", true, true );
    simpleParser = new SimpleParser( index.termProcessor );
  }

View Full Code Here


public class BitStreamIndexReaderTest extends TestCase {


  public void testSkipToEndOfList() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    String basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( new String[] { "a", "a", "c" } ) ).run();


    Index index = DiskBasedIndex.getInstance( basename + "-text", true, true );
    IndexIterator indexIterator = index.documents( "a" );
    assertEquals( Integer.MAX_VALUE, indexIterator.skipTo( Integer.MAX_VALUE ) );
    indexIterator.dispose();

View Full Code Here

  public void testSkipToBeyondUsedClusters() throws Exception {
    /* We test what happens when we skip to a document belonging to a local index larger 
     * than any index in which the term appears. */
    
    final String basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
        new IndexBuilder( basename, new StringArrayDocumentCollection( "A B", "B", "A", "A" ) ).run();
    BinIO.storeObject( DocumentalStrategies.uniform( 2, 4 ), basename + "-strategy" );
    new PartitionDocumentally( basename + "-text", basename + "-cluster", DocumentalStrategies.uniform( 2, 4 ), basename + "-strategy", 0, 1024, CompressionFlags.DEFAULT_STANDARD_INDEX, true, false, 0, 0, 0, ProgressLogger.DEFAULT_LOG_INTERVAL ).run();
    FileLinesCollection flc;
    flc = new FileLinesCollection( basename + "-cluster-0.terms", "ASCII" );
    BinIO.storeObject( new ShiftAddXorSignedStringMap( flc.iterator(), new MWHCFunction<CharSequence>( flc , TransformationStrategies.utf16() ) ), basename + "-cluster-0.termmap" );

View Full Code Here

  private BitStreamIndex index;
  private String basename;


  public void setUp() throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    basename = File.createTempFile( getClass().getSimpleName(), "test" ).getCanonicalPath();
    new IndexBuilder( basename, new StringArrayDocumentCollection( "a", "b", "c" ) ).run();
    index = DiskBasedIndex.getInstance( basename + "-text", true, true );
  }

View Full Code Here

0 1

TOP

Related Classes of it.unimi.dsi.mg4j.tool.IndexBuilder

com.martiansoftware.jsap.FlaggedOption

com.martiansoftware.jsap.JSAPResult

com.martiansoftware.jsap.SimpleJSAP

com.martiansoftware.jsap.Switch

com.martiansoftware.jsap.UnflaggedOption

it.unimi.dsi.io.FileLinesCollection

it.unimi.dsi.lang.ObjectParser

it.unimi.dsi.mg4j.document.DocumentCollectionBuilder

it.unimi.dsi.mg4j.document.DocumentFactory

it.unimi.dsi.mg4j.document.DocumentSequence

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.