Examples of IntArrayList


Examples of it.unimi.dsi.fastutil.ints.IntArrayList

  }

  public static VirtualDocumentResolver[] parseVirtualDocumentResolver( final String[] virtualDocumentSpec, final int[] indexedField, final DocumentFactory factory ) {
    final VirtualDocumentResolver[] virtualDocumentResolver = new VirtualDocumentResolver[ indexedField.length ];
    VirtualDocumentResolver defaultResolver = null;
    IntArrayList indexedFields = IntArrayList.wrap( indexedField );
    for ( int i = 0; i < virtualDocumentSpec.length; i++ )
      if ( virtualDocumentSpec[ i ].indexOf( ':' ) == -1 ) try {
        defaultResolver = (VirtualDocumentResolver)BinIO.loadObject( virtualDocumentSpec[ i ] );
      }
      catch ( IOException e ) {
        throw new RuntimeException( "An I/O error occurred while loading " + virtualDocumentSpec[ i ], e );
      }
      catch ( ClassNotFoundException e ) {
        throw new RuntimeException( "Cannot load " + virtualDocumentSpec[ i ], e );
      }
    for ( int i = 0; i < virtualDocumentResolver.length; i++ )
      virtualDocumentResolver[ i ] = defaultResolver;
    for ( int i = 0; i < virtualDocumentSpec.length; i++ ) {
      final int split = virtualDocumentSpec[ i ].indexOf( ':' );
      if ( split >= 0 ) {
        final String fieldName = virtualDocumentSpec[ i ].substring( 0, split );
        final int field = factory.fieldIndex( fieldName );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + fieldName + " is not part of factory " + factory.getClass().getName() );
        if ( !indexedFields.contains( field ) ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not being indexed" );
        if ( factory.fieldType( field ) != DocumentFactory.FieldType.VIRTUAL ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not virtual" );
        try {
          virtualDocumentResolver[ indexedFields.indexOf( field ) ] = (VirtualDocumentResolver)BinIO.loadObject( virtualDocumentSpec[ i ].substring( split + 1 ) );
        }
        catch ( IOException e ) {
          throw new RuntimeException( "An I/O error occurred while loading " + virtualDocumentSpec[ i ].substring( split + 1 ), e );
        }
        catch ( ClassNotFoundException e ) {
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

  }

  public static int[] parseVirtualDocumentGap( final String[] virtualDocumentGapSpec, final int[] indexedField, final DocumentFactory factory ) {
    final int[] virtualDocumentGap = new int[ indexedField.length ];
    int defaultGap = DEFAULT_VIRTUAL_DOCUMENT_GAP;
    IntArrayList indexedFields = IntArrayList.wrap( indexedField );
    for ( int i = 0; i < virtualDocumentGapSpec.length; i++ )
      if ( virtualDocumentGapSpec[ i ].indexOf( ':' ) == -1 ) try {
        defaultGap = Integer.parseInt( virtualDocumentGapSpec[ i ] );
        if ( defaultGap < 0 ) throw new NumberFormatException( "Gap can't be negative" );
      }
      catch ( NumberFormatException e ) {
        throw new RuntimeException( "Cannot parse gap correctly " + virtualDocumentGapSpec[ i ], e );
      }
    for ( int i = 0; i < virtualDocumentGap.length; i++ )
      virtualDocumentGap[ i ] = defaultGap;
    for ( int i = 0; i < virtualDocumentGapSpec.length; i++ ) {
      final int split = virtualDocumentGapSpec[ i ].indexOf( ':' );
      if ( split >= 0 ) {
        final String fieldName = virtualDocumentGapSpec[ i ].substring( 0, split );
        final int field = factory.fieldIndex( fieldName );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + fieldName + " is not part of factory " + factory.getClass().getName() );
        if ( !indexedFields.contains( field ) ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not being indexed" );
        if ( factory.fieldType( field ) != DocumentFactory.FieldType.VIRTUAL ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not virtual" );
        try {
          virtualDocumentGap[ indexedFields.indexOf( field ) ] = Integer.parseInt( virtualDocumentGapSpec[ i ].substring( split + 1 ) );
          if ( virtualDocumentGap[ indexedFields.indexOf( field ) ] < 0 ) throw new NumberFormatException( "Gap can't be negative" );
        }
        catch ( NumberFormatException e ) {
          throw new RuntimeException( "Cannot parse gap correctly " + virtualDocumentGapSpec[ i ], e );
        }
      }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

    }
    return virtualDocumentGap;
  }

  public static int[] parseFieldNames( final String[] indexedFieldName, final DocumentFactory factory, final boolean allSupported ) {
    final IntArrayList indexedFields = new IntArrayList();

    if ( indexedFieldName.length == 0 ) {
      for ( int i = 0; i < factory.numberOfFields(); i++ ) {
        DocumentFactory.FieldType type = factory.fieldType( i );
        if ( allSupported ) indexedFields.add( i );
        else if ( type != DocumentFactory.FieldType.VIRTUAL ) indexedFields.add( i );
        else LOGGER.warn( "Virtual field " + factory.fieldName( i ) + " is not being indexed; use -a or explicitly add field among the indexed ones" );
      }
    }
    else {
      for ( int i = 0; i < indexedFieldName.length; i++ ) {
        final int field = factory.fieldIndex( indexedFieldName[ i ] );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + indexedFieldName[ i ] + " is not part of factory " + factory.getClass().getName() );
        indexedFields.add( field );
      }
    }

    int[] indexedField = indexedFields.toIntArray();
    Arrays.sort( indexedField );
    return indexedField;
  }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

    termsOutputStream = new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.TERMS_EXTENSION ) ) );
    nonTermsOutputStream = exact ? new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERMS_EXTENSION ) ) ) : null;
    documentOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.DOCUMENT_OFFSETS_EXTENSION );
    termOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.TERM_OFFSETS_EXTENSION );
    nonTermOffsetsObs = exact? new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERM_OFFSETS_EXTENSION ) : null;
    fieldContent = new IntArrayList();

    if ( hasNonText ) nonTextZipDataOutputStream = new DataOutputStream( nonTextZipOutputStream = new ZipOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + ZipDocumentCollection.ZIP_EXTENSION ) ) ) );

    terms.clear();
    terms.trim( Scan.INITIAL_TERM_MAP_SIZE );
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

    Statement s = connection.createStatement();
    ResultSet rs = s.executeQuery( buildQuery( null ) );
   
    id2doc = new Int2IntOpenHashMap();
    id2doc.defaultReturnValue( -1 );
    final IntArrayList ids = new IntArrayList();
    int id;
    for( int i = 0; rs.next(); i++ ) {
      id = rs.getInt( 1 );
      ids.add( id );
      id2doc.put( i, id );
    }
   
    doc2id = ids.toIntArray();
    rs.close();
    s.close();
    connection.close();
  }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

  }

  @Override
  public IndexIterator documents( final CharSequence prefix, final int limit ) throws IOException, TooManyTermsException {
    final ArrayList<DocumentIterator> iterators = new ArrayList<DocumentIterator>( localIndex.length );
    final IntArrayList usedIndices = new IntArrayList();

    IndexIterator documentIterator;
    for ( int i = 0; i < localIndex.length; i++ ) {
      // TODO: check for limit globally
      documentIterator = localIndex[ i ].documents( prefix, limit );
      if ( documentIterator.hasNext() ) {
        iterators.add( documentIterator );
        usedIndices.add( i );
      }
    }
    // TODO: test that this clustered multiterm does work
    final IndexIterator result = concatenated ?
        new DocumentalConcatenatedClusterIndexIterator( (DocumentalClusterIndexReader)getReader(), iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) :
          new DocumentalMergedClusterIndexIterator( (DocumentalClusterIndexReader)getReader(), iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() );
    result.term( prefix );
    return result;
   
  }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

    return indexIterator;
  }

  public IndexIterator documents( final CharSequence term ) throws IOException {
    final ArrayList<IndexIterator> iterators = new ArrayList<IndexIterator>( indexReader.length );
    final IntArrayList usedIndices = new IntArrayList();
    for ( int i = 0; i < indexReader.length; i++ ) {
      if ( index.termFilter == null || index.termFilter[ i ].contains( term ) ) {
        IndexIterator it = indexReader[ i ].documents( term );
        if ( it.hasNext() ) {
          iterators.add( it );
          usedIndices.add( i );
        }
      }
    }

    if ( DEBUG ) LOGGER.debug( "Indices used for " + term + ": " + usedIndices );

    if ( iterators.isEmpty() ) return index.getEmptyIndexIterator( term );
    final IndexIterator indexIterator =
      index.concatenated ?
          new DocumentalConcatenatedClusterIndexIterator( this, iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) :
            new DocumentalMergedClusterIndexIterator( this, iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) ;
         
    indexIterator.term( term );
    return indexIterator;
  }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

        System.err.println( "Term frequency was " + f + " and position " + ( f - indexFrequency - 1 ) );
        throw e;
      }
    }
 
    IntArrayList l = new IntArrayList();
    ObjectArrayList<int[]> positions = new ObjectArrayList<int[]>();
   
    if ( ! jsapResult.getBoolean( "noSkip" ) ) {
      int start = 0, end = 0, result;
      try {
        for (i = 0; i < index.length; i++) {
         
          pl.expectedUpdates = numberOfTerms[ i ];
          pl.start("Verifying all skips in " + index[i] + "...");

          for (t = 0; t < numberOfTerms[ i ]; t++) {
            l.clear();
            positions.clear();
            IndexIterator documents = termLists ? indexReader[ i ].documents( terms[ i ].get( t ) ) : indexReader[ i ].documents( t );
            int d;
            while( ( d = documents.nextDocument() ) != -1 ) {
              l.add( d );
              if ( index[ i ].hasPositions ) positions.add( ArrayUtils.subarray( documents.positionArray(), 0, documents.count() ) );
            }
           
            for( start = 0; start < l.size(); start++ ) {
              for( end = start + 1; end < l.size(); end++ ) {
                IndexIterator indexIterator = termLists ? indexReader[ i ].documents( terms[ i ].get( t ) ) : indexReader[ i ].documents( t );
               
                result = indexIterator.skipTo( l.getInt( start ) );
                if ( indexIterator.document() != l.getInt( start ) || result != l.getInt( start ) ) throw new AssertionError( "Trying to skip to document " + l.getInt( start ) + " (term " + t + ") moved to " + indexIterator.document() + "(skipTo() returned " + result + ")" );
                result = indexIterator.skipTo( l.getInt( end ) );
                if ( indexIterator.document() != l.getInt( end ) || result != l.getInt( end ) ) throw new AssertionError( "Trying to skip to document " + l.getInt( end ) + " (term " + t + ") after a skip to " + start + " moved to " + indexIterator.document() + "(skipTo() returned " + result + ")" );
               
                if ( index[ i ].hasPositions ) {
                  // This catches wrong state reconstruction after skips.
                  indexIterator = termLists ? indexReader[ i ].documents( terms[ i ].get( t ) ) : indexReader[ i ].documents( t );
                  indexIterator.skipTo( l.getInt( start ) );
                  if ( indexIterator.document() != l.getInt( start ) ) throw new AssertionError(indexIterator.document() + " != " + l.getInt( start ) );
                  if ( indexIterator.count() != positions.get( start ).length ) throw new AssertionError(indexIterator.count() + " != " + positions.get( start ).length );
                  if ( ! Arrays.equals( positions.get( start ), ArrayUtils.subarray( indexIterator.positionArray(), 0, indexIterator.count() ) )
                     ) throw new AssertionError(Arrays.toString( positions.get( start ) ) + "!=" + Arrays.toString( ArrayUtils.subarray( indexIterator.positionArray(), 0, indexIterator.count() ) ) );
                  indexIterator.skipTo( l.getInt( end ) );
                  if ( indexIterator.document() != l.getInt( end )  ) throw new AssertionError(indexIterator.document() + " != " + l.getInt( end ) );
                  if ( indexIterator.count() != positions.get( end ).length ) throw new AssertionError(indexIterator.count() + " != " + positions.get( end ).length );
                  if ( ! Arrays.equals( positions.get( end ), ArrayUtils.subarray( indexIterator.positionArray(), 0, indexIterator.count() ) )
                     ) throw new AssertionError(Arrays.toString( positions.get( end ) ) + "!=" + Arrays.toString( ArrayUtils.subarray( indexIterator.positionArray(), 0, indexIterator.count() ) ) );
                }
               
              }
             
              IndexIterator indexIterator = termLists ? indexReader[ i ].documents( terms[ i ].get( t ) ) : indexReader[ i ].documents( t );
             
              result = indexIterator.skipTo( l.getInt( start ) );
              if ( indexIterator.document() != l.getInt( start ) || result != l.getInt( start ) ) throw new AssertionError("Trying to skip to document " + l.getInt( start ) + " (term " + t + ") moved to " + indexIterator.document() + "(skipTo() returned " + result + ")" );
              result = indexIterator.skipTo( Integer.MAX_VALUE );
              if ( indexIterator.hasNext() || result != Integer.MAX_VALUE ) throw new AssertionError("Trying to skip beyond end of list (term " + t + ") after a skip to " + start + " returned " + result + " (hasNext()=" + indexIterator.hasNext() + ")" );
             
             
            }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

      if ( word.length() == 0 ) continue;
      if ( !termProcessor.processTerm( word ) ) {
        pos++;
        continue;
      }
      IntArrayList positions = terms.get( word );
      if ( positions == null ) terms.put( word.copy(), positions = new IntArrayList() );
      positions.add( pos++ );
    }

    for ( MutableString term : terms.keySet() ) {
      ObjectArrayList<int[]> list = termMap.get( term );
      IntArrayList positions = terms.get( term );
      if ( list == null ) termMap.put( term, list = new ObjectArrayList<int[]>() );

      int[] t = new int[ positions.size() + 1 ];
      t[ 0 ] = documentIndex;
      System.arraycopy( positions.elements(), 0, t, 1, positions.size() );
      list.add( t );
    }

    return pos;
  }
View Full Code Here

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

      case TEXT:
        termMap[ i ] = new Object2ObjectOpenHashMap<MutableString, ObjectArrayList<int[]>>();
        break;
      case DATE:
      case INT:
        payloadPointers[ i ] = new IntArrayList();
        payloadContent[ i ] = new ObjectArrayList<Object>();
      }
    }

    int documentIndex = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.