Package it.unimi.dsi.fastutil.ints

Examples of it.unimi.dsi.fastutil.ints.IntArrayList

1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
      this.field = field;
      this.indexingType = indexingType;
      if ( indexingType != IndexingType.STANDARD && indexingType != IndexingType.REMAPPED ) throw new UnsupportedOperationException( "Non-standard payload-based indices support only standard and remapped indexing" );
      if ( indexingType == IndexingType.REMAPPED ) position = new long[ documentsPerBatch ];
      this.batchDir = batchDir;
      this.cutPoints = new IntArrayList();
      this.cutPoints.add( 0 );

      flags = new EnumMap<Component, Coding>( CompressionFlags.DEFAULT_PAYLOAD_INDEX );
      accumulatorStream = new FastByteArrayOutputStream();
      accumulator = new OutputBitStream( accumulatorStream );
View Full Code Here
535
536
537
538
539
540
541
542
543
544
545
    this.termProcessor = termProcessor;
    this.bufferSize = bufferSize;
    this.builder = builder;
    this.batchDir = batchDir;
    this.virtualDocumentGap = virtualDocumentGap;
    this.cutPoints = new IntArrayList();
    this.cutPoints.add( 0 );

    termMap = new Object2ReferenceOpenHashMap<MutableString, ByteArrayPostingList>( INITIAL_TERM_MAP_SIZE );

    flags = new EnumMap<Component, Coding>( CompressionFlags.DEFAULT_STANDARD_INDEX );
View Full Code Here
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527

  public static int[] parseQualifiedSizes( final String[] qualifiedSizes, final String defaultSize, final int[] indexedField, final DocumentFactory factory ) throws ParseException {
    final int[] size = new int[ indexedField.length ];
    String defaultSpec = defaultSize;
    IntArrayList indexedFields = IntArrayList.wrap( indexedField );
    for ( int i = 0; i < qualifiedSizes.length; i++ )
      if ( qualifiedSizes[ i ].indexOf( ':' ) == -1 ) defaultSpec = qualifiedSizes[ i ];
    for ( int i = 0; i < size.length; i++ )
      size[ i ] = (int)LongSizeStringParser.parseSize( defaultSpec );
    for ( int i = 0; i < qualifiedSizes.length; i++ ) {
      final int split = qualifiedSizes[ i ].indexOf( ':' );
      if ( split >= 0 ) {
        final String fieldName = qualifiedSizes[ i ].substring( 0, split );
        final int field = factory.fieldIndex( fieldName );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + fieldName + " is not part of factory " + factory.getClass().getName() );
        if ( !indexedFields.contains( field ) ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not being indexed" );
        size[ indexedFields.indexOf( field ) ] = (int)LongSizeStringParser.parseSize( qualifiedSizes[ i ].substring( split + 1 ) );
      }
    }
    return size;
  }
View Full Code Here
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
  }

  public static VirtualDocumentResolver[] parseVirtualDocumentResolver( final String[] virtualDocumentSpec, final int[] indexedField, final DocumentFactory factory ) {
    final VirtualDocumentResolver[] virtualDocumentResolver = new VirtualDocumentResolver[ indexedField.length ];
    VirtualDocumentResolver defaultResolver = null;
    IntArrayList indexedFields = IntArrayList.wrap( indexedField );
    for ( int i = 0; i < virtualDocumentSpec.length; i++ )
      if ( virtualDocumentSpec[ i ].indexOf( ':' ) == -1 ) try {
        defaultResolver = (VirtualDocumentResolver)BinIO.loadObject( virtualDocumentSpec[ i ] );
      }
      catch ( IOException e ) {
        throw new RuntimeException( "An I/O error occurred while loading " + virtualDocumentSpec[ i ], e );
      }
      catch ( ClassNotFoundException e ) {
        throw new RuntimeException( "Cannot load " + virtualDocumentSpec[ i ], e );
      }
    for ( int i = 0; i < virtualDocumentResolver.length; i++ )
      virtualDocumentResolver[ i ] = defaultResolver;
    for ( int i = 0; i < virtualDocumentSpec.length; i++ ) {
      final int split = virtualDocumentSpec[ i ].indexOf( ':' );
      if ( split >= 0 ) {
        final String fieldName = virtualDocumentSpec[ i ].substring( 0, split );
        final int field = factory.fieldIndex( fieldName );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + fieldName + " is not part of factory " + factory.getClass().getName() );
        if ( !indexedFields.contains( field ) ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not being indexed" );
        if ( factory.fieldType( field ) != DocumentFactory.FieldType.VIRTUAL ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not virtual" );
        try {
          virtualDocumentResolver[ indexedFields.indexOf( field ) ] = (VirtualDocumentResolver)BinIO.loadObject( virtualDocumentSpec[ i ].substring( split + 1 ) );
        }
        catch ( IOException e ) {
          throw new RuntimeException( "An I/O error occurred while loading " + virtualDocumentSpec[ i ].substring( split + 1 ), e );
        }
        catch ( ClassNotFoundException e ) {
View Full Code Here
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
  }

  public static int[] parseVirtualDocumentGap( final String[] virtualDocumentGapSpec, final int[] indexedField, final DocumentFactory factory ) {
    final int[] virtualDocumentGap = new int[ indexedField.length ];
    int defaultGap = DEFAULT_VIRTUAL_DOCUMENT_GAP;
    IntArrayList indexedFields = IntArrayList.wrap( indexedField );
    for ( int i = 0; i < virtualDocumentGapSpec.length; i++ )
      if ( virtualDocumentGapSpec[ i ].indexOf( ':' ) == -1 ) try {
        defaultGap = Integer.parseInt( virtualDocumentGapSpec[ i ] );
        if ( defaultGap < 0 ) throw new NumberFormatException( "Gap can't be negative" );
      }
      catch ( NumberFormatException e ) {
        throw new RuntimeException( "Cannot parse gap correctly " + virtualDocumentGapSpec[ i ], e );
      }
    for ( int i = 0; i < virtualDocumentGap.length; i++ )
      virtualDocumentGap[ i ] = defaultGap;
    for ( int i = 0; i < virtualDocumentGapSpec.length; i++ ) {
      final int split = virtualDocumentGapSpec[ i ].indexOf( ':' );
      if ( split >= 0 ) {
        final String fieldName = virtualDocumentGapSpec[ i ].substring( 0, split );
        final int field = factory.fieldIndex( fieldName );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + fieldName + " is not part of factory " + factory.getClass().getName() );
        if ( !indexedFields.contains( field ) ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not being indexed" );
        if ( factory.fieldType( field ) != DocumentFactory.FieldType.VIRTUAL ) throw new IllegalArgumentException( "Field " + factory.fieldName( field ) + " is not virtual" );
        try {
          virtualDocumentGap[ indexedFields.indexOf( field ) ] = Integer.parseInt( virtualDocumentGapSpec[ i ].substring( split + 1 ) );
          if ( virtualDocumentGap[ indexedFields.indexOf( field ) ] < 0 ) throw new NumberFormatException( "Gap can't be negative" );
        }
        catch ( NumberFormatException e ) {
          throw new RuntimeException( "Cannot parse gap correctly " + virtualDocumentGapSpec[ i ], e );
        }
      }
View Full Code Here
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
    }
    return virtualDocumentGap;
  }

  public static int[] parseFieldNames( final String[] indexedFieldName, final DocumentFactory factory, final boolean allSupported ) {
    final IntArrayList indexedFields = new IntArrayList();

    if ( indexedFieldName.length == 0 ) {
      for ( int i = 0; i < factory.numberOfFields(); i++ ) {
        DocumentFactory.FieldType type = factory.fieldType( i );
        if ( allSupported ) indexedFields.add( i );
        else if ( type != DocumentFactory.FieldType.VIRTUAL ) indexedFields.add( i );
        else LOGGER.warn( "Virtual field " + factory.fieldName( i ) + " is not being indexed; use -a or explicitly add field among the indexed ones" );
      }
    }
    else {
      for ( int i = 0; i < indexedFieldName.length; i++ ) {
        final int field = factory.fieldIndex( indexedFieldName[ i ] );
        if ( field < 0 ) throw new IllegalArgumentException( "Field " + indexedFieldName[ i ] + " is not part of factory " + factory.getClass().getName() );
        indexedFields.add( field );
      }
    }

    int[] indexedField = indexedFields.toIntArray();
    Arrays.sort( indexedField );
    return indexedField;
  }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    termsOutputStream = new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.TERMS_EXTENSION ) ) );
    nonTermsOutputStream = exact ? new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERMS_EXTENSION ) ) ) : null;
    documentOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.DOCUMENT_OFFSETS_EXTENSION );
    termOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.TERM_OFFSETS_EXTENSION );
    nonTermOffsetsObs = exact? new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERM_OFFSETS_EXTENSION ) : null;
    fieldContent = new IntArrayList();

    if ( hasNonText ) nonTextZipDataOutputStream = new DataOutputStream( nonTextZipOutputStream = new ZipOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + ZipDocumentCollection.ZIP_EXTENSION ) ) ) );

    terms.clear();
    terms.trim( Scan.INITIAL_TERM_MAP_SIZE );
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    Statement s = connection.createStatement();
    ResultSet rs = s.executeQuery( buildQuery( null ) );
   
    id2doc = new Int2IntOpenHashMap();
    id2doc.defaultReturnValue( -1 );
    final IntArrayList ids = new IntArrayList();
    int id;
    for( int i = 0; rs.next(); i++ ) {
      id = rs.getInt( 1 );
      ids.add( id );
      id2doc.put( i, id );
    }
   
    doc2id = ids.toIntArray();
    rs.close();
    s.close();
    connection.close();
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
  }

  @Override
  public IndexIterator documents( final CharSequence prefix, final int limit ) throws IOException, TooManyTermsException {
    final ArrayList<DocumentIterator> iterators = new ArrayList<DocumentIterator>( localIndex.length );
    final IntArrayList usedIndices = new IntArrayList();

    IndexIterator documentIterator;
    for ( int i = 0; i < localIndex.length; i++ ) {
      // TODO: check for limit globally
      documentIterator = localIndex[ i ].documents( prefix, limit );
      if ( documentIterator.hasNext() ) {
        iterators.add( documentIterator );
        usedIndices.add( i );
      }
    }
    // TODO: test that this clustered multiterm does work
    final IndexIterator result = concatenated ?
        new DocumentalConcatenatedClusterIndexIterator( (DocumentalClusterIndexReader)getReader(), iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) :
          new DocumentalMergedClusterIndexIterator( (DocumentalClusterIndexReader)getReader(), iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() );
    result.term( prefix );
    return result;
   
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    return indexIterator;
  }

  public IndexIterator documents( final CharSequence term ) throws IOException {
    final ArrayList<IndexIterator> iterators = new ArrayList<IndexIterator>( indexReader.length );
    final IntArrayList usedIndices = new IntArrayList();
    for ( int i = 0; i < indexReader.length; i++ ) {
      if ( index.termFilter == null || index.termFilter[ i ].contains( term ) ) {
        IndexIterator it = indexReader[ i ].documents( term );
        if ( it.hasNext() ) {
          iterators.add( it );
          usedIndices.add( i );
        }
      }
    }

    if ( DEBUG ) LOGGER.debug( "Indices used for " + term + ": " + usedIndices );

    if ( iterators.isEmpty() ) return index.getEmptyIndexIterator( term );
    final IndexIterator indexIterator =
      index.concatenated ?
          new DocumentalConcatenatedClusterIndexIterator( this, iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) :
            new DocumentalMergedClusterIndexIterator( this, iterators.toArray( IndexIterators.EMPTY_ARRAY ), usedIndices.toIntArray() ) ;
         
    indexIterator.term( term );
    return indexIterator;
  }
View Full Code Here
TOP

Related Classes of it.unimi.dsi.fastutil.ints.IntArrayList

  • com.browseengine.bobo.facets.data.FacetDataCache
  • com.browseengine.bobo.facets.data.MultiValueFacetDataCache
  • com.browseengine.bobo.facets.data.MultiValueWithWeightFacetDataCache
  • com.browseengine.bobo.facets.data.TermIntList
  • com.browseengine.bobo.facets.filter.FacetRangeFilter
  • com.browseengine.bobo.facets.impl.CompactMultiValueFacetHandler
  • com.facebook.presto.operator.ChannelHash
  • com.facebook.presto.operator.GroupByHash$ChannelBuilder
  • com.facebook.presto.operator.index.UnloadedIndexKeyRecordSet
  • com.facebook.presto.operator.JoinHash
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.