Examples of org.apache.lucene.index.SortedSetDocValues

org.apache.lucene.index.SortedSetDocValues
A per-document set of presorted byte[] values.
Per-Document values in a SortedDocValues are deduplicated, dereferenced, and sorted into a dictionary of unique values. A pointer to the dictionary value (ordinal) can be retrieved for each document. Ordinals are dense and in increasing sorted order.

    final Arc<Long> scratchArc = new Arc<Long>();
    final IntsRef scratchInts = new IntsRef();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst); 
    final BytesRef ref = new BytesRef();
    final ByteArrayDataInput input = new ByteArrayDataInput();
    return new SortedSetDocValues() {
      long currentOrd;


      @Override
      public long nextOrd() {
        if (input.eof()) {

View Full Code Here

    }
  }
  
  @Override
  public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
    SortedSetDocValues sortedSetDV = in.getSortedSetDocValues(field);
    if (sortedSetDV == null) {
      return null;
    } else {
      return new SortingSortedSetDocValues(sortedSetDV, docMap);
    }

View Full Code Here

     * Returns a DocIdSet with documents that should be permitted in search
     * results.
     */
    @Override
    public DocIdSet getDocIdSet(AtomicReaderContext context, final Bits acceptDocs) throws IOException {
      final SortedSetDocValues docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), query.field);
      // Cannot use FixedBitSet because we require long index (ord):
      final OpenBitSet termSet = new OpenBitSet(docTermOrds.getValueCount());
      TermsEnum termsEnum = query.getTermsEnum(new Terms() {
        
        @Override
        public Comparator<BytesRef> getComparator() {
          return BytesRef.getUTF8SortedAsUnicodeComparator();
        }
        
        @Override
        public TermsEnum iterator(TermsEnum reuse) {
          return docTermOrds.termsEnum();
        }


        @Override
        public long getSumTotalTermFreq() {
          return -1;
        }


        @Override
        public long getSumDocFreq() {
          return -1;
        }


        @Override
        public int getDocCount() {
          return -1;
        }


        @Override
        public long size() {
          return -1;
        }


        @Override
        public boolean hasOffsets() {
          return false;
        }


        @Override
        public boolean hasPositions() {
          return false;
        }
        
        @Override
        public boolean hasPayloads() {
          return false;
        }
      });
      
      assert termsEnum != null;
      if (termsEnum.next() != null) {
        // fill into a OpenBitSet
        do {
          termSet.set(termsEnum.ord());
        } while (termsEnum.next() != null);
      } else {
        return null;
      }
      
      return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
        @Override
        protected final boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException {
          docTermOrds.setDocument(doc);
          long ord;
          // TODO: we could track max bit set and early terminate (since they come in sorted order)
          while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
            if (termSet.get(ord)) {
              return true;
            }
          }
          return false;

View Full Code Here

    final Arc<Long> scratchArc = new Arc<Long>();
    final IntsRef scratchInts = new IntsRef();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst); 
    final BytesRef ref = new BytesRef();
    final ByteArrayDataInput input = new ByteArrayDataInput();
    return new SortedSetDocValues() {
      long currentOrd;


      @Override
      public long nextOrd() {
        if (input.eof()) {

View Full Code Here

  }


  // TODO: this if DocTermsIndex was already created, we
  // should share it...
  public SortedSetDocValues getDocTermOrds(AtomicReader reader, String field) throws IOException {
    SortedSetDocValues dv = reader.getSortedSetDocValues(field);
    if (dv != null) {
      return dv;
    }
    
    SortedDocValues sdv = reader.getSortedDocValues(field);

View Full Code Here

    final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
    final LongNumericDocValues ordinals = getNumeric(ords.get(field.number));
    // but the addresses to the ord stream are in RAM
    final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number));
    
    return new SortedSetDocValues() {
      long offset;
      long endOffset;
      
      @Override
      public long nextOrd() {

View Full Code Here


    // test bad field
    terms = cache.getTerms(reader, "bogusfield");


    // getDocTermOrds
    SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField");
    int numEntries = cache.getCacheEntries().length;
    // ask for it again, and check that we didnt create any additional entries:
    termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField");
    assertEquals(numEntries, cache.getCacheEntries().length);


    for (int i = 0; i < NUM_DOCS; i++) {
      termOrds.setDocument(i);
      // This will remove identical terms. A DocTermOrds doesn't return duplicate ords for a docId
      List<BytesRef> values = new ArrayList<BytesRef>(new LinkedHashSet<BytesRef>(Arrays.asList(multiValued[i])));
      for (BytesRef v : values) {
        if (v == null) {
          // why does this test use null values... instead of an empty list: confusing
          break;
        }
        long ord = termOrds.nextOrd();
        assert ord != SortedSetDocValues.NO_MORE_ORDS;
        BytesRef scratch = new BytesRef();
        termOrds.lookupOrd(ord, scratch);
        assertEquals(v, scratch);
      }
      assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd());
    }


    // test bad field
    termOrds = cache.getDocTermOrds(reader, "bogusfield");
    assertTrue(termOrds.getValueCount() == 0);


    FieldCache.DEFAULT.purge(reader);
  }

View Full Code Here

    assertEquals(0, sorted.getOrd(0));
    assertEquals(1, sorted.getValueCount());
    sorted.get(0, scratch);
    assertEquals("sorted value", scratch.utf8ToString());
    
    SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted");
    sortedSet.setDocument(0);
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(1, sortedSet.getValueCount());
    
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted");
    assertTrue(bits instanceof Bits.MatchAllBits);
    
    // Numeric type: can be retrieved via getInts() and so on
    Ints numeric = FieldCache.DEFAULT.getInts(ar, "numeric", false);
    assertEquals(42, numeric.get(0));
    
    try {
      FieldCache.DEFAULT.getTerms(ar, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      FieldCache.DEFAULT.getTermsIndex(ar, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      FieldCache.DEFAULT.getDocTermOrds(ar, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      new DocTermOrds(ar, null, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric");
    assertTrue(bits instanceof Bits.MatchAllBits);
    
    // SortedSet type: can be retrieved via getDocTermOrds() 
    if (defaultCodecSupportsSortedSet()) {
      try {
        FieldCache.DEFAULT.getInts(ar, "sortedset", false);
        fail();
      } catch (IllegalStateException expected) {}
    
      try {
        FieldCache.DEFAULT.getTerms(ar, "sortedset");
        fail();
      } catch (IllegalStateException expected) {}
    
      try {
        FieldCache.DEFAULT.getTermsIndex(ar, "sortedset");
        fail();
      } catch (IllegalStateException expected) {}
      
      try {
        new DocTermOrds(ar, null, "sortedset");
        fail();
      } catch (IllegalStateException expected) {}
    
      sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset");
      sortedSet.setDocument(0);
      assertEquals(0, sortedSet.nextOrd());
      assertEquals(1, sortedSet.nextOrd());
      assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
      assertEquals(2, sortedSet.getValueCount());
    
      bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset");
      assertTrue(bits instanceof Bits.MatchAllBits);
    }

View Full Code Here

    SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
    assertEquals(-1, sorted.getOrd(0));
    sorted.get(0, scratch);
    assertTrue(scratch.bytes == BinaryDocValues.MISSING);
    
    SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
    sortedSet.setDocument(0);
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    
    Bits bits = cache.getDocsWithField(ar, "bogusbits");
    assertFalse(bits.get(0));
    
    // check that we cached nothing

View Full Code Here

    SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
    assertEquals(-1, sorted.getOrd(0));
    sorted.get(0, scratch);
    assertTrue(scratch.bytes == BinaryDocValues.MISSING);
    
    SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
    sortedSet.setDocument(0);
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    
    Bits bits = cache.getDocsWithField(ar, "bogusbits");
    assertFalse(bits.get(0));
    
    // check that we cached nothing

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.index.SortedSetDocValues

org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesProducer

org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesProducer

org.apache.lucene.codecs.diskdv.DiskDocValuesProducer

org.apache.lucene.codecs.DocValuesConsumer

org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer

org.apache.lucene.codecs.lucene45.Lucene45DocValuesProducer

org.apache.lucene.codecs.memory.DirectDocValuesProducer

org.apache.lucene.codecs.memory.MemoryDocValuesProducer

org.apache.lucene.codecs.simpletext.SimpleTextDocValuesReader

org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.