Examples of org.apache.lucene.index.SortedDocValues

org.apache.lucene.index.SortedDocValues
A per-document byte[] with presorted values.
Per-Document values in a SortedDocValues are deduplicated, dereferenced, and sorted into a dictionary of unique values. A pointer to the dictionary value (ordinal) can be retrieved for each document. Ordinals are dense and in increasing sorted order.

    }


    @Override
    public SortedDocValues getSorted(FieldInfo field) throws IOException {
      assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED;
      SortedDocValues values = in.getSorted(field);
      assert values != null;
      return new AssertingAtomicReader.AssertingSortedDocValues(values, maxDoc);
    }

View Full Code Here

    NumericEntry entry = ords.get(field.number);
    IndexInput data = this.data.clone();
    data.seek(entry.offset);
    final BlockPackedReader ordinals = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
    
    return new SortedDocValues() {


      @Override
      public int getOrd(int docID) {
        return (int) ordinals.get(docID);
      }

View Full Code Here


  @Override
  public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
    SortedSetEntry ss = sortedSets.get(field.number);
    if (ss.format == SORTED_SET_SINGLE_VALUED_SORTED) {
      final SortedDocValues values = getSorted(field);
      return DocValues.singleton(values);
    } else if (ss.format != SORTED_SET_WITH_ADDRESSES) {
      throw new AssertionError();
    }

View Full Code Here

   * The default implementation calls {@link #addSortedField}, passing
   * an Iterable that merges ordinals and values and filters deleted documents .
   */
  public void mergeSortedField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedDocValues> toMerge) throws IOException {
    final AtomicReader readers[] = mergeState.readers.toArray(new AtomicReader[toMerge.size()]);
    final SortedDocValues dvs[] = toMerge.toArray(new SortedDocValues[toMerge.size()]);
    
    // step 1: iterate thru each sub and mark terms still in use
    TermsEnum liveTerms[] = new TermsEnum[dvs.length];
    for (int sub = 0; sub < liveTerms.length; sub++) {
      AtomicReader reader = readers[sub];
      SortedDocValues dv = dvs[sub];
      Bits liveDocs = reader.getLiveDocs();
      if (liveDocs == null) {
        liveTerms[sub] = dv.termsEnum();
      } else {
        LongBitSet bitset = new LongBitSet(dv.getValueCount());
        for (int i = 0; i < reader.maxDoc(); i++) {
          if (liveDocs.get(i)) {
            int ord = dv.getOrd(i);
            if (ord >= 0) {
              bitset.set(ord);
            }
          }
        }
        liveTerms[sub] = new BitsFilteredTermsEnum(dv.termsEnum(), bitset);
      }
    }
    
    // step 2: create ordinal map (this conceptually does the "merging")
    final OrdinalMap map = new OrdinalMap(this, liveTerms);

View Full Code Here

          assertNull(info, rightValues);
        }
      }
      
      {
        SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field);
        SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field);
        if (leftValues != null && rightValues != null) {
          // numOrds
          assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
          // ords
          BytesRef scratchLeft = new BytesRef();
          BytesRef scratchRight = new BytesRef();
          for (int i = 0; i < leftValues.getValueCount(); i++) {
            final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i));
            final BytesRef right = rightValues.lookupOrd(i);
            assertEquals(info, left, right);
          }
          // bytes
          for(int docID=0;docID<leftReader.maxDoc();docID++) {
            final BytesRef left = BytesRef.deepCopyOf(leftValues.get(docID));
            final BytesRef right = rightValues.get(docID);
            assertEquals(info, left, right);
          }
        } else {
          assertNull(info, leftValues);
          assertNull(info, rightValues);
        }
      }
      
      {
        SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field);
        SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field);
        if (leftValues != null && rightValues != null) {
          // numOrds
          assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
          // ords
          for (int i = 0; i < leftValues.getValueCount(); i++) {
            final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i));
            final BytesRef right = rightValues.lookupOrd(i);
            assertEquals(info, left, right);
          }
          // ord lists
          for(int docID=0;docID<leftReader.maxDoc();docID++) {
            leftValues.setDocument(docID);
            rightValues.setDocument(docID);
            long ord;
            while ((ord = leftValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
              assertEquals(info, ord, rightValues.nextOrd());
            }
            assertEquals(info, SortedSetDocValues.NO_MORE_ORDS, rightValues.nextOrd());
          }
        } else {
          assertNull(info, leftValues);
          assertNull(info, rightValues);
        }
      }
      
      {
        SortedNumericDocValues leftValues = MultiDocValues.getSortedNumericValues(leftReader, field);
        SortedNumericDocValues rightValues = MultiDocValues.getSortedNumericValues(rightReader, field);
        if (leftValues != null && rightValues != null) {
          for (int i = 0; i < leftReader.maxDoc(); i++) {
            leftValues.setDocument(i);
            long expected[] = new long[leftValues.count()];
            for (int j = 0; j < expected.length; j++) {
              expected[j] = leftValues.valueAt(j);
            }
            rightValues.setDocument(i);
            for (int j = 0; j < expected.length; j++) {
              assertEquals(info, expected[j], rightValues.valueAt(j));
            }
            assertEquals(info, expected.length, rightValues.count());
          }
        } else {
          assertNull(info, leftValues);
          assertNull(info, rightValues);
        }

View Full Code Here

    
    // 3. execute the algorithm  (required in every "logic" test)
    Benchmark benchmark = execBenchmark(algLines);


    DirectoryReader r = DirectoryReader.open(benchmark.getRunData().getDirectory());
    SortedDocValues idx = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(r), "country");
    final int maxDoc = r.maxDoc();
    assertEquals(1000, maxDoc);
    for(int i=0;i<1000;i++) {
      assertTrue("doc " + i + " has null country", idx.getOrd(i) != -1);
    }
    r.close();
  }

View Full Code Here

    }
  }


  @Override
  public SortedDocValues getSortedDocValues(String field) throws IOException {
    SortedDocValues sortedDV = in.getSortedDocValues(field);
    if (sortedDV == null) {
      return null;
    } else {
      return new SortingSortedDocValues(sortedDV, docMap);
    }

View Full Code Here

    final Arc<Long> firstArc = new Arc<>();
    final Arc<Long> scratchArc = new Arc<>();
    final IntsRef scratchInts = new IntsRef();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
    
    return new SortedDocValues() {
      final BytesRef term = new BytesRef();


      @Override
      public int getOrd(int docID) {
        return (int) docToOrd.get(docID);

View Full Code Here

    final IndexInput in = data.clone();
    final BytesRef scratch = new BytesRef();
    final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
    final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));


    return new SortedDocValues() {
      final BytesRef term = new BytesRef();


      @Override
      public int getOrd(int docID) {
        if (docID < 0 || docID >= maxDoc) {

View Full Code Here

    }
  }
  
  @Test
  public void testSortedDocValuesField() throws Exception {
    SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD);
    int maxDoc = reader.maxDoc();
    for (int i = 0; i < maxDoc; i++) {
      final BytesRef bytes = dv.get(i);
      assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.index.SortedDocValues

com.browseengine.bobo.sort.DocComparatorSource$StringOrdComparatorSource

org.apache.lucene.benchmark.byTask.TestPerfTasksLogic

org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesProducer

org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesProducer

org.apache.lucene.codecs.diskdv.DiskDocValuesProducer

org.apache.lucene.codecs.DocValuesConsumer

org.apache.lucene.codecs.lucene40.Lucene40DocValuesReader

org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer

org.apache.lucene.codecs.lucene45.Lucene45DocValuesProducer

org.apache.lucene.codecs.lucene49.Lucene49DocValuesProducer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.