Package org.apache.lucene.util

Examples of org.apache.lucene.util.IntsRef

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    final Map<CategoryListParams,Iterable<CategoryPath>> categoryLists = createCategoryListMapping(categories);

    // for each CLP we add a different field for drill-down terms as well as for
    // counting list data.
    IntsRef ordinals = new IntsRef(32); // should be enough for most common applications
    for (Entry<CategoryListParams, Iterable<CategoryPath>> e : categoryLists.entrySet()) {
      final CategoryListParams clp = e.getKey();
      final String field = clp.field;

      // build category list data
      ordinals.length = 0; // reset
      int maxNumOrds = 0;
      for (CategoryPath cp : e.getValue()) {
        int ordinal = taxonomyWriter.addCategory(cp);
        maxNumOrds += cp.length; // ordinal and potentially all parents
        if (ordinals.ints.length < maxNumOrds) {
          ordinals.grow(maxNumOrds);
        }
        ordinals.ints[ordinals.length++] = ordinal;
      }
      Map<String,BytesRef> categoriesData = getCategoryListData(clp, ordinals, e.getValue());
     
View Full Code Here
474
475
476
477
478
479
480
481
482
483
484
      // Build FST:
      BytesRef previousAnalyzed = null;
      BytesRef analyzed = new BytesRef();
      BytesRef surface = new BytesRef();
      IntsRef scratchInts = new IntsRef();
      ByteArrayDataInput input = new ByteArrayDataInput();

      // Used to remove duplicate surface forms (but we
      // still index the hightest-weight one).  We clear
      // this when we see a new analyzed form, so it cannot
View Full Code Here
239
240
241
242
243
244
245
246
      fstBuilder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, fstOutputs);
      indexStart = out.getFilePointer();
      ////System.out.println("VGW: field=" + fieldInfo.name);

      // Always put empty string in
      fstBuilder.add(new IntsRef(), termsFilePointer);
      startTermsFilePointer = termsFilePointer;
    }
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
   
    // per-thread resources
    final BytesReader in = fst.getBytesReader();
    final Arc<Long> firstArc = new Arc<Long>();
    final Arc<Long> scratchArc = new Arc<Long>();
    final IntsRef scratchInts = new IntsRef();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
   
    return new SortedDocValues() {
      @Override
      public int getOrd(int docID) {
        return (int) docToOrd.get(docID);
      }

      @Override
      public void lookupOrd(int ord, BytesRef result) {
        try {
          in.setPosition(0);
          fst.getFirstArc(firstArc);
          IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
          result.bytes = new byte[output.length];
          result.offset = 0;
          result.length = 0;
          Util.toBytesRef(output, result);
        } catch (IOException bogus) {
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
   
    // per-thread resources
    final BytesReader in = fst.getBytesReader();
    final Arc<Long> firstArc = new Arc<Long>();
    final Arc<Long> scratchArc = new Arc<Long>();
    final IntsRef scratchInts = new IntsRef();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
    final BytesRef ref = new BytesRef();
    final ByteArrayDataInput input = new ByteArrayDataInput();
    return new SortedSetDocValues() {
      long currentOrd;

      @Override
      public long nextOrd() {
        if (input.eof()) {
          return NO_MORE_ORDS;
        } else {
          currentOrd += input.readVLong();
          return currentOrd;
        }
      }
     
      @Override
      public void setDocument(int docID) {
        docToOrds.get(docID, ref);
        input.reset(ref.bytes, ref.offset, ref.length);
        currentOrd = 0;
      }

      @Override
      public void lookupOrd(long ord, BytesRef result) {
        try {
          in.setPosition(0);
          fst.getFirstArc(firstArc);
          IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
          result.bytes = new byte[output.length];
          result.offset = 0;
          result.length = 0;
          Util.toBytesRef(output, result);
        } catch (IOException bogus) {
View Full Code Here
599
600
601
602
603
604
605
606
607
608
609
    public void seekExact(long ord) throws IOException {
      // TODO: would be better to make this simpler and faster.
      // but we dont want to introduce a bug that corrupts our enum state!
      bytesReader.setPosition(0);
      fst.getFirstArc(firstArc);
      IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
      scratchBytes.bytes = new byte[output.length];
      scratchBytes.offset = 0;
      scratchBytes.length = 0;
      Util.toBytesRef(output, scratchBytes);
      // TODO: we could do this lazily, better to try to push into FSTEnum though?
View Full Code Here
53
54
55
56
57
58
59
60
61
62
    testFacetIDs(data10k, 10000);
    testFacetIDs(data501871, 501871);
  }

  private static IntsRef newIntsRef(int[] data) {
    IntsRef res = new IntsRef(data.length);
    System.arraycopy(data, 0, res.ints, 0, data.length);
    res.length = data.length;
    return res;
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    BytesRef bytes = new BytesRef(values.length); // at least one byte per value

    // -- Looping 100 times as a warm up --------------------------
    for (int i = 100; i != 0; --i) {
      IntsRef data = newIntsRef(values);
      encoder.encode(data, bytes);
    }
    // -----------------------------------------------------------

    long encodeTime = 0;
    for (int factor = loopFactor; factor > 0; --factor) {
      IntsRef data = newIntsRef(values);
      long start = System.currentTimeMillis();
      encoder.encode(data, bytes);
      encodeTime += System.currentTimeMillis() - start;
    }

    IntsRef decoded = new IntsRef(values.length);
    int encodedSize = bytes.length;
    IntDecoder decoder = encoder.createMatchingDecoder();
   
    // -- Looping 100 times as a warm up --------------------------
    for (int i = 100; i != 0; --i) {
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
    meta.writeVInt(field.number);
    meta.writeByte(FST);
    meta.writeLong(data.getFilePointer());
    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
    Builder<Long> builder = new Builder<Long>(INPUT_TYPE.BYTE1, outputs);
    IntsRef scratch = new IntsRef();
    long ord = 0;
    for (BytesRef v : values) {
      builder.add(Util.toIntsRef(v, scratch), ord);
      ord++;
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  private static IntsRef uniqueSortedData, data;
 
  @BeforeClass
  public static void beforeClassEncodingTest() throws Exception {
    int capacity = atLeast(10000);
    data = new IntsRef(capacity);
    for (int i = 0; i < 10; i++) {
      data.ints[i] = i + 1; // small values
    }
    for (int i = 10; i < data.ints.length; i++) {
      data.ints[i] = random().nextInt(Integer.MAX_VALUE - 1) + 1; // some encoders don't allow 0
View Full Code Here
TOP

Related Classes of org.apache.lucene.util.IntsRef

  • org.apache.lucene.analysis.charfilter.NormalizeCharMap$Builder
  • org.apache.lucene.analysis.hunspell.Dictionary
  • org.apache.lucene.analysis.hunspell.Stemmer
  • org.apache.lucene.analysis.hunspell.TestDictionary
  • org.apache.lucene.analysis.ja.dict.TestTokenInfoDictionary
  • org.apache.lucene.analysis.ja.dict.UserDictionary
  • org.apache.lucene.analysis.ja.util.TokenInfoDictionaryBuilder
  • org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter$Builder
  • org.apache.lucene.analysis.synonym.SynonymMap$Builder
  • org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader$FieldIndexData
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.