Package org.apache.lucene.index

Examples of org.apache.lucene.index.AtomicReader


    Directory dirTmp = getDirectory(new File(indexPath.toString() + ".tmp"));

    IndexWriter w = null;
    IndexWriter w2 = null;
    AtomicReader r = null;
    boolean success = false;
    try {
      Analyzer gramAnalyzer = new AnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
          @Override
          protected Analyzer getWrappedAnalyzer(String fieldName) {
            return indexAnalyzer;
          }

          @Override
          protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
            if (fieldName.equals("textgrams") && minPrefixChars > 0) {
              return new TokenStreamComponents(components.getTokenizer(),
                                               new EdgeNGramTokenFilter(matchVersion,
                                                                        components.getTokenStream(),
                                                                        1, minPrefixChars));
            } else {
              return components;
            }
          }
        };

      w = new IndexWriter(dirTmp,
                          getIndexWriterConfig(matchVersion, gramAnalyzer));
      BytesRef text;
      Document doc = new Document();
      FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
      ft.setIndexOptions(IndexOptions.DOCS_ONLY);
      ft.setOmitNorms(true);
      Field textField = new Field(TEXT_FIELD_NAME, "", ft);
      doc.add(textField);

      Field textGramField = new Field("textgrams", "", ft);
      doc.add(textGramField);

      Field textDVField = new BinaryDocValuesField(TEXT_FIELD_NAME, new BytesRef());
      doc.add(textDVField);

      // TODO: use threads...?
      Field weightField = new NumericDocValuesField("weight", 0);
      doc.add(weightField);

      Field payloadField;
      if (iter.hasPayloads()) {
        payloadField = new BinaryDocValuesField("payloads", new BytesRef());
        doc.add(payloadField);
      } else {
        payloadField = null;
      }

      //long t0 = System.nanoTime();
      while ((text = iter.next()) != null) {
        String textString = text.utf8ToString();
        textField.setStringValue(textString);
        textGramField.setStringValue(textString);
        textDVField.setBytesValue(text);
        weightField.setLongValue(iter.weight());
        if (iter.hasPayloads()) {
          payloadField.setBytesValue(iter.payload());
        }
        w.addDocument(doc);
      }
      //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");

      r = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(w, false));
      //long t1 = System.nanoTime();
      w.rollback();

      final int maxDoc = r.maxDoc();

      final NumericDocValues weights = r.getNumericDocValues("weight");

      final Sorter.DocComparator comparator = new Sorter.DocComparator() {
          @Override
          public int compare(int docID1, int docID2) {
            final long v1 = weights.get(docID1);
            final long v2 = weights.get(docID2);
            // Reverse sort (highest weight first);
            // java7 only:
            //return Long.compare(v2, v1);
            if (v1 > v2) {
              return -1;
            } else if (v1 < v2) {
              return 1;
            } else {
              return 0;
            }
          }
        };

      r = SortingAtomicReader.wrap(r, new Sorter() {
          @Override
          public Sorter.DocMap sort(AtomicReader reader) throws IOException {
            return Sorter.sort(maxDoc, comparator);
          }

          @Override
          public String getID() {
            return "Weight";
          }
        });
     
      w2 = new IndexWriter(dir,
                           getIndexWriterConfig(matchVersion, indexAnalyzer));
      w2.addIndexes(new IndexReader[] {r});
      r.close();

      //System.out.println("sort time: " + ((System.nanoTime()-t1)/1000000) + " msec");

      searcher = new IndexSearcher(DirectoryReader.open(w2, false));
      w2.close();
View Full Code Here


    }
  }

  private OpenBitSetDISI initialResult(AtomicReaderContext context, int logic, int[] index)
      throws IOException {
    AtomicReader reader = context.reader();
    OpenBitSetDISI result;
    /**
     * First AND operation takes place against a completely false
     * bitset and will always return zero results.
     */
    if (logic == AND) {
      result = new OpenBitSetDISI(getDISI(chain[index[0]], context), reader.maxDoc());
      ++index[0];
    } else if (logic == ANDNOT) {
      result = new OpenBitSetDISI(getDISI(chain[index[0]], context), reader.maxDoc());
      result.flip(0, reader.maxDoc()); // NOTE: may set bits for deleted docs.
      ++index[0];
    } else {
      result = new OpenBitSetDISI(reader.maxDoc());
    }
    return result;
  }
View Full Code Here

      }

      final Map<Integer, JoinScore> docToJoinScore = new HashMap<Integer, JoinScore>();
      if (multipleValuesPerDocument) {
        if (scoreDocsInOrder) {
          AtomicReader slowCompositeReader = SlowCompositeReaderWrapper.wrap(toSearcher.getIndexReader());
          Terms terms = slowCompositeReader.terms(toField);
          if (terms != null) {
            DocsEnum docsEnum = null;
            TermsEnum termsEnum = null;
            SortedSet<BytesRef> joinValues = new TreeSet<BytesRef>(BytesRef.getUTF8SortedAsUnicodeComparator());
            joinValues.addAll(joinValueToJoinScores.keySet());
            for (BytesRef joinValue : joinValues) {
              termsEnum = terms.iterator(termsEnum);
              if (termsEnum.seekExact(joinValue)) {
                docsEnum = termsEnum.docs(slowCompositeReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE);
                JoinScore joinScore = joinValueToJoinScores.get(joinValue);

                for (int doc = docsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docsEnum.nextDoc()) {
                  // First encountered join value determines the score.
                  // Something to keep in mind for many-to-many relations.
View Full Code Here

   * of the filters that have been added.
   */
  @Override
  public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
    FixedBitSet res = null;
    final AtomicReader reader = context.reader();
   
    boolean hasShouldClauses = false;
    for (final FilterClause fc : clauses) {
      if (fc.getOccur() == Occur.SHOULD) {
        hasShouldClauses = true;
        final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
        if (disi == null) continue;
        if (res == null) {
          res = new FixedBitSet(reader.maxDoc());
        }
        res.or(disi);
      }
    }
    if (hasShouldClauses && res == null)
      return null;
   
    for (final FilterClause fc : clauses) {
      if (fc.getOccur() == Occur.MUST_NOT) {
        if (res == null) {
          assert !hasShouldClauses;
          res = new FixedBitSet(reader.maxDoc());
          res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs
        }
        final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
        if (disi != null) {
          res.andNot(disi);
        }
      }
    }
   
    for (final FilterClause fc : clauses) {
      if (fc.getOccur() == Occur.MUST) {
        final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
        if (disi == null) {
          return null; // no documents can match
        }
        if (res == null) {
          res = new FixedBitSet(reader.maxDoc());
          res.or(disi);
        } else {
          res.and(disi);
        }
      }
View Full Code Here

      assertTrue("ndv(" + (i-1) + ")=" + ndv.get(i-1) + ",ndv(" + i + ")=" + ndv.get(i), ndv.get(i-1) <= ndv.get(i));
    }
  }

  public void testSortingMP() throws IOException {
    final AtomicReader sortedReader1 = SortingAtomicReader.wrap(SlowCompositeReaderWrapper.wrap(reader), sorter);
    final AtomicReader sortedReader2 = SlowCompositeReaderWrapper.wrap(sortedReader);

    assertSorted(sortedReader1);
    assertSorted(sortedReader2);
   
    assertReaderEquals("", sortedReader1, sortedReader2);
View Full Code Here

  private void migrateIndex(Directory indexDir, FacetIndexingParams fip) throws Exception {
    final Map<String,Term> fieldTerms = FacetsPayloadMigrationReader.buildFieldTermsMap(indexDir, fip);
    DirectoryReader reader = DirectoryReader.open(indexDir);
    List<AtomicReaderContext> leaves = reader.leaves();
    int numReaders = leaves.size();
    AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
    for (int i = 0; i < numReaders; i++) {
      wrappedLeaves[i] = new FacetsPayloadMigrationReader(leaves.get(i).reader(), fieldTerms);
    }
   
    IndexWriter writer = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
View Full Code Here

        new Filter() {
          @Override
          public DocIdSet getDocIdSet(AtomicReaderContext context,
              Bits acceptDocs) throws IOException {
            final boolean nullBitset = random().nextInt(10) == 5;
            final AtomicReader reader = context.reader();
            DocsEnum termDocsEnum = reader.termDocsEnum(new Term("field", "0"));
            if (termDocsEnum == null) {
              return null; // no docs -- return null
            }
            final BitSet bitSet = new BitSet(reader.maxDoc());
            int d;
            while ((d = termDocsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
              bitSet.set(d, true);
            }
            return new DocIdSet() {
             
              @Override
              public Bits bits() throws IOException {
                if (nullBitset) {
                  return null;
                }
                return new Bits() {
                 
                  @Override
                  public boolean get(int index) {
                    assertTrue("filter was called for a non-matching doc",
                        bitSet.get(index));
                    return bitSet.get(index);
                  }
                 
                  @Override
                  public int length() {
                    return bitSet.length();
                  }
                 
                };
              }
             
              @Override
              public DocIdSetIterator iterator() throws IOException {
                assertTrue(
                    "iterator should not be called if bitset is present",
                    nullBitset);
                return reader.termDocsEnum(new Term("field", "0"));
              }
             
            };
          }
        }, FilteredQuery.QUERY_FIRST_FILTER_STRATEGY);
View Full Code Here

    doc.add(new TextField("content", new StringReader(
        "a a b c d e a f g h i j a b k k")));
    writer.addDocument(doc);

    final IndexReader readerFromWriter = writer.getReader();
    AtomicReader r = SlowCompositeReaderWrapper.wrap(readerFromWriter);

    DocsAndPositionsEnum tp = r.termPositionsEnum(new Term("content", "a"));
   
    int count = 0;
    assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
    // "a" occurs 4 times
    assertEquals(4, tp.freq());
View Full Code Here

    @Override
    public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
        boolean topScorer, Bits acceptDocs) throws IOException {
      assert !terms.isEmpty();
      final AtomicReader reader = context.reader();
      final Bits liveDocs = acceptDocs;
      PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];

      final Terms fieldTerms = reader.terms(field);
      if (fieldTerms == null) {
        return null;
      }

      // Reuse single TermsEnum below:
View Full Code Here

  // for testing
  int hitCount, missCount;

  @Override
  public DocIdSet getDocIdSet(AtomicReaderContext context, final Bits acceptDocs) throws IOException {
    final AtomicReader reader = context.reader();
    final Object key = reader.getCoreCacheKey();

    DocIdSet docIdSet = cache.get(key);
    if (docIdSet != null) {
      hitCount++;
    } else {
View Full Code Here

TOP

Related Classes of org.apache.lucene.index.AtomicReader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.