Examples of org.apache.lucene.index.DirectoryReader

org.apache.lucene.index.DirectoryReader
An IndexReader which reads indexes with multiple segments.

   * {@link OrdinalMap} with a mapping from the original ordinal to the new
   * ordinal.
   */
  public void addTaxonomy(Directory taxoDir, OrdinalMap map) throws IOException {
    ensureOpen();
    DirectoryReader r = DirectoryReader.open(taxoDir);
    try {
      final int size = r.numDocs();
      final OrdinalMap ordinalMap = map;
      ordinalMap.setSize(size);
      int base = 0;
      TermsEnum te = null;
      DocsEnum docs = null;
      for (final AtomicReaderContext ctx : r.leaves()) {
        final AtomicReader ar = ctx.reader();
        final Terms terms = ar.terms(Consts.FULL);
        te = terms.iterator(te);
        while (te.next() != null) {
          String value = te.term().utf8ToString();
          CategoryPath cp = new CategoryPath(value, delimiter);
          final int ordinal = addCategory(cp);
          docs = te.docs(null, docs, DocsEnum.FLAG_NONE);
          ordinalMap.addMapping(docs.nextDoc() + base, ordinal);
        }
        base += ar.maxDoc(); // no deletions, so we're ok
      }
      ordinalMap.addDone();
    } finally {
      r.close();
    }
  }

View Full Code Here

    return result;
  }


  private boolean applyDeletes(Directory directory, IndexWriter indexWriter, String shard, boolean emitDeletes)
      throws IOException {
    DirectoryReader reader = DirectoryReader.open(directory);
    try {
      LOG.info("Applying deletes in reader [{0}]", reader);
      CompositeReaderContext compositeReaderContext = reader.getContext();
      List<AtomicReaderContext> leaves = compositeReaderContext.leaves();
      BlurPartitioner blurPartitioner = new BlurPartitioner();
      Text key = new Text();
      int numberOfShards = _shardContext.getTableContext().getDescriptor().getShardCount();
      int shardId = BlurUtil.getShardIndex(shard);
      for (AtomicReaderContext context : leaves) {
        AtomicReader atomicReader = context.reader();
        Fields fields = atomicReader.fields();
        Terms terms = fields.terms(BlurConstants.ROW_ID);
        if (terms != null) {
          TermsEnum termsEnum = terms.iterator(null);
          BytesRef ref = null;
          while ((ref = termsEnum.next()) != null) {
            key.set(ref.bytes, ref.offset, ref.length);
            int partition = blurPartitioner.getPartition(key, null, numberOfShards);
            if (shardId != partition) {
              return false;
            }
            if (emitDeletes) {
              indexWriter.deleteDocuments(new Term(BlurConstants.ROW_ID, BytesRef.deepCopyOf(ref)));
            }
          }
        }
      }
    } finally {
      reader.close();
    }
    return true;
  }

View Full Code Here

      if (_usingLocalTmpindex) {
        // since we have flushed to disk then we do not need to index the
        // delete.
        flushToTmpIndex();
        _localTmpWriter.close(false);
        DirectoryReader reader = DirectoryReader.open(_localTmpDir);
        _recordRateCounter.mark(reader.numDocs());
        _writer.addIndexes(reader);
        reader.close();
        resetLocalTmp();
        _rowOverFlowCount.increment(1);
      } else {
        if (_documents.isEmpty()) {
          if (_deletedRowId != null) {

View Full Code Here

    }


    private void copyAndOptimizeInFlightDir() throws IOException {
      CopyRateDirectory copyRateDirectory = new CopyRateDirectory(_finalDir, _copyRateCounter);
      copyRateDirectory.setLockFactory(NoLockFactory.getNoLockFactory());
      DirectoryReader reader = DirectoryReader.open(_localDir);
      IndexWriter writer = new IndexWriter(copyRateDirectory, _conf.clone());
      writer.addIndexes(reader);
      writer.close();
      rm(_localPath);
    }

View Full Code Here

  @Override
  public void refresh() throws IOException {
    if (!_open.get()) {
      return;
    }
    DirectoryReader oldReader = _indexReaderRef.get();
    DirectoryReader reader = DirectoryReader.openIfChanged(oldReader);
    if (reader != null) {
      _indexReaderRef.set(reader);
      _closer.close(oldReader);
    }
  }

View Full Code Here

    throw new RuntimeException("Read-only shard");
  }


  @Override
  public IndexSearcherClosable getIndexReader() throws IOException {
    final DirectoryReader reader = _indexReaderRef.get();
    reader.incRef();
    return new IndexSearcherClosable(reader, null) {


      @Override
      public Directory getDirectory() {
        return _directory;
      }


      @Override
      public void close() throws IOException {
        reader.decRef();
      }
    };
  }

View Full Code Here

  @Override
  protected DirectoryTaxonomyReader doOpenIfChanged() throws IOException {
    ensureOpen();
    
    // This works for both NRT and non-NRT readers (i.e. an NRT reader remains NRT).
    final DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
    if (r2 == null) {
      return null; // no changes, nothing to do
    }


    // check if the taxonomy was recreated
    boolean success = false;
    try {
      boolean recreated = false;
      if (taxoWriter == null) {
        // not NRT, check epoch from commit data
        String t1 = indexReader.getIndexCommit().getUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH);
        String t2 = r2.getIndexCommit().getUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH);
        if (t1 == null) {
          if (t2 != null) {
            recreated = true;
          }
        } else if (!t1.equals(t2)) {

View Full Code Here

    if (defaultCodecSupportsSortedSet()) {
      doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
      doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
    }
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
    AtomicReader ar = getOnlySegmentReader(ir);
    
    BytesRef scratch = new BytesRef();
    
    // Binary type: can be retrieved via getTerms()
    try {
      FieldCache.DEFAULT.getInts(ar, "binary", false);
      fail();
    } catch (IllegalStateException expected) {}
    
    BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
    binary.get(0, scratch);
    assertEquals("binary value", scratch.utf8ToString());
    
    try {
      FieldCache.DEFAULT.getTermsIndex(ar, "binary");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      FieldCache.DEFAULT.getDocTermOrds(ar, "binary");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      new DocTermOrds(ar, null, "binary");
      fail();
    } catch (IllegalStateException expected) {}
    
    Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary");
    assertTrue(bits.get(0));
    
    // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
    try {
      FieldCache.DEFAULT.getInts(ar, "sorted", false);
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      new DocTermOrds(ar, null, "sorted");
      fail();
    } catch (IllegalStateException expected) {}
    
    binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
    binary.get(0, scratch);
    assertEquals("sorted value", scratch.utf8ToString());
    
    SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
    assertEquals(0, sorted.getOrd(0));
    assertEquals(1, sorted.getValueCount());
    sorted.get(0, scratch);
    assertEquals("sorted value", scratch.utf8ToString());
    
    SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted");
    sortedSet.setDocument(0);
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(1, sortedSet.getValueCount());
    
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted");
    assertTrue(bits.get(0));
    
    // Numeric type: can be retrieved via getInts() and so on
    Ints numeric = FieldCache.DEFAULT.getInts(ar, "numeric", false);
    assertEquals(42, numeric.get(0));
    
    try {
      FieldCache.DEFAULT.getTerms(ar, "numeric", true);
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      FieldCache.DEFAULT.getTermsIndex(ar, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      FieldCache.DEFAULT.getDocTermOrds(ar, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    try {
      new DocTermOrds(ar, null, "numeric");
      fail();
    } catch (IllegalStateException expected) {}
    
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric");
    assertTrue(bits.get(0));
    
    // SortedSet type: can be retrieved via getDocTermOrds() 
    if (defaultCodecSupportsSortedSet()) {
      try {
        FieldCache.DEFAULT.getInts(ar, "sortedset", false);
        fail();
      } catch (IllegalStateException expected) {}
    
      try {
        FieldCache.DEFAULT.getTerms(ar, "sortedset", true);
        fail();
      } catch (IllegalStateException expected) {}
    
      try {
        FieldCache.DEFAULT.getTermsIndex(ar, "sortedset");
        fail();
      } catch (IllegalStateException expected) {}
      
      try {
        new DocTermOrds(ar, null, "sortedset");
        fail();
      } catch (IllegalStateException expected) {}
    
      sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset");
      sortedSet.setDocument(0);
      assertEquals(0, sortedSet.nextOrd());
      assertEquals(1, sortedSet.nextOrd());
      assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
      assertEquals(2, sortedSet.getValueCount());
    
      bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset");
      assertTrue(bits.get(0));
    }
    
    ir.close();
    dir.close();
  }

View Full Code Here

  public void testNonexistantFields() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
    
    AtomicReader ar = getOnlySegmentReader(ir);
    
    final FieldCache cache = FieldCache.DEFAULT;
    cache.purgeAllCaches();
    assertEquals(0, cache.getCacheEntries().length);
    
    Bytes bytes = cache.getBytes(ar, "bogusbytes", true);
    assertEquals(0, bytes.get(0));


    Shorts shorts = cache.getShorts(ar, "bogusshorts", true);
    assertEquals(0, shorts.get(0));
    
    Ints ints = cache.getInts(ar, "bogusints", true);
    assertEquals(0, ints.get(0));
    
    Longs longs = cache.getLongs(ar, "boguslongs", true);
    assertEquals(0, longs.get(0));
    
    Floats floats = cache.getFloats(ar, "bogusfloats", true);
    assertEquals(0, floats.get(0), 0.0f);
    
    Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true);
    assertEquals(0, doubles.get(0), 0.0D);
    
    BytesRef scratch = new BytesRef();
    BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
    binaries.get(0, scratch);
    assertEquals(0, scratch.length);
    
    SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
    assertEquals(-1, sorted.getOrd(0));
    sorted.get(0, scratch);
    assertEquals(0, scratch.length);
    
    SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
    sortedSet.setDocument(0);
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    
    Bits bits = cache.getDocsWithField(ar, "bogusbits");
    assertFalse(bits.get(0));
    
    // check that we cached nothing
    assertEquals(0, cache.getCacheEntries().length);
    ir.close();
    dir.close();
  }

View Full Code Here

    doc.add(new StoredField("bogusterms", "bogus"));
    doc.add(new StoredField("bogustermsindex", "bogus"));
    doc.add(new StoredField("bogusmultivalued", "bogus"));
    doc.add(new StoredField("bogusbits", "bogus"));
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
    
    AtomicReader ar = getOnlySegmentReader(ir);
    
    final FieldCache cache = FieldCache.DEFAULT;
    cache.purgeAllCaches();
    assertEquals(0, cache.getCacheEntries().length);
    
    Bytes bytes = cache.getBytes(ar, "bogusbytes", true);
    assertEquals(0, bytes.get(0));


    Shorts shorts = cache.getShorts(ar, "bogusshorts", true);
    assertEquals(0, shorts.get(0));
    
    Ints ints = cache.getInts(ar, "bogusints", true);
    assertEquals(0, ints.get(0));
    
    Longs longs = cache.getLongs(ar, "boguslongs", true);
    assertEquals(0, longs.get(0));
    
    Floats floats = cache.getFloats(ar, "bogusfloats", true);
    assertEquals(0, floats.get(0), 0.0f);
    
    Doubles doubles = cache.getDoubles(ar, "bogusdoubles", true);
    assertEquals(0, doubles.get(0), 0.0D);
    
    BytesRef scratch = new BytesRef();
    BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
    binaries.get(0, scratch);
    assertEquals(0, scratch.length);
    
    SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
    assertEquals(-1, sorted.getOrd(0));
    sorted.get(0, scratch);
    assertEquals(0, scratch.length);
    
    SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
    sortedSet.setDocument(0);
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    
    Bits bits = cache.getDocsWithField(ar, "bogusbits");
    assertFalse(bits.get(0));
    
    // check that we cached nothing
    assertEquals(0, cache.getCacheEntries().length);
    ir.close();
    dir.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.index.DirectoryReader

com.browseengine.bobo.facets.attribute.AttributesFacetHandlerTest

com.browseengine.bobo.test.BasicIndexingTest

com.browseengine.bobo.test.BoboTestCase

com.browseengine.bobo.test.FacetHandlerTest

com.browseengine.bobo.test.FacetNameTest

com.browseengine.bobo.test.FacetNotValuesTest

com.google.sitebricks.persist.disk.IndexSet

com.tistory.devyongsik.crescent.admin.service.IndexFileManageServiceImpl

org.apache.blur.filter.FilterCacheTest

org.apache.blur.lucene.codec.Blur022CodecTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.