Examples of org.apache.lucene.index.TrackingIndexWriter

org.apache.lucene.index.TrackingIndexWriter
Class that tracks changes to a delegated IndexWriter, used by {@link ControlledRealTimeReopenThread} to ensure specificchanges are visible. Create this class (passing your IndexWriter), and then pass this class to {@link ControlledRealTimeReopenThread}. Be sure to make all changes via the TrackingIndexWriter, otherwise {@link ControlledRealTimeReopenThread} won't know about the changes. @lucene.experimental


    if (VERBOSE) {
      System.out.println("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
    }


    genWriter = new TrackingIndexWriter(writer);


    final SearcherFactory sf = new SearcherFactory() {
        @Override
        public IndexSearcher newSearcher(IndexReader r) throws IOException {
          TestControlledRealTimeReopenThread.this.warmCalled = true;

View Full Code Here

    Directory d = newDirectory();
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch signal = new CountDownLatch(1);


    LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
    final TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
    final SearcherManager manager = new SearcherManager(_writer, false, null);
    Document doc = new Document();
    doc.add(newTextField("test", "test", Field.Store.YES));
    writer.addDocument(doc);
    manager.maybeRefresh();
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          signal.await();
          manager.maybeRefresh();
          writer.deleteDocuments(new TermQuery(new Term("foo", "barista")));
          manager.maybeRefresh(); // kick off another reopen so we inc. the internal gen
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          latch.countDown(); // let the add below finish
        }
      }
    };
    t.start();
    _writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
    final long lastGen = writer.updateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen


    assertFalse(manager.isSearcherCurrent()); // false since there is a delete in the queue
    
    IndexSearcher searcher = manager.acquire();
    try {

View Full Code Here


    if (VERBOSE) {
      System.out.println("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
    }


    genWriter = new TrackingIndexWriter(writer);


    final SearcherFactory sf = new SearcherFactory() {
        @Override
        public IndexSearcher newSearcher(IndexReader r) throws IOException {
          TestControlledRealTimeReopenThread.this.warmCalled = true;

View Full Code Here

    Directory d = newDirectory();
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch signal = new CountDownLatch(1);


    LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
    final TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
    final SearcherManager manager = new SearcherManager(_writer, false, null);
    Document doc = new Document();
    doc.add(newTextField("test", "test", Field.Store.YES));
    writer.addDocument(doc);
    manager.maybeRefresh();
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          signal.await();
          manager.maybeRefresh();
          writer.deleteDocuments(new TermQuery(new Term("foo", "barista")));
          manager.maybeRefresh(); // kick off another reopen so we inc. the internal gen
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          latch.countDown(); // let the add below finish
        }
      }
    };
    t.start();
    _writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
    final long lastGen = writer.updateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen


    assertFalse(manager.isSearcherCurrent()); // false since there is a delete in the queue
    
    IndexSearcher searcher = manager.acquire();
    try {

View Full Code Here

                                                     new MockAnalyzer(random()));
    config.setIndexDeletionPolicy(sdp);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    final IndexWriter iw = new IndexWriter(dir, config);
    SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
    final TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
    ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread =
      new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0);


    controlledRealTimeReopenThread.setDaemon(true);
    controlledRealTimeReopenThread.start();


    List<Thread> commitThreads = new ArrayList<Thread>();


    for (int i = 0; i < 500; i++) {
      if (i > 0 && i % 50 == 0) {
        Thread commitThread =  new Thread(new Runnable() {
            @Override
            public void run() {
              try {
                iw.commit();
                IndexCommit ic = sdp.snapshot();
                for (String name : ic.getFileNames()) {
                  //distribute, and backup
                  //System.out.println(names);
                  assertTrue(dir.fileExists(name));
                }
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          });
        commitThread.start();
        commitThreads.add(commitThread);
      }
      Document d = new Document();
      d.add(new TextField("count", i + "", Field.Store.NO));
      d.add(new TextField("content", content, Field.Store.YES));
      long start = System.currentTimeMillis();
      long l = tiw.addDocument(d);
      controlledRealTimeReopenThread.waitForGeneration(l);
      long wait = System.currentTimeMillis() - start;
      assertTrue("waited too long for generation " + wait,
                 wait < (maxStaleSecs *1000));
      IndexSearcher searcher = sm.acquire();

View Full Code Here


    if (VERBOSE) {
      System.out.println("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
    }


    genWriter = new TrackingIndexWriter(writer);


    final SearcherFactory sf = new SearcherFactory() {
        @Override
        public IndexSearcher newSearcher(IndexReader r) throws IOException {
          TestControlledRealTimeReopenThread.this.warmCalled = true;

View Full Code Here

    Directory d = newDirectory();
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch signal = new CountDownLatch(1);


    LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
    final TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
    final SearcherManager manager = new SearcherManager(_writer, false, null);
    Document doc = new Document();
    doc.add(newTextField("test", "test", Field.Store.YES));
    writer.addDocument(doc);
    manager.maybeRefresh();
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          signal.await();
          manager.maybeRefresh();
          writer.deleteDocuments(new TermQuery(new Term("foo", "barista")));
          manager.maybeRefresh(); // kick off another reopen so we inc. the internal gen
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          latch.countDown(); // let the add below finish
        }
      }
    };
    t.start();
    _writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
    final long lastGen = writer.updateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen


    assertFalse(manager.isSearcherCurrent()); // false since there is a delete in the queue
    
    IndexSearcher searcher = manager.acquire();
    try {

View Full Code Here

                                                     new MockAnalyzer(random()));
    config.setIndexDeletionPolicy(sdp);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    final IndexWriter iw = new IndexWriter(dir, config);
    SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
    final TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
    ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread =
      new ControlledRealTimeReopenThread<>(tiw, sm, maxStaleSecs, 0);


    controlledRealTimeReopenThread.setDaemon(true);
    controlledRealTimeReopenThread.start();


    List<Thread> commitThreads = new ArrayList<>();


    for (int i = 0; i < 500; i++) {
      if (i > 0 && i % 50 == 0) {
        Thread commitThread =  new Thread(new Runnable() {
            @Override
            public void run() {
              try {
                iw.commit();
                IndexCommit ic = sdp.snapshot();
                for (String name : ic.getFileNames()) {
                  //distribute, and backup
                  //System.out.println(names);
                  assertTrue(slowFileExists(dir, name));
                }
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          });
        commitThread.start();
        commitThreads.add(commitThread);
      }
      Document d = new Document();
      d.add(new TextField("count", i + "", Field.Store.NO));
      d.add(new TextField("content", content, Field.Store.YES));
      long start = System.currentTimeMillis();
      long l = tiw.addDocument(d);
      controlledRealTimeReopenThread.waitForGeneration(l);
      long wait = System.currentTimeMillis() - start;
      assertTrue("waited too long for generation " + wait,
                 wait < (maxStaleSecs *1000));
      IndexSearcher searcher = sm.acquire();

View Full Code Here

    private void openIndex(String indexId) throws IOException {


        Directory cachedFSDir = null;
        IndexWriter writer = null;
        GeonetworkNRTManager nrtManager = null;
        TrackingIndexWriter trackingIndexWriter;
        try {
            cachedFSDir = _directoryFactory.createIndexDirectory(indexId, luceneConfig);
            IndexWriterConfig conf = new IndexWriterConfig(Geonet.LUCENE_VERSION, SearchManager.getAnalyzer(indexId, false));
            ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler();
            conf.setMergeScheduler(mergeScheduler);
            writer = new IndexWriter(cachedFSDir, conf);
            trackingIndexWriter = new TrackingIndexWriter(writer);
            nrtManager = new GeonetworkNRTManager(luceneConfig, indexId,
                    trackingIndexWriter, writer, null, true, taxonomyIndexTracker);
        } catch (CorruptIndexException e) {
            IOUtils.closeQuietly(nrtManager);
            IOUtils.closeQuietly(writer);

View Full Code Here


    if (VERBOSE) {
      System.out.println("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
    }


    genWriter = new TrackingIndexWriter(writer);


    final SearcherFactory sf = new SearcherFactory() {
        @Override
        public IndexSearcher newSearcher(IndexReader r) throws IOException {
          TestControlledRealTimeReopenThread.this.warmCalled = true;

View Full Code Here

0 1

TOP

Related Classes of org.apache.lucene.index.TrackingIndexWriter

org.apache.lucene.search.TestControlledRealTimeReopenThread

org.fao.geonet.kernel.search.index.LuceneIndexLanguageTracker

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.