Examples of TrackingIndexWriter

org.apache.lucene.index.TrackingIndexWriter
Class that tracks changes to a delegated IndexWriter, used by {@link ControlledRealTimeReopenThread} to ensure specificchanges are visible. Create this class (passing your IndexWriter), and then pass this class to {@link ControlledRealTimeReopenThread}. Be sure to make all changes via the TrackingIndexWriter, otherwise {@link ControlledRealTimeReopenThread} won't know about the changes. @lucene.experimental
org.apache.lucene.search.NRTManager.TrackingIndexWriter

Examples of org.apache.lucene.index.TrackingIndexWriter

    Directory d = newDirectory();
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch signal = new CountDownLatch(1);


    LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
    final TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
    final SearcherManager manager = new SearcherManager(_writer, false, null);
    Document doc = new Document();
    doc.add(newTextField("test", "test", Field.Store.YES));
    writer.addDocument(doc);
    manager.maybeRefresh();
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          signal.await();
          manager.maybeRefresh();
          writer.deleteDocuments(new TermQuery(new Term("foo", "barista")));
          manager.maybeRefresh(); // kick off another reopen so we inc. the internal gen
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          latch.countDown(); // let the add below finish
        }
      }
    };
    t.start();
    _writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
    final long lastGen = writer.updateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen


    assertFalse(manager.isSearcherCurrent()); // false since there is a delete in the queue
    
    IndexSearcher searcher = manager.acquire();
    try {

View Full Code Here

Examples of org.apache.lucene.index.TrackingIndexWriter

                                                     new MockAnalyzer(random()));
    config.setIndexDeletionPolicy(sdp);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    final IndexWriter iw = new IndexWriter(dir, config);
    SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
    final TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
    ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread =
      new ControlledRealTimeReopenThread<>(tiw, sm, maxStaleSecs, 0);


    controlledRealTimeReopenThread.setDaemon(true);
    controlledRealTimeReopenThread.start();


    List<Thread> commitThreads = new ArrayList<>();


    for (int i = 0; i < 500; i++) {
      if (i > 0 && i % 50 == 0) {
        Thread commitThread =  new Thread(new Runnable() {
            @Override
            public void run() {
              try {
                iw.commit();
                IndexCommit ic = sdp.snapshot();
                for (String name : ic.getFileNames()) {
                  //distribute, and backup
                  //System.out.println(names);
                  assertTrue(slowFileExists(dir, name));
                }
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          });
        commitThread.start();
        commitThreads.add(commitThread);
      }
      Document d = new Document();
      d.add(new TextField("count", i + "", Field.Store.NO));
      d.add(new TextField("content", content, Field.Store.YES));
      long start = System.currentTimeMillis();
      long l = tiw.addDocument(d);
      controlledRealTimeReopenThread.waitForGeneration(l);
      long wait = System.currentTimeMillis() - start;
      assertTrue("waited too long for generation " + wait,
                 wait < (maxStaleSecs *1000));
      IndexSearcher searcher = sm.acquire();

View Full Code Here

Examples of org.apache.lucene.search.NRTManager.TrackingIndexWriter

    commitWriter = new IndexWriter(commitDirectory, conf);
    
    mainWriter = new IndexWriter(mainDirectory, conf);
    BufferStore.init(128, 128);
    
    indexImporter = new IndexImporter(new TrackingIndexWriter(mainWriter), new ReentrantReadWriteLock(), shardContext,
        TimeUnit.MINUTES, 10);
  }

View Full Code Here

Examples of org.apache.lucene.search.NRTManager.TrackingIndexWriter

      public IndexSearcher newSearcher(IndexReader reader) throws IOException {
        return new IndexSearcherClosableNRT(reader, searchExecutor, _nrtManagerRef, _directory);
      }
    };


    _trackingWriter = new TrackingIndexWriter(_writer);
    _indexImporter = new IndexImporter(_trackingWriter, _lock, _shardContext, TimeUnit.SECONDS, 10);
    _nrtManagerRef.set(new NRTManager(_trackingWriter, _searcherFactory, APPLY_ALL_DELETES));
    // start commiter


    _committer = new Thread(new Committer());

View Full Code Here

Examples of org.apache.lucene.search.NRTManager.TrackingIndexWriter


    Directory dir = newFSDirectory(_TestUtil.getTempDir("livefieldupdates"));
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));


    final IndexWriter _w = new IndexWriter(dir, iwc);
    final TrackingIndexWriter w = new TrackingIndexWriter(_w);


    final NRTManager mgr = new NRTManager(w, new SearcherFactory() {
        @Override
        public IndexSearcher newSearcher(IndexReader r) {
          return new IndexSearcher(r);
        }
      });


    final Integer missing = -1;


    final LiveFieldValues<Integer> rt = new LiveFieldValues<Integer>(mgr, missing) {
        @Override
        protected Integer lookupFromSearcher(IndexSearcher s, String id) throws IOException {
          TermQuery tq = new TermQuery(new Term("id", id));
          TopDocs hits = s.search(tq, 1);
          assertTrue(hits.totalHits <= 1);
          if (hits.totalHits == 0) {
            return null;
          } else {
            Document doc = s.doc(hits.scoreDocs[0].doc);
            return (Integer) doc.getField("field").numericValue();
          }
        }
    };


    int numThreads = _TestUtil.nextInt(random(), 2, 5);
    if (VERBOSE) {
      System.out.println(numThreads + " threads");
    }


    final CountDownLatch startingGun = new CountDownLatch(1);
    List<Thread> threads = new ArrayList<Thread>();


    final int iters = atLeast(1000);
    final int idCount = _TestUtil.nextInt(random(), 100, 10000);


    final double reopenChance = random().nextDouble()*0.01;
    final double deleteChance = random().nextDouble()*0.25;
    final double addChance = random().nextDouble()*0.5;
    
    for(int t=0;t<numThreads;t++) {
      final int threadID = t;
      final Random threadRandom = new Random(random().nextLong());
      Thread thread = new Thread() {


          @Override
          public void run() {
            try {
              Map<String,Integer> values = new HashMap<String,Integer>();
              List<String> allIDs = Collections.synchronizedList(new ArrayList<String>());


              startingGun.await();
              for(int iter=0; iter<iters;iter++) {
                // Add/update a document
                Document doc = new Document();
                // Threads must not update the same id at the
                // same time:
                if (threadRandom.nextDouble() <= addChance) {
                  String id = String.format(Locale.ROOT, "%d_%04x", threadID, threadRandom.nextInt(idCount));
                  Integer field = threadRandom.nextInt(Integer.MAX_VALUE);
                  doc.add(new StringField("id", id, Field.Store.YES));
                  doc.add(new IntField("field", field.intValue(), Field.Store.YES));
                  w.updateDocument(new Term("id", id), doc);
                  rt.add(id, field);
                  if (values.put(id, field) == null) {
                    allIDs.add(id);
                  }
                }


                if (allIDs.size() > 0 && threadRandom.nextDouble() <= deleteChance) {
                  String randomID = allIDs.get(threadRandom.nextInt(allIDs.size()));
                  w.deleteDocuments(new Term("id", randomID));
                  rt.delete(randomID);
                  values.put(randomID, missing);
                }


                if (threadRandom.nextDouble() <= reopenChance || rt.size() > 10000) {

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.