Examples of org.apache.lucene.index.SnapshotDeletionPolicy

org.apache.lucene.index.SnapshotDeletionPolicy
A {@link IndexDeletionPolicy} that wraps around any other{@link IndexDeletionPolicy} and adds the ability to hold andlater release a single "snapshot" of an index. While the snapshot is held, the {@link IndexWriter} will notremove any files associated with it even if the index is otherwise being actively, arbitrarily changed. Because we wrap another arbitrary {@link IndexDeletionPolicy}, this gives you the freedom to continue using whatever {@link IndexDeletionPolicy} you would normally want to use with yourindex. Note that you can re-use a single instance of SnapshotDeletionPolicy across multiple writers as long as they are against the same index Directory. Any snapshot held when a writer is closed will "survive" when the next writer is opened.
WARNING: This API is a new and experimental and may suddenly change.

  }


  public void testReuseAcrossWriters() throws Exception {
    Directory dir = new MockRAMDirectory();


    SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    IndexWriter writer = new IndexWriter(dir, true,new StandardAnalyzer(), dp);
    // Force frequent commits
    writer.setMaxBufferedDocs(2);
    Document doc = new Document();
    doc.add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
    for(int i=0;i<7;i++)
      writer.addDocument(doc);
    IndexCommit cp = (IndexCommit) dp.snapshot();
    copyFiles(dir, cp);
    writer.close();
    copyFiles(dir, cp);
    
    writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
    copyFiles(dir, cp);
    for(int i=0;i<7;i++)
      writer.addDocument(doc);
    copyFiles(dir, cp);
    writer.close();
    copyFiles(dir, cp);
    dp.release();
    writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
    writer.close();
    try {
      copyFiles(dir, cp);
      fail("did not hit expected IOException");

View Full Code Here


  private void runTest(Directory dir) throws Exception {
    // Run for ~7 seconds
    final long stopTime = System.currentTimeMillis() + 7000;


    SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    final IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);


    // Force frequent commits
    writer.setMaxBufferedDocs(2);

View Full Code Here

  }


  @Test
  public void testSnapshotLastCommitTwice() throws Exception {
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    writer.addDocument(new Document());
    writer.commit();
    
    String s1 = "s1";
    String s2 = "s2";
    IndexCommit ic1 = sdp.snapshot(s1);
    IndexCommit ic2 = sdp.snapshot(s2);
    assertTrue(ic1 == ic2); // should be the same instance
    
    // create another commit
    writer.addDocument(new Document());
    writer.commit();
    
    // release "s1" should not delete "s2"
    sdp.release(s1);
    writer.deleteUnusedFiles();
    checkSnapshotExists(dir, ic2);
    
    writer.close();
    dir.close();

View Full Code Here

  @Test
  public void testMissingCommits() throws Exception {
    // Tests the behavior of SDP when commits that are given at ctor are missing
    // on onInit().
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    writer.addDocument(new Document());
    writer.commit();
    IndexCommit ic = sdp.snapshot("s1");


    // create another commit, not snapshotted.
    writer.addDocument(new Document());
    writer.close();


    // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1"
    // commit.
    new IndexWriter(dir, getConfig(random, null)).close();
    
    assertFalse("snapshotted commit should not exist", dir.fileExists(ic.getSegmentsFileName()));
    
    // Now reinit SDP from the commits in the index - the snapshot id should not
    // exist anymore.
    sdp = getDeletionPolicy(sdp.getSnapshots());
    new IndexWriter(dir, getConfig(random, sdp)).close();
    
    try {
      sdp.getSnapshot("s1");
      fail("snapshot s1 should not exist");
    } catch (IllegalStateException e) {
      // expected.
    }
    dir.close();

View Full Code Here

  protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
    return getDeletionPolicy(null);
  }


  protected SnapshotDeletionPolicy getDeletionPolicy(Map<String, String> snapshots) throws IOException {
    return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots);
  }

View Full Code Here


  private void runTest(Random random, Directory dir) throws Exception {
    // Run for ~1 seconds
    final long stopTime = System.currentTimeMillis() + 1000;


    SnapshotDeletionPolicy dp = getDeletionPolicy();
    final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(dp)
        .setMaxBufferedDocs(2));
    writer.commit();

View Full Code Here


  
  @Test
  public void testBasicSnapshots() throws Exception {
    int numSnapshots = 3;
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    
    // Create 3 snapshots: snapshot0, snapshot1, snapshot2
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
    writer.close();
    
    assertSnapshotExists(dir, sdp, numSnapshots);


    // open a reader on a snapshot - should succeed.
    IndexReader.open(sdp.getSnapshot("snapshot0"), true).close();


    // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone.
    sdp = getDeletionPolicy();
    writer = new IndexWriter(dir, getConfig(random, sdp));
    writer.deleteUnusedFiles();
    writer.close();
    assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size());
    
    for (int i = 0; i < numSnapshots; i++) {
      try {
        sdp.getSnapshot("snapshot" + i);
        fail("snapshot shouldn't have existed, but did: snapshot" + i);
      } catch (IllegalStateException e) {
        // expected - snapshot should not exist
      }
    }

View Full Code Here

  }


  @Test
  public void testMultiThreadedSnapshotting() throws Exception {
    Directory dir = newDirectory();
    final SnapshotDeletionPolicy sdp = getDeletionPolicy();
    final IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));


    Thread[] threads = new Thread[10];
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new Thread() {
        @Override
        public void run() {
          try {
            writer.addDocument(new Document());
            writer.commit();
            sdp.snapshot(getName());
          } catch (Exception e) {
            throw new RuntimeException(e);
          }
        }
      };
      threads[i].setName("t" + i);
    }
    
    for (Thread t : threads) {
      t.start();
    }
    
    for (Thread t : threads) {
      t.join();
    }


    // Do one last commit, so that after we release all snapshots, we stay w/ one commit
    writer.addDocument(new Document());
    writer.commit();
    
    for (Thread t : threads) {
      sdp.release(t.getName());
      writer.deleteUnusedFiles();
    }
    assertEquals(1, IndexReader.listCommits(dir).size());
    writer.close();
    dir.close();

View Full Code Here


  @Test
  public void testRollbackToOldSnapshot() throws Exception {
    int numSnapshots = 2;
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
    writer.close();


    // now open the writer on "snapshot0" - make sure it succeeds
    writer = new IndexWriter(dir, getConfig(random, sdp).setIndexCommit(sdp.getSnapshot("snapshot0")));
    // this does the actual rollback
    writer.commit();
    writer.deleteUnusedFiles();
    assertSnapshotExists(dir, sdp, numSnapshots - 1);
    writer.close();
    
    // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
    String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
    assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
    dir.close();
  }

View Full Code Here

  }


  @Test
  public void testReleaseSnapshot() throws Exception {
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, 1, "snapshot");
    
    // Create another commit - we must do that, because otherwise the "snapshot"
    // files will still remain in the index, since it's the last commit.
    writer.addDocument(new Document());
    writer.commit();
    
    // Release
    String snapId = "snapshot0";
    String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName();
    sdp.release(snapId);
    try {
      sdp.getSnapshot(snapId);
      fail("should not have succeeded to get an unsnapshotted id");
    } catch (IllegalStateException e) {
      // expected
    }
    assertNull(sdp.getSnapshots().get(snapId));
    writer.deleteUnusedFiles();
    writer.close();
    assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName));
    dir.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.lucene.index.SnapshotDeletionPolicy

org.apache.blur.manager.writer.BlurNRTIndex

org.apache.lucene.index.TestSnapshotDeletionPolicy

org.apache.lucene.replicator.http.HttpReplicatorTest

org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest

org.apache.lucene.replicator.IndexAndTaxonomyRevision$SnapshotDirectoryTaxonomyWriter

org.apache.lucene.replicator.IndexAndTaxonomyRevisionTest

org.apache.lucene.replicator.IndexReplicationClientTest

org.apache.lucene.replicator.IndexRevisionTest

org.apache.lucene.replicator.LocalReplicatorTest

org.apache.lucene.replicator.SessionTokenTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.