Examples of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue$RecoveryFile

Package com.google.enterprise.connector.util.diffing

Examples of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue$RecoveryFile

com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue
A File that has some of the recovery logic. Original recovery files' names contained a single nanosecond timestamp, eg. recovery.10220010065599398 . These turned out to be flawed because nanosecond times can go "back in time" between JVM restarts. Updated recovery files' names contain a wall clock millis timestamp followed by an underscore followed by a nanotimestamp, eg. recovery.702522216012_10220010065599398 .

   * Tests the normal operation where a sequence of batches are processed to
   * completion in order. The first batch begins with a null checkpoint.
   */
  public void testStartResumeTraversal() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 2, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 4, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 0);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 0);
    assertFalse(q.resume(checkpoint).iterator().hasNext());
   }

View Full Code Here

   * Tests replaying the first batch which has a null checkpoint and then
   * resuming.
   */
  public void testRetryStartThenResume() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    String checkpoint = null;
    checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    // Replay the first batch.
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 2, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 4, 2);
    assertFalse(q.resume(checkpoint).iterator().hasNext());
  }

View Full Code Here

  /**
   * Tests retrying a batch after the first one.
   */
  public void testRetryThenResume() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    // We call q.resume but do not set checkpoint forward to reflect the
    // changes from the batch.
    checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 2, 2);
    // Since we have not advanced checkpoint we replay the batch.
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 2, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 4, 2);
    assertFalse(q.resume(checkpoint).iterator().hasNext());
  }

View Full Code Here

  /**
   * Tests resuming from a checkpoint that is half way through the first batch.
   */
  public void testHalfStartBatch() throws IOException {
    ChangeSource changeSource = new MockChangeSource(8);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(3);
    q.start(null);
    Iterator<CheckpointAndChange> it = q.resume(null).iterator();
    CheckpointAndChange checkpointAndChange = it.next();
    MockChangeSource.validateChange(0, checkpointAndChange.getChange());
    assertTrue(it.hasNext());
    checkpointAndChange = it.next();
    MockChangeSource.validateChange(1, checkpointAndChange.getChange());
    assertTrue(it.hasNext());
    String checkpoint = checkChangesAndReturnLastCheckpoint(
        q.resume(checkpointAndChange.getCheckpoint().toString()), 2, 3);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 5, 3);
    assertFalse(q.resume(checkpoint).iterator().hasNext());
  }

View Full Code Here

  /**
   * Tests resume for a null checkpoint when the {@link ChangeSource} is empty.
   */
  public void testStartWithEmptyChangeSource() throws IOException {
    ChangeSource changeSource = new MockChangeSource(0);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(3);
    q.start(null);
    Iterator<CheckpointAndChange> it = q.resume(null).iterator();
    assertFalse(it.hasNext());
  }

View Full Code Here

  /**
   * Tests resume for a second checkpoint when the {@link ChangeSource} is empty.
   */
  public void testResumeWithEmptyChangeSource() throws IOException {
    ChangeSource changeSource = new MockChangeSource(2);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    Iterator<CheckpointAndChange> it = q.resume(checkpoint).iterator();
    assertFalse(it.hasNext());
  }

View Full Code Here

   * Tests resume for a null checkpoint when the {@link ChangeSource} has half
   * the requested changes.
   */
  public void testStartWithPartialChangeSource() throws IOException {
    ChangeSource changeSource = new MockChangeSource(2);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(3);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    Iterator<CheckpointAndChange> it = q.resume(checkpoint).iterator();
    assertFalse(it.hasNext());
  }

View Full Code Here

   * Tests resume for a second checkpoint when the {@link ChangeSource} has half
   * the requested changes.
   */
  public void testResumeWithPartialChangeSource() throws IOException {
    ChangeSource changeSource = new MockChangeSource(5);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(3);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 3);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 3, 2);
    assertFalse(q.resume(checkpoint).iterator().hasNext());
  }

View Full Code Here

            MockChangeSource.newChange(1), null,
            MockChangeSource.newChange(2), null, null,
            MockChangeSource.newChange(3),
            MockChangeSource.newChange(4));
    ChangeSource changeSource = new MockChangeSource(changes);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(5);
    q.start(null);
    String checkpoint = null;
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 0, 2);
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 2, 1);
    Iterator<CheckpointAndChange> it = q.resume(checkpoint).iterator();
    assertFalse(it.hasNext());
    checkpoint = checkChangesAndReturnLastCheckpoint(q.resume(checkpoint), 3, 2);
    it = q.resume(checkpoint).iterator();
    assertFalse(it.hasNext());
  }

View Full Code Here

    return result;
  }


  public void testRecovery() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    List<CheckpointAndChange> firstBatch = q.resume(null);
    String checkpoint = firstBatch.get(1).getCheckpoint().toString();
    List<CheckpointAndChange> secondBatch = q.resume(checkpoint);


    CheckpointAndChangeQueue q2 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q2.setMaximumQueueSize(2);
    q2.start(checkpoint);
    List<CheckpointAndChange> secondBatchAgain = q2.resume(checkpoint);
    assertEquals(secondBatch, secondBatchAgain);
  }

View Full Code Here

0 1 2

TOP

Related Classes of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue$RecoveryFile

com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueueTest

com.google.enterprise.connector.util.diffing.testing.FakeDocumentSnapshotRepositoryMonitorManager

com.google.gson.JsonParser

com.google.gson.stream.JsonReader

org.json.JSONObject

org.json.JSONWriter

java.io.BufferedReader

java.io.OutputStreamWriter

java.io.Writer

java.io.InputStreamReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.