Examples of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue

Package com.google.enterprise.connector.util.diffing

Examples of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue

com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue
Queue of {@code CheckpointAndChange} objects to be processed by the{@link DiffingConnectorTraversalManager}. Objects are added to the queue from a {@link ChangeSource} and assigned a {@link DiffingConnectorCheckpoint}. The client accesses objects by calling {@link #resume(String)}. To facilitate retry of processing for objects in the queue {@code CheckpointAndChange} objects remain until the clientindicates they have completed processing by calling {@link #resume(String)}with the object's checkpoint or a later object's checkpoint. @since 2.8

    assertEquals(secondBatch, secondBatchAgain);
  }


  public void testRepeatedRecoveryAtSameCheckpoint() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    List<CheckpointAndChange> firstBatch = q.resume(null);
    String checkpoint = firstBatch.get(1).getCheckpoint().toString();
    List<CheckpointAndChange> secondBatch = q.resume(checkpoint);


    CheckpointAndChangeQueue q2 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q2.setMaximumQueueSize(2);
    q2.start(checkpoint);
    List<CheckpointAndChange> secondBatchAgain = q2.resume(checkpoint);
    assertEquals(secondBatch, secondBatchAgain);


    CheckpointAndChangeQueue q3 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory,clientFactory);
    q3.setMaximumQueueSize(2);
    q3.start(checkpoint);
    List<CheckpointAndChange> secondBatchThrice = q3.resume(checkpoint);
    assertEquals(secondBatch, secondBatchThrice);


    CheckpointAndChangeQueue q4 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q4.setMaximumQueueSize(2);
    q4.start(checkpoint);
    List<CheckpointAndChange> secondBatchFourthTime = q4.resume(checkpoint);
    assertEquals(secondBatch, secondBatchFourthTime);
  }

View Full Code Here

    assertEquals(secondBatch, secondBatchFourthTime);
  }


  public void testRepeatedResumeAtSameCheckpointOfSameQueue() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    List<CheckpointAndChange> firstBatch = q.resume(null);
    String checkpoint = firstBatch.get(1).getCheckpoint().toString();
    List<CheckpointAndChange> secondBatch = q.resume(checkpoint);


    CheckpointAndChangeQueue q2 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q2.setMaximumQueueSize(2);
    q2.start(checkpoint);
    List<CheckpointAndChange> secondBatchAgain = q2.resume(checkpoint);
    assertEquals(secondBatch, secondBatchAgain);


    List<CheckpointAndChange> secondBatchThrice = q2.resume(checkpoint);
    assertEquals(secondBatch, secondBatchThrice);
  }

View Full Code Here

    assertEquals(secondBatch, secondBatchThrice);
  }


  public void testPartialRecovery() throws IOException {
    ChangeSource changeSource = new MockChangeSource(10);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(4);
    q.start(null);
    List<CheckpointAndChange> firstBatch = q.resume(null);
    String checkpoint = firstBatch.get(3).getCheckpoint().toString();
    List<CheckpointAndChange> secondBatch = q.resume(checkpoint);
    checkpoint = secondBatch.get(1).getCheckpoint().toString();


    CheckpointAndChangeQueue q2 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q2.setMaximumQueueSize(4);
    q2.start(checkpoint);
    List<CheckpointAndChange> secondBatchRedoEnd = q2.resume(checkpoint);
    assertEquals(secondBatch.get(2), secondBatchRedoEnd.get(0));
    assertEquals(secondBatch.get(3), secondBatchRedoEnd.get(1));
  }

View Full Code Here

  private static final String EMPTY_RECOVERY_FILE =
      "{\"MON\":{},\"Q\":[]}SENTINAL";


  public void testWriteAndReadJson() throws Exception {
    ChangeSource changeSource = new MockChangeSource(10);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.start(null);
    assertEquals(EMPTY_RECOVERY_FILE, getRecoveryFile(q));
    List<CheckpointAndChange> firstBatch = q.resume(null);
    String original = getRecoveryFile(q);


    CheckpointAndChangeQueue q2 = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    assertEquals(EMPTY_RECOVERY_FILE, getRecoveryFile(q2));
    q2.new LoadingQueueReader().readJson(new StringReader(original));
    assertEquals(original, getRecoveryFile(q2));
  }

View Full Code Here

  }


  public void testRecoveryStateCleanup() throws IOException {
    final int NUM_RESUME_CALLS = 20;
    ChangeSource changeSource = new MockChangeSource(NUM_RESUME_CALLS * 3);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    String checkpoint = null;
    q.start(checkpoint);
    for (int i = 0; i < NUM_RESUME_CALLS; i++) {
       List<CheckpointAndChange> batch = q.resume(checkpoint);
       checkpoint = batch.get(1).getCheckpoint().toString();
       assertTrue(1 >= persistDir.listFiles().length);
    }
    assertTrue(1 == persistDir.listFiles().length);
  }

View Full Code Here

   * in call to clean().
   */
  public void testCleanPersistDirNotExist() throws Exception {
    final int NUM_RESUME_CALLS = 20;
    ChangeSource changeSource = new MockChangeSource(NUM_RESUME_CALLS * 3);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(null);
    assertTrue(persistDir.exists());
    assertTrue(1 >= persistDir.listFiles().length);


    // A call to clean() should delete the persistDir and all its files. 
    q.clean();
    assertFalse(persistDir.exists());
    assertNull(persistDir.listFiles());


    // A subsequent call to clean() should not throw NullPointerException.
    q.clean();
    assertFalse(persistDir.exists());
    assertNull(persistDir.listFiles());


    // Recreate persistDir for the benefit of tearDown().
    q.ensurePersistDirExists();
    assertTrue(persistDir.exists());
  }

View Full Code Here

    assertTrue(persistDir.exists());
  }


  public void testTooManyRecoveryFiles() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    {
      File persistFile = new File(persistDir, "recovery.1234");
      FileWriter writer = new FileWriter(persistFile);
      writer.write("omonee-harmony");
      writer.close();
    }
    {
      File persistFile = new File(persistDir, "recovery.90");
      FileWriter writer = new FileWriter(persistFile);
      writer.write("kwami.fitzpatrik");
      writer.close();
    }
    {
      File persistFile = new File(persistDir, "recovery.-123");
      FileWriter writer = new FileWriter(persistFile);
      writer.write("funny\nfarm\nman");
      writer.close();
    }
    q.setMaximumQueueSize(2);
    try {
      q.start(DiffingConnectorCheckpoint.newFirst().toString());
      fail("Should have failed on too many recovery files.");
    } catch(IOException e) {
      assertTrue(-1 != e.getMessage().indexOf("Found too many recovery files: "));
    }
  }

View Full Code Here

    }
  }


  public void testInvalidRecoveryFilename() throws IOException {
    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    File persistFile = new File(persistDir, "recovery.sugar-n-spice");
    FileWriter writer = new FileWriter(persistFile);
    writer.write("oh so not relevent");
    writer.close();
    q.setMaximumQueueSize(2);
    try {
      q.start(DiffingConnectorCheckpoint.newFirst().toString());
      fail("Should have failed on invalid recovery filename.");
    } catch(IOException e) {
      assertTrue(-1 != e.getMessage().indexOf("Invalid recovery filename: "));
    }
  }

View Full Code Here

    File persistDirA = testDirectoryManager.makeDirectory("queue-A");
    File persistDirB = testDirectoryManager.makeDirectory("queue-B");


    { /* Make recovery file that finishes 2nd batch. */
      ChangeSource changeSource = new MockChangeSource(6);
      CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
          persistDirA, internalFactory, clientFactory);
      q.setMaximumQueueSize(2);
      q.start(null);
      List<CheckpointAndChange> firstBatch = q.resume(null);
      String checkpoint = firstBatch.get(1).getCheckpoint().toString();
      List<CheckpointAndChange> secondBatch = q.resume(checkpoint);
      checkpoint = secondBatch.get(1).getCheckpoint().toString();
      q.resume(checkpoint);
      File recoveryFile = persistDirA.listFiles()[0];
      recoveryFile.renameTo(new File(persistDir, recoveryFile.getName()));
    }


    List<CheckpointAndChange> secondBatch;
    String checkpoint;
    { /* Make recovery file that finishes 1st batch. */
      ChangeSource changeSource = new MockChangeSource(6);
      CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
          persistDirB, internalFactory, clientFactory);
      q.setMaximumQueueSize(2);
      q.start(null);
      List<CheckpointAndChange> firstBatch = q.resume(null);
      checkpoint = firstBatch.get(1).getCheckpoint().toString();
      secondBatch = q.resume(checkpoint);
      File recoveryFile = persistDirB.listFiles()[0];
      recoveryFile.renameTo(new File(persistDir, recoveryFile.getName()));
    }


    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(checkpoint);
    List<CheckpointAndChange> secondBatchAgain = q.resume(checkpoint);
    assertEquals(secondBatch, secondBatchAgain);
    assertTrue(deleteDir(persistDirA));
    assertTrue(deleteDir(persistDirB));
  }

View Full Code Here


    String checkpoint;
    List<CheckpointAndChange> secondBatch;
    {
      ChangeSource changeSource = new MockChangeSource(6);
      CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
          persistDirAux, internalFactory, clientFactory);
      q.setMaximumQueueSize(2);
      q.start(null);
      List<CheckpointAndChange> firstBatch = q.resume(null);
      checkpoint = firstBatch.get(1).getCheckpoint().toString();
      secondBatch = q.resume(checkpoint);
      File recoveryFile = persistDirAux.listFiles()[0];
      recoveryFile.renameTo(new File(persistDir, recoveryFile.getName()));
    }


    File persistFile = new File(persistDir, "recovery." + System.nanoTime());
    FileWriter writer = new FileWriter(persistFile);
    writer.write("i iZ brokens\nyar\t\t\n?");
    writer.close();


    ChangeSource changeSource = new MockChangeSource(6);
    CheckpointAndChangeQueue q = new CheckpointAndChangeQueue(changeSource,
        persistDir, internalFactory, clientFactory);
    q.setMaximumQueueSize(2);
    q.start(checkpoint);
    List<CheckpointAndChange> secondBatchAgain = q.resume(checkpoint);
    assertEquals(secondBatch, secondBatchAgain);


    assertTrue(deleteDir(persistDirAux));
  }

View Full Code Here

0 1 2

TOP

Related Classes of com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueue

com.google.enterprise.connector.util.diffing.CheckpointAndChangeQueueTest

com.google.enterprise.connector.util.diffing.testing.FakeDocumentSnapshotRepositoryMonitorManager

com.google.gson.JsonParser

com.google.gson.stream.JsonReader

org.json.JSONObject

org.json.JSONWriter

java.io.BufferedReader

java.io.OutputStreamWriter

java.io.Writer

java.io.InputStreamReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.