Package com.google.enterprise.connector.spi

Examples of com.google.enterprise.connector.spi.DocumentList

Boundary cases are important for {@link #checkpoint()}: The typical pattern for consuming an object that implements this interface is as follows (disregarding exception handling):
 DocumentList docList = ... Document doc; while (doc = docList.nextDocument()) { handleDoc(doc); if (whatever reason) break; } String check = doclist.checkpoint(); 
Note: because of the restriction that the next call to {@link #nextDocument()} invalidates the previous Document, and thereare similar restrictions in the {@link Document} interface, it is possibleto provide a single stateful object that implements {@link DocumentList}, {@link Document} and {@link Property}, by returning {@code this}(or {@code null}) to all calls to {@link #nextDocument()} and{@link Document#findProperty(String)}. However, if preferred, the implementor may also use separate objects for each of those interfaces. @since 1.0

  @Test
  public void testStartTraversal() throws RepositoryException {
    TraversalManager trav = session.getTraversalManager();
    trav.setBatchHint(500);
    DocumentList docList = trav.startTraversal();
    assertNotNull(docList);
    Document doc = docList.nextDocument();
    assertNotNull(doc);
    while (doc != null) {
      Property propDocId = doc.findProperty(SpiConstants.PROPNAME_DOCID);
      assertNotNull(propDocId);
      assertTrue(propDocId.nextValue().toString().contains("social"));
      doc = docList.nextDocument();
    }
  }
View Full Code Here


        "MockRepositoryEventLog1.txt");
    MockRepository r = new MockRepository(mrel);
    QueryManager qm = new MockJcrQueryManager(r.getStore());
    TraversalManager qtm = new JcrTraversalManager(qm);
    MockPusher pusher = new MockPusher(System.out);
    DocumentList documentList = qtm.startTraversal();
    int counter = 0;
    Document document = null;
    while ((document = documentList.nextDocument()) != null) {
      pusher.take(document);
      counter++;
    }
    Assert.assertEquals(4, counter);
    Assert.assertEquals(4, pusher.getTotalDocs());
View Full Code Here

      return poll();
    }

    private DocumentList poll() {
      try {
        DocumentList result = traversalResults.poll(getPollTimeout(),
            TimeUnit.MILLISECONDS);
        if(result == null) {
          LOGGER.warning("poll returned null document.");
        }
        return result;
View Full Code Here

        traversalContext.traversalTimeLimitSeconds() * 1000 * 2;
    NeverEndingDocumentlistTraversalManager traversalManager =
        new NeverEndingDocumentlistTraversalManager(100) {
          @Override public DocumentList startTraversal() {
            clock.adjustTime(batchMillis);
            return new DocumentList() {
              public Document nextDocument() { throw new RuntimeException(); }
              public String checkpoint() { return "0"; }
            };
          }
        };
View Full Code Here

    final Traverser[] traverserHolder = new Traverser[1];
    NeverEndingDocumentlistTraversalManager traversalManager =
        new NeverEndingDocumentlistTraversalManager(100) {
          @Override public DocumentList startTraversal() {
            traverserHolder[0].cancelBatch();
            return new DocumentList() {
              public Document nextDocument() { throw new RuntimeException(); }
              public String checkpoint() { throw new RuntimeException(); }
            };
          }
        };
View Full Code Here

    } catch (javax.jcr.RepositoryException e) {
      throw new RepositoryException(e);
    }

    if (useThisNode) {
      DocumentList result = new JcrDocumentList(thisNode, nodes);
      return result;
    }

    return (nodes.hasNext()) ? new JcrDocumentList(nodes) : null;
  }
View Full Code Here

    {
      MockRepositoryDocument mockDoc = r.getStore().getDocByID("doc2");
      Document doc = new JcrDocument(new MockJcrNode(mockDoc));
      String checkpointString = JcrDocumentList.checkpoint(doc);
      DocumentList documentList = qtm.resumeTraversal(checkpointString);
      int counter = countDocuments(documentList);
      Assert.assertEquals(2, counter);
    }

    {
      MockRepositoryDocument mockDoc = r.getStore().getDocByID("doc4");
      Document doc = new JcrDocument(new MockJcrNode(mockDoc));
      String checkpointString = JcrDocumentList.checkpoint(doc);
      DocumentList documentList = qtm.resumeTraversal(checkpointString);
      int counter = countDocuments(documentList);
      Assert.assertEquals(0, counter);
    }
  }
View Full Code Here

        new MockRepositoryEventList("MockRepositoryEventLog1.txt");
    MockRepository r = new MockRepository(mrel);
    QueryManager qm = new MockJcrQueryManager(r.getStore());
    TraversalManager qtm = new JcrTraversalManager(qm);
    {
      DocumentList documentList = qtm.startTraversal();
      int counter = countDocuments(documentList);
      Assert.assertEquals(4, counter);
    }
  }
View Full Code Here

    runTraversal(null);
  }

  public void testInactive() {
    try {
      DocumentList docs = tm.startTraversal();
      tm.deactivate();
      docs.nextDocument();
      fail("DocumentList stayed active despite inactive TraversalManager.");
    } catch (RepositoryException re) {
      assertTrue(re.getMessage().contains(
          "Inactive FileTraversalManager referanced."));
    }
View Full Code Here

    }
  }

  private void runTraversal(String checkpoint) throws RepositoryException {
    monitorManager.getCheckpointAndChangeQueue().setMaximumQueueSize(BATCH_SIZE);
    DocumentList docs = null;
    if (checkpoint == null){
      docs =  tm.startTraversal();
      assertEquals(1, monitorManager.getStartCount());
      // start calls stop, in case things were running, because CM doesn't always.
      assertEquals(1, monitorManager.getStopCount());
      // clean is called after stop is called, for same reason as stop is called.
      assertEquals(1, monitorManager.getCleanCount());
      assertEquals(1, monitorManager.getGuaranteeCount());
    } else {
      docs = tm.resumeTraversal(checkpoint);
      assertEquals(1, monitorManager.getStartCount());
      // resume doesn't call stop.
      assertEquals(0, monitorManager.getStopCount());
      // Doesn't call clean.
      assertEquals(0, monitorManager.getCleanCount());
      assertEquals(1, monitorManager.getGuaranteeCount());
    }

    for (int k = 0; k < BATCH_SIZE; ++k) {
      Document doc = docs.nextDocument();
      assertNotNull(doc);
    }
    assertNull(docs.nextDocument());

    // TODO: Investigate: this loop looks weird cause it goes for BATCH_COUNT-1.
    for (int batch = 1; batch < BATCH_COUNT; batch++) {
      docs = tm.resumeTraversal(docs.checkpoint());
      assertEquals(1 + batch, monitorManager.getGuaranteeCount());
      for (int k = 0; k < BATCH_SIZE; ++k) {
        Document doc = docs.nextDocument();
        assertNotNull(doc);
        String docId = Value.getSingleValueString(doc, SpiConstants.PROPNAME_DOCID);
        assertEquals(String.format("/foo/bar/file.%d", batch * BATCH_SIZE + k),
            docId);
      }
      assertNull(docs.nextDocument());
    }

    docs = tm.resumeTraversal(docs.checkpoint());
    for (int k = 0; k < EXTRA; ++k) {
      assertNotNull(docs.nextDocument());
    }
    assertNull(docs.nextDocument());
  }
View Full Code Here

TOP

Related Classes of com.google.enterprise.connector.spi.DocumentList

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.