// Copyright 2007 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.enterprise.connector.afyd;
import com.google.enterprise.connector.spi.Document;
import com.google.enterprise.connector.spi.DocumentList;
import com.google.enterprise.connector.spi.RepositoryException;
import com.google.gdata.data.Entry;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.logging.Logger;
/**
* This class is the stateless heart of the Afyd connector family. Given a list
* of users, the location of a properties file, and a provider, this class uses
* extreme laziness to traverse the entries available from a service.
*
* See the comments on traversalStep() for details.
*
* @author amsmith@google.com (Your Name Here)
*
*/
public class StatelessDocumentList implements DocumentList {
/** The logger for this class. */
private static final Logger LOGGER =
Logger.getLogger(StatelessDocumentList.class.getName());
/** Property key for storing checkpoints. Used as "checkpoint.foo" */
private static final String P_CHECKPOINT = "checkpoint";
/** Property key for storing the last user touched in the traversal. */
private static final String P_LAST_USER = "user";
/** An impossible user name to signal that traversal has just started. */
private static final String START_DUMMY = " START"; // starts with space
/** An impossible user name to signal that traversal has just finished. */
private static final String FINISH_DUMMY = " FINISH"; // starts with space
private List users;
private FeedEntryProvider provider;
private String propertiesFilename;
/**
* The properties object that is kept synced with the file named in the
* 'propertiesFilename' field.
*/
private Properties properties;
/**
* Name of user who's entry list is cached for use in traversalStep()
* (TRANSIENT)
*/
private String cachedUser;
/**
* A cached list of entries for use in traversalStep()
* (TRANSIENT)
*/
private List cachedEntries;
/**
* @param users List of String user names for users of the hosted domain that
* is naturally SORTED, ASCENDING.
* @param propertiesFilename The filename of a the properties file storing
* this connector's state.
* @param provider A source of ordered feed entries. These are the native
* "documents" that this document list will contain.
*/
public StatelessDocumentList( List users, String propertiesFilename,
FeedEntryProvider provider) throws RepositoryException {
this.users = users;
this.propertiesFilename = propertiesFilename;
this.provider = provider;
loadProperties();
}
public Document nextDocument() throws RepositoryException {
try {
Document doc = null;
do {
doc = traversalStep();
} while (doc == null);
return doc;
} catch (NoMoreStepsException nmse) {
// swallowing exception, this is expected to be thrown regularly
return null;
} catch (RepositoryException re) {
LOGGER.severe(re.toString());
throw re;
}
}
/**
* {@inheritDoc}
*
* This connector manages its own checkpoints internally however traversal
* state will be lost if this method is not called.
*/
public String checkpoint() throws RepositoryException {
storeProperties();
return "See " + propertiesFilename;
}
/**
* This method attempts a single step in the traversal of the logical list-
* of-lists that is made up of all entries for each user in checkpointed order
* over all users in user list order. Occasionally a step will not return a
* document because it was necessary to fetch more entries for a user or move
* on to the next user. In this case traversalStep() will return null,
* indicating that traversalStep() should be immediately called again to try
* to get the next document. When the traversal has actually completed (when
* the last document for the last user has been returned) traversalStep() will
* throw a NoMoreStepsException. The next call to traversalStep() after
* this exception will begin a new traversal.
*
* This complicated logic is used to allow the traversal to be effectively
* stateless (aside from the contents of properties which is backed by a file)
* meaning that the lifetime of the AfydConnector object does not effect the
* visible DocumentList.
*
* @return a Document if possible, null if should be called again
* @throws NoMoreStepsException when traversal has actually finished
* (distinct from simply needing more steps).
*/
private Document traversalStep()
throws RepositoryException, NoMoreStepsException {
// Recall what the last iteration thought that our current user should be
// (who may not be an active user any more) and abort if they hit the end.
String lastUser = properties.getProperty(P_LAST_USER, START_DUMMY);
if (lastUser.equals(FINISH_DUMMY)) {
properties.remove(P_LAST_USER); // start fresh next time
throw new NoMoreStepsException();
}
// Make sure we are working with an active user and abort if not.
String user = getClosestExistingUser(lastUser);
if (user == null) {
properties.remove(P_LAST_USER); // start fresh next time
throw new NoMoreStepsException();
}
// Recall their checkpoint string.
String userCheckpointKey = P_CHECKPOINT + "." + user;
String checkpoint = properties.getProperty(userCheckpointKey, null);
// Make sure their entries are cached (checkpoint ignored in this case).
if (cachedEntries == null || !user.equals(cachedUser)) {
cachedUser = user;
cachedEntries = provider.getOrderedEntriesForUser(user, checkpoint);
}
// Try to consume the first element of the list.
Entry entry = null;
if (cachedEntries.size() > 0) {
entry = (Entry) cachedEntries.get(0);
cachedEntries.remove(0);
}
// Check if we have finished all of the (cached) entries for this user.
// If entries have been added since the cache was stored these entries will
// not be visible until the next traversal begins (likewise for users that
// were recently added by have a name "before" the current one).
if (cachedEntries.size() == 0) {
// This means we have finished all of the entries for this user, move on.
cachedEntries = null;
user = getNextExistingUser(user);
if (user == null) {
// This will trigger NoMoreStepsException in next call so we can return
// an entry if we have one.
user = FINISH_DUMMY;
}
}
// Save any state changes.
if (entry != null) {
String updatedCheckpoint = provider.getCheckpointForEntry(entry);
properties.setProperty(userCheckpointKey, updatedCheckpoint);
}
properties.setProperty(P_LAST_USER, user);
// Make a document from the entry if we actually got one.
if (entry != null) {
return EntryDocumentizer.makeDocument(entry);
} else {
return null;
}
}
/**
* An exception used to indicate that no more steps are possible on the
* current traversal.
*/
static class NoMoreStepsException extends Exception {
// nothing special
}
/**
* Examines the users List and returns the target user if they are on the list
* or the next user in the list after where they would have been (lexically).
*/
private String getClosestExistingUser(String target) {
if (target.equals(START_DUMMY)) {
return users.size() > 0 ? (String) users.get(0) : null;
}
int index = Collections.binarySearch(users, target);
if (index >= 0) {
return (String) users.get(index);
} else {
int insertionIndex = -(index + 1);
if (insertionIndex < users.size()) {
return (String) users.get(insertionIndex);
} else {
return null;
}
}
}
/**
* Gets the first user that comes after the given user in the current user
* list. If the target exists, unlike getClosestExistingUser(), the target
* will NOT be returned, otherwise it is the same as getClosestExistingUser().
*/
private String getNextExistingUser(String target) {
int index = Collections.binarySearch(users, target);
if (index >= 0) {
if (index + 1 < users.size()) {
return (String) users.get(index + 1);
} else {
return null;
}
} else {
int insertionIndex = -(index + 1);
if (insertionIndex < users.size()) {
return (String) users.get(insertionIndex);
} else {
return null;
}
}
}
/**
* Populates (overwriting) the properties field of this class with data from
* the file named by propertiesFilename. If the named file does not exist,
* an empty properties object is used. The file is not created.
*/
private void loadProperties() throws RepositoryException {
properties = new Properties();
try {
properties.load(new FileInputStream(propertiesFilename));
} catch (FileNotFoundException fnfe){
// this is OK, even expected on first run
LOGGER.info(fnfe.toString());
} catch (IOException ioe) {
LOGGER.severe(ioe.toString());
throw new RepositoryException(ioe);
}
}
/**
* Commits the contents of the properties field of this class to the file
* named by the propertiesFilename. If the file does not exist, it is
* created.
*/
private void storeProperties() throws RepositoryException {
try {
properties.store(new FileOutputStream(propertiesFilename), null);
} catch (IOException ioe) {
LOGGER.severe(ioe.toString());
throw new RepositoryException(ioe);
}
}
}