/*
* Synchronizer.java
*
* Created on May 2, 2007, 8:49 AM
*
* To change this template, choose Tools | Template Manager
* and open the template in the editor.
*/
package org.atomojo.app.sync;
import java.io.IOException;
import java.net.URI;
import java.sql.SQLException;
import java.util.Date;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.atomojo.app.App;
import org.atomojo.app.AppException;
import org.atomojo.app.AtomResource;
import org.atomojo.app.Storage;
import org.atomojo.app.auth.AuthCredentials;
import org.atomojo.app.auth.User;
import org.atomojo.app.client.EntryCollection;
import org.atomojo.app.client.FeedClient;
import org.atomojo.app.client.FeedDestination;
import org.atomojo.app.client.IntrospectionClient;
import org.atomojo.app.client.XMLRepresentationParser;
import org.atomojo.app.db.DB;
import org.atomojo.app.db.DB.MediaEntryListener;
import org.atomojo.app.db.Entry;
import org.atomojo.app.db.EntryMedia;
import org.atomojo.app.db.Feed;
import org.atomojo.app.db.SyncProcess;
import org.infoset.xml.Attribute;
import org.infoset.xml.Document;
import org.infoset.xml.Element;
import org.infoset.xml.XMLException;
import org.restlet.Client;
import org.restlet.Context;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.ChallengeResponse;
import org.restlet.data.ChallengeScheme;
import org.restlet.data.Cookie;
import org.restlet.data.MediaType;
import org.restlet.data.Method;
import org.restlet.data.Protocol;
import org.restlet.data.Reference;
import org.restlet.data.Status;
import org.restlet.service.MetadataService;
/**
*
* @author alex
*/
public class PullSynchronizer implements Synchronizer
{
Logger log;
DB db;
Storage storage;
SyncProcess proc;
Date syncTime;
MetadataService metaService;
User user;
int errorCount;
boolean additive;
App app;
public PullSynchronizer(Logger log,MetadataService metaService,User user,DB db,Storage storage,SyncProcess proc)
{
this.db = db;
this.metaService = metaService;
this.storage = storage;
this.log = log;
this.user = user;
this.proc = proc;
this.syncTime = null;
this.errorCount = 0;
this.additive = true;
this.app = new App(log,db,storage,metaService);
}
public boolean isAdditive() {
return additive;
}
public void setAdditive(boolean flag) {
this.additive = flag;
}
public int getErrorCount() {
return errorCount;
}
public SyncProcess getProcess() {
return proc;
}
public void sync()
throws SyncException
{
errorCount = 0;
if (syncTime==null) {
syncTime = new Date();
}
if (proc.getRemoteApp()==null) {
throw new SyncException("The remote application cannot be found.");
}
String startPath = proc.getSyncTarget().getPath();
log.info("Starting introspection on "+proc.getRemoteApp().getIntrospection());
final URI root = proc.getRemoteApp().getRoot();
IntrospectionClient client = new IntrospectionClient(log,proc.getRemoteApp().getIntrospection());
final AuthCredentials auth = proc.getRemoteApp().getAuthCredentials();
if (auth!=null) {
if (auth.getScheme().equals("cookie")) {
log.info("Using cookie based authentication.");
Cookie cookie = new Cookie(auth.getName(),auth.getPassword());
cookie.setPath("/");
client.setCookie(cookie);
} else {
log.info("Using identity based authentication.");
client.setIdentity(auth.getName(),auth.getPassword());
}
}
final Set<String> paths = additive ? null : new TreeSet<String>();
try {
client.introspect(new IntrospectionClient.ServiceListener() {
XMLRepresentationParser parser = new XMLRepresentationParser();
int workspaceCount = 0;
public void onStartWorkspace(String title) {
// TODO: handle more than one workspace
workspaceCount++;
log.info("Workspace: "+workspaceCount);
}
public void onCollection(EntryCollection collection) {
if (workspaceCount!=1) {
// We only process the first workspace
return;
}
URI location = collection.getLocation();
log.info("Processing feed: "+location);
FeedClient feedClient = new FeedClient(location);
if (auth!=null) {
if (auth.getScheme().equals("cookie")) {
Cookie cookie = new Cookie(auth.getName(),auth.getPassword());
cookie.setPath("/");
feedClient.setCookie(cookie);
} else {
feedClient.setIdentity(auth.getName(),auth.getPassword());
}
}
try {
Response response = feedClient.get(new FeedDestination() {
Set<UUID> entries = additive ? null : new TreeSet<UUID>();
Feed feed = null;
public void onFeed(Document feedDoc) {
URI baseURI = feedDoc.getBaseURI();
// Make sure to remove the xml:base on the feed element for storage;
feedDoc.getDocumentElement().getAttributes().remove(Attribute.XML_BASE);
URI relative = root.relativize(baseURI).normalize();
if (relative.isAbsolute()) {
log.severe("Cannot make relative URI of '"+baseURI+"' using root '"+root+"'");
errorCount++;
return;
}
String fpath = relative.toString();
if (fpath.length()>0 && fpath.charAt(fpath.length()-1)!='/') {
int lastSlash = fpath.lastIndexOf('/');
fpath = fpath.substring(0,lastSlash+1);
}
log.info("Feed base URI='"+baseURI+"', relative='"+relative+"', feed path='"+fpath+"'");
if (!additive) {
String [] segments = fpath.split("\\/");
String current = null;
for (int i=0; i<segments.length; i++) {
if (current==null) {
if (segments[i].length()>0) {
current = segments[i]+"/";
} else {
current = segments[i];
}
} else {
current += segments[i]+"/";
}
log.info("Adding path: '"+current+"'");
paths.add(current);
}
}
try {
feed = app.createFeed(fpath,feedDoc);
} catch (AppException ex) {
if (ex.getStatus().getCode()==Status.CLIENT_ERROR_CONFLICT.getCode()) {
log.info(ex.getMessage());
log.info("Feed already exists, retrieving...");
try {
feed = app.getFeed(fpath);
} catch (AppException cex) {
log.log(Level.SEVERE,"Cannot get feed due to exception.",cex);
errorCount++;
}
try {
app.updateFeed(feed, feedDoc);
} catch (AppException updateEx) {
log.log(Level.SEVERE,"Cannot update feed due to exception.",updateEx);
errorCount++;
}
} else {
log.log(Level.SEVERE,"Failed to create feed due to exception.",ex);
errorCount++;
}
}
log.info("Feed ID: "+feed.getUUID()+", path='"+fpath+"'");
}
public void onEntry(Document entryDoc) {
if (feed==null) {
return;
}
entryDoc.getDocumentElement().localizeNamespaceDeclarations();
org.atomojo.app.client.Entry index = new org.atomojo.app.client.Entry(entryDoc);
index.index();
UUID entryId = null;
try {
String idS = index.getId();
if (idS==null) {
entryId = UUID.randomUUID();
index.setId("urn:uuid:"+entryId);
index.update();
} else {
entryId = UUID.fromString(idS.substring(9));
}
} catch (IllegalArgumentException ex) {
log.severe("Ignoring entry with bad UUID: "+index.getId());
errorCount++;
return;
}
log.info("Entry: "+entryId);
String src = null;
Element content = entryDoc.getDocumentElement().getFirstElementNamed(AtomResource.CONTENT_NAME);
URI baseURI = null;
MediaType contentType = null;
if (content!=null) {
src = content.getAttributeValue("src");
String type = content.getAttributeValue("type");
if (type!=null) {
contentType = MediaType.valueOf(type);
}
baseURI = content.getBaseURI();
}
if (entries!=null) {
entries.add(entryId);
}
Entry entry = null;
try {
entry = feed.findEntry(entryId);
} catch (SQLException ex) {
log.log(Level.SEVERE,"Cannot find entry "+entryId+" due to exception.",ex);
errorCount++;
}
EntryMedia resource = null;
boolean hasMedia = false;
if (entry!=null) {
try {
Iterator<EntryMedia> resources = entry.getResources();
if (resources.hasNext()) {
hasMedia = true;
while (resources.hasNext()) {
resource = resources.next();
if (!resource.getName().equals(src)) {
resource = null;
}
}
}
} catch (SQLException ex) {
log.log(Level.SEVERE,"Cannot enumerate entry "+index.getId()+" media due to exception.",ex);
errorCount++;
}
}
if (entry==null || (src!=null && !hasMedia) || (hasMedia && src==null)) {
if (entry!=null) {
// delete the entry because it changed to have a media or non-media content (rare)
try {
app.deleteEntry(feed,entry);
} catch (AppException ex) {
if (ex.getStatus()==Status.SERVER_ERROR_INTERNAL) {
log.log(Level.SEVERE,ex.getMessage(),ex);
} else {
log.severe("Status="+ex.getStatus().getCode()+", "+ex.getMessage());
}
errorCount++;
return;
}
}
if (src==null) {
// we have a regular entry
try {
app.createEntry(user,feed,entryDoc);
} catch (AppException ex) {
log.severe("Failed to create entry "+index.getId());
if (ex.getStatus()==Status.SERVER_ERROR_INTERNAL) {
log.log(Level.SEVERE,ex.getMessage(),ex);
} else {
log.severe("Status="+ex.getStatus().getCode()+", "+ex.getMessage());
}
errorCount++;
}
} else {
try {
EntryMedia media = feed.findEntryResource(src);
if (media!=null) {
// We have a conflicting media entry. We'll delete
// the local one to use the pulled one
Entry otherEntry = media.getEntry();
final String fpath = feed.getPath();
otherEntry.delete(new MediaEntryListener() {
public void onDelete(EntryMedia resource) {
try {
storage.deleteMedia(fpath,feed.getUUID(),resource.getName());
} catch (IOException ex) {
log.log(Level.SEVERE,"Cannot delete media "+resource.getName(),ex);
}
}
});
storage.deleteEntry(fpath,feed.getUUID(),otherEntry.getUUID());
}
} catch (SQLException ex) {
log.log(Level.SEVERE,"Database error while processing local media reference "+src,ex);
} catch (IOException ex) {
log.log(Level.SEVERE,"I/O error while deleting entry for media "+src,ex);
}
// we have a media entry
URI srcRef = baseURI.resolve(src);
Client client = new Client(new Context(log),Protocol.valueOf(srcRef.getScheme()));
client.getContext().getAttributes().put("hostnameVerifier", org.apache.commons.ssl.HostnameVerifier.DEFAULT);
Request request = new Request(Method.GET,new Reference(srcRef.toString()));
if (auth!=null) {
if (auth.getScheme().equals("cookie")) {
Cookie cookie = new Cookie(auth.getName(),auth.getPassword());
cookie.setPath("/");
request.getCookies().add(cookie);
} else {
request.setChallengeResponse(new ChallengeResponse(ChallengeScheme.HTTP_BASIC,auth.getName(),auth.getPassword()));
}
}
Response response = client.handle(request);
if (!response.getStatus().isSuccess()) {
log.log(Level.SEVERE,"Failed to retrieve media, status="+response.getStatus().getCode()+", src="+srcRef);
errorCount++;
return;
}
if (contentType!=null) {
// The entry's media type wins. Sometimes file resources do not
// report the media type correctly
response.getEntity().setMediaType(contentType);
}
try {
entry = app.createMediaEntry(user,feed,response.getEntity(),src,entryId);
app.updateEntry(user,feed,entry,entryDoc);
} catch (AppException ex) {
log.severe("Failed to create media entry "+index.getId()+", src="+srcRef);
if (ex.getStatus()==Status.SERVER_ERROR_INTERNAL) {
log.log(Level.SEVERE,ex.getMessage(),ex);
} else {
log.severe("Status="+ex.getStatus().getCode()+", "+ex.getMessage());
}
errorCount++;
}
}
} else {
try {
app.updateEntry(user,feed,entryId,entryDoc);
} catch (AppException ex) {
if (ex.getStatus()==Status.SERVER_ERROR_INTERNAL) {
log.log(Level.SEVERE,ex.getMessage(),ex);
} else {
log.severe("Status="+ex.getStatus().getCode()+", "+ex.getMessage());
}
errorCount++;
return;
}
if (src!=null) {
URI srcRef = baseURI.resolve(src);
Client client = new Client(new Context(log),Protocol.valueOf(srcRef.getScheme()));
client.getContext().getAttributes().put("hostnameVerifier", org.apache.commons.ssl.HostnameVerifier.DEFAULT);
/*
Request headRequest = new Request(Method.HEAD,new Reference(srcRef.toString()));
if (auth!=null) {
headRequest.setChallengeResponse(new ChallengeResponse(ChallengeScheme.HTTP_BASIC,auth.getName(),auth.getPassword()));
}
Response headResponse = client.handle(headRequest);
if (!headResponse.getStatus().isSuccess()) {
log.log(Level.SEVERE,"Failed to retrieve media head, status="+headResponse.getStatus().getCode()+", src="+srcRef);
errorCount++;
return;
}
boolean outOfDate = true;
if (headResponse.isEntityAvailable()) {
outOfDate = headResponse.getEntity().getModificationDate().after(resource.getEdited());
if (outOfDate) {
log.info("Out of date: "+headResponse.getEntity().getModificationDate()+" > "+resource.getEdited());
}
}
*/
Request request = new Request(Method.GET,new Reference(srcRef.toString()));
if (auth!=null) {
if (auth.getScheme().equals("cookie")) {
Cookie cookie = new Cookie(auth.getName(),auth.getPassword());
cookie.setPath("/");
request.getCookies().add(cookie);
} else {
request.setChallengeResponse(new ChallengeResponse(ChallengeScheme.HTTP_BASIC,auth.getName(),auth.getPassword()));
}
}
Date edited = new Date(resource.getEdited().getTime());
request.getConditions().setUnmodifiedSince(edited);
log.info("Attempting update media from "+srcRef.toString()+", edited="+edited);
Response response = client.handle(request);
if (response.getStatus().getCode()==304) {
log.info("No change (304)");
return;
} else if (!response.getStatus().isSuccess()) {
log.log(Level.SEVERE,"Failed to retrieve media, status="+response.getStatus().getCode()+", src="+srcRef);
errorCount++;
return;
}
if (contentType!=null) {
// The entry's media type wins. Sometimes file resources do not
// report the media type correctly
response.getEntity().setMediaType(contentType);
}
try {
app.updateMedia(feed,src,response.getEntity());
} catch (AppException ex) {
if (ex.getStatus()==Status.SERVER_ERROR_INTERNAL) {
log.log(Level.SEVERE,ex.getMessage(),ex);
} else {
log.severe("Failed to update media entry "+index.getId()+", src="+srcRef);
log.severe("Status="+ex.getStatus().getCode()+", "+ex.getMessage());
}
errorCount++;
}
}
}
}
public void onEnd() {
if (additive) {
return;
}
if (feed!=null) {
log.info("Removing extra entries...");
try {
Iterator<Entry> feedEntries = feed.getEntries();
while (feedEntries.hasNext()) {
Entry entry = feedEntries.next();
//log.info(entry.getUUID()+"?");
if (!entries.contains(entry.getUUID())) {
log.info("Delete ");
app.deleteEntry(feed, entry);
}
}
} catch (Exception ex) {
log.log(Level.SEVERE,"Error while checking for extra entries for feed "+(feed==null ? "" : feed.getUUID().toString()),ex);
}
}
}
});
if (!response.getStatus().isSuccess()) {
errorCount++;
log.severe("Can't get feed for location "+location+", http status="+response.getStatus().getCode());
}
} catch (IOException ex) {
log.log(Level.SEVERE,"I/O error while processing location "+location,ex);
errorCount++;
} catch (XMLException ex) {
log.log(Level.SEVERE,"XML error while processing location "+location,ex);
errorCount++;
}
}
public void onEndWorkspace() {
log.info("Workspace ended.");
if (!additive) {
log.info("Removing extra feeds.");
try {
Feed root = db.getRoot();
if (root!=null) {
checkFeedInSync(root,paths);
}
} catch (Exception ex) {
log.log(Level.SEVERE,"Error while processing additive=false check.",ex);
errorCount++;
}
}
}
});
} catch (IOException ex) {
log.log(Level.SEVERE,"I/O error during introspection.",ex);
errorCount++;
} catch (XMLException ex) {
log.log(Level.SEVERE,"XML error during introspection.",ex);
errorCount++;
}
log.info("Finished pull synchronization, errors="+errorCount);
}
protected void checkFeedInSync(Feed feed,Set<String> paths)
throws SQLException,AppException
{
String path = feed.getPath();
if (!paths.contains(path)) {
log.info("Extra: '"+path+"'");
app.delete(feed);
return;
}
Iterator<Feed> children = feed.getChildren();
while (children.hasNext()) {
checkFeedInSync(children.next(),paths);
}
}
protected void syncFeed(Feed feed,URI location)
{
}
public Date getSynchronizedAt() {
return syncTime;
}
public void setSynchronizationAt(Date time) {
syncTime = time;
}
}