/* $Id: SharePointRepository.java 996524 2010-09-13 13:38:01Z kwright $ */
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.connectors.sharepoint;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.crawler.system.Logging;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
import org.apache.manifoldcf.core.common.*;
import org.apache.manifoldcf.core.extmimemap.ExtensionMimeMap;
import org.apache.manifoldcf.core.util.URLEncoder;
import org.apache.manifoldcf.core.util.URLDecoder;
import java.io.*;
import java.util.Date;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Locale;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
import java.util.concurrent.TimeUnit;
import java.net.*;
import org.apache.log4j.Logger;
import org.apache.log4j.Level;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.NTCredentials;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
/** This is the "repository connector" for Microsoft SharePoint.
* Document identifiers for this connector come in three forms:
* (1) An "S" followed by the encoded subsite/library path, which represents the encoded relative path from the root site to a library. [deprecated and no longer supported];
* (2) A "D" followed by a subsite/library/folder/file path, which represents the relative path from the root site to a file. [deprecated and no longer supported]
* (3) Six different kinds of unencoded path, each of which starts with a "/" at the beginning, where the "/" represents the root site of the connection, as follows:
* /sitepath/ - the relative path to a site. The path MUST both begin and end with a single "/".
* /sitepath/libraryname// - the relative path to a library. The path MUST begin with a single "/" and end with "//".
* /sitepath/libraryname//folderfilepath - the relative path to a file. The path MUST begin with a single "/" and MUST include a "//" after the library, and must NOT end with a "/".
* /sitepath/listname/// - the relative path to a list. The path MUST begin with a single "/" and end with "///".
* /sitepath/listname///rowid - the relative path to a list item. The path MUST begin with a single "/" and MUST include a "///" after the list name, and must NOT end in a "/".
* /sitepath/listname///rowid//attachment_filename - the relative path to a list attachment. The path MUST begin with a single "/", MUST include a "///" after the list name, and
* MUST include a "//" separating the rowid from the filename.
*/
public class SharePointRepository extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
{
public static final String _rcsid = "@(#)$Id: SharePointRepository.java 996524 2010-09-13 13:38:01Z kwright $";
// Properties we need
public final static String wsddPathProperty = "org.apache.manifoldcf.sharepoint.wsddpath";
// Activities we log
public final static String ACTIVITY_FETCH = "fetch";
protected final static long sessionExpirationInterval = 300000L;
private boolean supportsItemSecurity = false;
private boolean dspStsWorks = true;
private boolean attachmentsSupported = false;
private boolean activeDirectoryAuthority = true;
private String serverProtocol = null;
private String serverUrl = null;
private String fileBaseUrl = null;
private String userName = null;
private String strippedUserName = null;
private String password = null;
private String ntlmDomain = null;
private String serverName = null;
private String serverLocation = null;
private String encodedServerLocation = null;
private int serverPort = -1;
private SPSProxyHelper proxy = null;
private long sessionTimeout;
// SSL support
private String keystoreData = null;
private IKeystoreManager keystoreManager = null;
private HttpClientConnectionManager connectionManager = null;
private HttpClient httpClient = null;
// Current host name
private static String currentHost = null;
static
{
// Find the current host name
try
{
java.net.InetAddress addr = java.net.InetAddress.getLocalHost();
// Get hostname
currentHost = addr.getHostName();
}
catch (UnknownHostException e)
{
}
}
// Turn off AXIS debug output that we don't want
static
{
Logger logger = Logger.getLogger("org.apache.axis.ConfigurationException");
logger.setLevel(Level.INFO);
}
/** Deny access token for default authority */
private final static String defaultAuthorityDenyToken = GLOBAL_DENY_TOKEN;
/** Constructor.
*/
public SharePointRepository()
{
}
/** Set up a session */
protected void getSession()
throws ManifoldCFException
{
if (proxy == null)
{
String serverVersion = params.getParameter( SharePointConfig.PARAM_SERVERVERSION );
if (serverVersion == null)
serverVersion = "2.0";
supportsItemSecurity = !serverVersion.equals("2.0");
dspStsWorks = serverVersion.equals("2.0") || serverVersion.equals("3.0");
attachmentsSupported = !serverVersion.equals("2.0");
String authorityType = params.getParameter( SharePointConfig.PARAM_AUTHORITYTYPE );
if (authorityType == null)
authorityType = "ActiveDirectory";
activeDirectoryAuthority = authorityType.equals("ActiveDirectory");
serverProtocol = params.getParameter( SharePointConfig.PARAM_SERVERPROTOCOL );
if (serverProtocol == null)
serverProtocol = "http";
try
{
String serverPort = params.getParameter( SharePointConfig.PARAM_SERVERPORT );
if (serverPort == null || serverPort.length() == 0)
{
if (serverProtocol.equals("https"))
this.serverPort = 443;
else
this.serverPort = 80;
}
else
this.serverPort = Integer.parseInt(serverPort);
}
catch (NumberFormatException e)
{
throw new ManifoldCFException(e.getMessage(),e);
}
serverLocation = params.getParameter(SharePointConfig.PARAM_SERVERLOCATION);
if (serverLocation == null)
serverLocation = "";
if (serverLocation.endsWith("/"))
serverLocation = serverLocation.substring(0,serverLocation.length()-1);
if (serverLocation.length() > 0 && !serverLocation.startsWith("/"))
serverLocation = "/" + serverLocation;
encodedServerLocation = serverLocation;
serverLocation = decodePath(serverLocation);
userName = params.getParameter(SharePointConfig.PARAM_SERVERUSERNAME);
password = params.getObfuscatedParameter(SharePointConfig.PARAM_SERVERPASSWORD);
int index = userName.indexOf("\\");
if (index != -1)
{
strippedUserName = userName.substring(index+1);
ntlmDomain = userName.substring(0,index);
}
else
{
strippedUserName = null;
ntlmDomain = null;
}
serverUrl = serverProtocol + "://" + serverName;
if (serverProtocol.equals("https"))
{
if (serverPort != 443)
serverUrl += ":" + Integer.toString(serverPort);
}
else
{
if (serverPort != 80)
serverUrl += ":" + Integer.toString(serverPort);
}
fileBaseUrl = serverUrl + encodedServerLocation;
// Set up ssl if indicated
keystoreData = params.getParameter(SharePointConfig.PARAM_SERVERKEYSTORE);
int connectionTimeout = 60000;
int socketTimeout = 900000;
connectionManager = new PoolingHttpClientConnectionManager();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
SSLConnectionSocketFactory myFactory = null;
if (keystoreData != null)
{
keystoreManager = KeystoreManagerFactory.make("",keystoreData);
myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
}
if (strippedUserName != null)
{
credentialsProvider.setCredentials(
new AuthScope(serverName,serverPort),
new NTCredentials(strippedUserName, password, currentHost, ntlmDomain));
}
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
.setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
.setDefaultSocketConfig(SocketConfig.custom()
.setTcpNoDelay(true)
.setSoTimeout(socketTimeout)
.build())
.setDefaultCredentialsProvider(credentialsProvider);
if (myFactory != null)
builder.setSSLSocketFactory(myFactory);
builder.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy());
httpClient = builder.build();
proxy = new SPSProxyHelper( serverUrl, encodedServerLocation, serverLocation, userName, password,
org.apache.manifoldcf.core.common.CommonsHTTPSender.class, "client-config.wsdd",
httpClient );
}
sessionTimeout = System.currentTimeMillis() + sessionExpirationInterval;
}
protected void expireSession()
throws ManifoldCFException
{
serverUrl = null;
fileBaseUrl = null;
userName = null;
strippedUserName = null;
password = null;
ntlmDomain = null;
serverLocation = null;
encodedServerLocation = null;
serverPort = -1;
keystoreData = null;
keystoreManager = null;
proxy = null;
httpClient = null;
if (connectionManager != null)
connectionManager.shutdown();
connectionManager = null;
}
/** Return the list of activities that this connector supports (i.e. writes into the log).
*@return the list.
*/
@Override
public String[] getActivitiesList()
{
return new String[]{ACTIVITY_FETCH};
}
/** Connect.
*@param configParameters is the set of configuration parameters, which
* in this case describe the root directory.
*/
@Override
public void connect(ConfigParams configParameters)
{
super.connect(configParameters);
// This is needed by getBins()
serverName = configParameters.getParameter( SharePointConfig.PARAM_SERVERNAME );
}
/** Close the connection. Call this before discarding the repository connector.
*/
@Override
public void disconnect()
throws ManifoldCFException
{
serverUrl = null;
fileBaseUrl = null;
userName = null;
strippedUserName = null;
password = null;
ntlmDomain = null;
serverName = null;
serverLocation = null;
encodedServerLocation = null;
serverPort = -1;
keystoreData = null;
keystoreManager = null;
proxy = null;
httpClient = null;
if (connectionManager != null)
connectionManager.shutdown();
connectionManager = null;
super.disconnect();
}
/** Get the bin name string for a document identifier. The bin name describes the queue to which the
* document will be assigned for throttling purposes. Throttling controls the rate at which items in a
* given queue are fetched; it does not say anything about the overall fetch rate, which may operate on
* multiple queues or bins.
* For example, if you implement a web crawler, a good choice of bin name would be the server name, since
* that is likely to correspond to a real resource that will need real throttle protection.
*@param documentIdentifier is the document identifier.
*@return the bin name.
*/
@Override
public String[] getBinNames(String documentIdentifier)
{
return new String[]{serverName};
}
/** Get the maximum number of documents to amalgamate together into one batch, for this connector.
*@return the maximum number. 0 indicates "unlimited".
*/
@Override
public int getMaxDocumentRequest()
{
// Since we went to a carrydown-based implementation, having this greater than 1 does not help.
return 1;
}
/** Test the connection. Returns a string describing the connection integrity.
*@return the connection's status as a displayable string.
*/
@Override
public String check()
throws ManifoldCFException
{
getSession();
try
{
URL urlServer = new URL( serverUrl );
}
catch ( MalformedURLException e )
{
return "Illegal SharePoint url: "+e.getMessage();
}
try
{
proxy.checkConnection( "/", supportsItemSecurity );
}
catch ( ServiceInterruption e )
{
return "SharePoint temporarily unavailable: "+e.getMessage();
}
catch (ManifoldCFException e)
{
return e.getMessage();
}
return super.check();
}
/** This method is periodically called for all connectors that are connected but not
* in active use.
*/
@Override
public void poll()
throws ManifoldCFException
{
if (proxy != null && System.currentTimeMillis() >= sessionTimeout)
expireSession();
if (connectionManager != null)
connectionManager.closeIdleConnections(60000L,TimeUnit.MILLISECONDS);
}
/** This method is called to assess whether to count this connector instance should
* actually be counted as being connected.
*@return true if the connector instance is actually connected.
*/
@Override
public boolean isConnected()
{
return connectionManager != null;
}
/** Request arbitrary connector information.
* This method is called directly from the API in order to allow API users to perform any one of several connector-specific
* queries.
*@param output is the response object, to be filled in by this method.
*@param command is the command, which is taken directly from the API request.
*@return true if the resource is found, false if not. In either case, output may be filled in.
*/
@Override
public boolean requestInfo(Configuration output, String command)
throws ManifoldCFException
{
if (command.startsWith("fields/"))
{
String library;
String sitePath;
String remainder = command.substring("fields/".length());
try
{
int index = remainder.indexOf("/");
if (index == -1)
{
library = remainder;
sitePath = "";
}
else
{
library = remainder.substring(0,index);
sitePath = remainder.substring(index+1);
}
Map<String,String> fieldSet = getLibFieldList(sitePath,library);
Iterator<String> iter = fieldSet.keySet().iterator();
while (iter.hasNext())
{
String fieldName = iter.next();
String displayName = fieldSet.get(fieldName);
ConfigurationNode node = new ConfigurationNode("field");
ConfigurationNode child;
child = new ConfigurationNode("name");
child.setValue(fieldName);
node.addChild(node.getChildCount(),child);
child = new ConfigurationNode("display_name");
child.setValue(displayName);
node.addChild(node.getChildCount(),child);
output.addChild(output.getChildCount(),node);
}
}
catch (ServiceInterruption e)
{
ManifoldCF.createServiceInterruptionNode(output,e);
}
catch (ManifoldCFException e)
{
ManifoldCF.createErrorNode(output,e);
}
}
else if (command.startsWith("listfields/"))
{
String listName;
String sitePath;
String remainder = command.substring("listfields/".length());
try
{
int index = remainder.indexOf("/");
if (index == -1)
{
listName = remainder;
sitePath = "";
}
else
{
listName = remainder.substring(0,index);
sitePath = remainder.substring(index+1);
}
Map<String,String> fieldSet = getListFieldList(sitePath,listName);
Iterator<String> iter = fieldSet.keySet().iterator();
while (iter.hasNext())
{
String fieldName = iter.next();
String displayName = fieldSet.get(fieldName);
ConfigurationNode node = new ConfigurationNode("field");
ConfigurationNode child;
child = new ConfigurationNode("name");
child.setValue(fieldName);
node.addChild(node.getChildCount(),child);
child = new ConfigurationNode("display_name");
child.setValue(displayName);
node.addChild(node.getChildCount(),child);
output.addChild(output.getChildCount(),node);
}
}
catch (ServiceInterruption e)
{
ManifoldCF.createServiceInterruptionNode(output,e);
}
catch (ManifoldCFException e)
{
ManifoldCF.createErrorNode(output,e);
}
}
else if (command.startsWith("sites/"))
{
try
{
String sitePath = command.substring("sites/".length());
List<NameValue> sites = getSites(sitePath);
int i = 0;
while (i < sites.size())
{
NameValue site = sites.get(i++);
ConfigurationNode node = new ConfigurationNode("site");
ConfigurationNode child;
child = new ConfigurationNode("name");
child.setValue(site.getValue());
node.addChild(node.getChildCount(),child);
child = new ConfigurationNode("display_name");
child.setValue(site.getPrettyName());
node.addChild(node.getChildCount(),child);
output.addChild(output.getChildCount(),node);
}
}
catch (ServiceInterruption e)
{
ManifoldCF.createServiceInterruptionNode(output,e);
}
catch (ManifoldCFException e)
{
ManifoldCF.createErrorNode(output,e);
}
}
else if (command.startsWith("libraries/"))
{
try
{
String sitePath = command.substring("libraries/".length());
List<NameValue> libs = getDocLibsBySite(sitePath);
int i = 0;
while (i < libs.size())
{
NameValue lib = libs.get(i++);
ConfigurationNode node = new ConfigurationNode("library");
ConfigurationNode child;
child = new ConfigurationNode("name");
child.setValue(lib.getValue());
node.addChild(node.getChildCount(),child);
child = new ConfigurationNode("display_name");
child.setValue(lib.getPrettyName());
node.addChild(node.getChildCount(),child);
output.addChild(output.getChildCount(),node);
}
}
catch (ServiceInterruption e)
{
ManifoldCF.createServiceInterruptionNode(output,e);
}
catch (ManifoldCFException e)
{
ManifoldCF.createErrorNode(output,e);
}
}
else if (command.startsWith("lists/"))
{
try
{
String sitePath = command.substring("lists/".length());
List<NameValue> libs = getListsBySite(sitePath);
int i = 0;
while (i < libs.size())
{
NameValue lib = libs.get(i++);
ConfigurationNode node = new ConfigurationNode("list");
ConfigurationNode child;
child = new ConfigurationNode("name");
child.setValue(lib.getValue());
node.addChild(node.getChildCount(),child);
child = new ConfigurationNode("display_name");
child.setValue(lib.getPrettyName());
node.addChild(node.getChildCount(),child);
output.addChild(output.getChildCount(),node);
}
}
catch (ServiceInterruption e)
{
ManifoldCF.createServiceInterruptionNode(output,e);
}
catch (ManifoldCFException e)
{
ManifoldCF.createErrorNode(output,e);
}
}
else
return super.requestInfo(output,command);
return true;
}
/** Queue "seed" documents. Seed documents are the starting places for crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in ISeedingActivity object.
*
* This method can choose to find repository changes that happen only during the specified time interval.
* The seeds recorded by this method will be viewed by the framework based on what the
* getConnectorModel() method returns.
*
* It is not a big problem if the connector chooses to create more seeds than are
* strictly necessary; it is merely a question of overall work required.
*
* The times passed to this method may be interpreted for greatest efficiency. The time ranges
* any given job uses with this connector will not overlap, but will proceed starting at 0 and going
* to the "current time", each time the job is run. For continuous crawling jobs, this method will
* be called once, when the job starts, and at various periodic intervals as the job executes.
*
* When a job's specification is changed, the framework automatically resets the seeding start time to 0. The
* seeding start time may also be set to 0 on each job run, depending on the connector model returned by
* getConnectorModel().
*
* Note that it is always ok to send MORE documents rather than less to this method.
*@param activities is the interface this method should use to perform whatever framework actions are desired.
*@param spec is a document specification (that comes from the job).
*@param startTime is the beginning of the time range to consider, inclusive.
*@param endTime is the end of the time range to consider, exclusive.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*/
@Override
public void addSeedDocuments(ISeedingActivity activities, DocumentSpecification spec,
long startTime, long endTime, int jobMode)
throws ManifoldCFException, ServiceInterruption
{
// Check the session
getSession();
// Add just the root.
activities.addSeedDocument("/");
}
/** Get document versions given an array of document identifiers.
* This method is called for EVERY document that is considered. It is
* therefore important to perform as little work as possible here.
*@param documentIdentifiers is the array of local document identifiers, as understood by this connector.
*@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers.
* A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document
* had an empty version string.
*@param activities is the interface this method should use to perform whatever framework actions are desired.
*@param spec is the current document specification for the current job. If there is a dependency on this
* specification, then the version string should include the pertinent data, so that reingestion will occur
* when the specification changes. This is primarily useful for metadata.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
*@return the corresponding version strings, with null in the places where the document no longer exists.
* Empty version strings indicate that there is no versioning ability for the corresponding document, and the document
* will always be processed.
*/
@Override
public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
getSession();
// Get the forced acls. (We need this only for the case where documents have their own acls)
String[] forcedAcls = getAcls(spec);
// Look at the metadata attributes.
// So that the version strings are comparable, we will put them in an array first, and sort them.
String pathAttributeName = null;
MatchMap matchMap = new MatchMap();
int i = 0;
while (i < spec.getChildCount())
{
SpecificationNode n = spec.getChild(i++);
if (n.getType().equals("pathnameattribute"))
pathAttributeName = n.getAttributeValue("value");
else if (n.getType().equals("pathmap"))
{
// Path mapping info also needs to be looked at, because it affects what is
// ingested.
String pathMatch = n.getAttributeValue("match");
String pathReplace = n.getAttributeValue("replace");
matchMap.appendMatchPair(pathMatch,pathReplace);
}
}
// Calculate the part of the version string that comes from path name and mapping.
// This starts with = since ; is used by another optional component (the forced acls)
StringBuilder pathNameAttributeVersion = new StringBuilder();
if (pathAttributeName != null)
pathNameAttributeVersion.append("=").append(pathAttributeName).append(":").append(matchMap);
String[] rval = new String[documentIdentifiers.length];
i = 0;
while (i < rval.length)
{
// Check if we should abort
activities.checkJobStillActive();
String documentIdentifier = documentIdentifiers[i];
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Getting version of '" + documentIdentifier + "'");
if ( documentIdentifier.startsWith("D") || documentIdentifier.startsWith("S") )
{
// Old-style document identifier. We don't recognize these anymore, so signal deletion.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Removing old-style document identifier '"+documentIdentifier+"'");
rval[i] = null;
}
else if (documentIdentifier.startsWith("/"))
{
// New-style document identifier. A double-slash marks the separation between the library and folder/file levels.
// A triple-slash marks the separation between a list name and list row ID.
int dListSeparatorIndex = documentIdentifier.indexOf("///");
int dLibSeparatorIndex = documentIdentifier.indexOf("//");
if (dListSeparatorIndex != -1)
{
// === List-style identifier ===
if (dListSeparatorIndex == documentIdentifier.length() - 3)
{
// == List path! ==
if (checkIncludeList(documentIdentifier.substring(0,documentIdentifier.length()-3),spec))
// This is the path for the list: No versioning
rval[i] = "";
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: List specification no longer includes list '"+documentIdentifier+"' - removing");
rval[i] = null;
}
}
else
{
// == List item or attachment path! ==
// Convert the modified document path to an unmodified one, plus a library path.
String decodedListPath = documentIdentifier.substring(0,dListSeparatorIndex);
String itemAndAttachment = documentIdentifier.substring(dListSeparatorIndex+2);
String decodedItemPath = decodedListPath + itemAndAttachment;
int cutoff = decodedListPath.lastIndexOf("/");
String sitePath = decodedListPath.substring(0,cutoff);
String list = decodedListPath.substring(cutoff+1);
String encodedSitePath = encodePath(sitePath);
int attachmentSeparatorIndex = itemAndAttachment.indexOf("//",1);
if (attachmentSeparatorIndex == -1)
{
// == List item path! ==
if (checkIncludeListItem(decodedItemPath,spec))
{
// This file is included, so calculate a version string. This will include metadata info, so get that first.
MetadataInformation metadataInfo = getMetadataSpecification(decodedItemPath,spec);
String[] accessTokens = activities.retrieveParentData(documentIdentifier, "accessTokens");
String[] denyTokens = activities.retrieveParentData(documentIdentifier, "denyTokens");
String[] listIDs = activities.retrieveParentData(documentIdentifier, "guids");
String[] listFields = activities.retrieveParentData(documentIdentifier, "fields");
String[] displayURLs = activities.retrieveParentData(documentIdentifier, "displayURLs");
String listID;
if (listIDs.length >= 1)
listID = listIDs[0];
else
listID = null;
String displayURL;
if (displayURLs.length >= 1)
displayURL = displayURLs[0];
else
displayURL = null;
if (listID != null)
{
String[] sortedMetadataFields = getInterestingFieldSetSorted(metadataInfo,listFields);
// Sort access tokens so they are comparable in the version string
java.util.Arrays.sort(accessTokens);
java.util.Arrays.sort(denyTokens);
// Next, get the actual timestamp field for the file.
ArrayList metadataDescription = new ArrayList();
metadataDescription.add("Modified");
metadataDescription.add("Created");
metadataDescription.add("ID");
metadataDescription.add("GUID");
// The document path includes the library, with no leading slash, and is decoded.
String decodedItemPathWithoutSite = decodedItemPath.substring(cutoff+1);
Map<String,String> values = proxy.getFieldValues( metadataDescription, encodedSitePath, listID, "/Lists/" + decodedItemPathWithoutSite, dspStsWorks );
String modifiedDate = values.get("Modified");
String createdDate = values.get("Created");
String id = values.get("ID");
String guid = values.get("GUID");
if (modifiedDate != null)
{
// Item has a modified date so we presume it exists.
Date modifiedDateValue = DateParser.parseISO8601Date(modifiedDate);
Date createdDateValue = DateParser.parseISO8601Date(createdDate);
// Build version string
String versionToken = modifiedDate;
// Revamped version string on 9/21/2013 to make parseability better
StringBuilder sb = new StringBuilder();
packList(sb,sortedMetadataFields,'+');
packList(sb,accessTokens,'+');
packList(sb,denyTokens,'+');
packDate(sb,modifiedDateValue);
packDate(sb,createdDateValue);
pack(sb,id,'+');
pack(sb,guid,'+');
pack(sb,displayURL,'+');
// The rest of this is unparseable
sb.append(versionToken);
sb.append(pathNameAttributeVersion);
// Added 9/7/07
sb.append("_").append(fileBaseUrl);
//
rval[i] = sb.toString();
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Complete version string for '"+documentIdentifier+"': " + rval[i]);
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Can't get version of '"+documentIdentifier+"' because it has no modify date");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Can't get version of '"+documentIdentifier+"' because list '"+decodedListPath+"' does not exist - removing");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: List item '"+documentIdentifier+"' is no longer included - removing");
rval[i] = null;
}
}
else
{
// == List item attachment path! ==
if (checkIncludeListItemAttachment(decodedItemPath,spec))
{
// To save work, we retrieve most of what we need in version info from the parent.
// Retrieve modified and created dates
String[] modifiedDateSet = activities.retrieveParentData(documentIdentifier, "modifiedDate");
String[] createdDateSet = activities.retrieveParentData(documentIdentifier, "createdDate");
String[] accessTokens = activities.retrieveParentData(documentIdentifier, "accessTokens");
String[] denyTokens = activities.retrieveParentData(documentIdentifier, "denyTokens");
String[] urlSet = activities.retrieveParentData(documentIdentifier, "url");
// Only one modifiedDate and createdDate can be used. If there's more than one, just pick one - the item will be reindexed
// anyhow.
String modifiedDate;
if (modifiedDateSet.length >= 1)
modifiedDate = modifiedDateSet[0];
else
modifiedDate = null;
String createdDate;
if (createdDateSet.length >= 1)
createdDate = createdDateSet[0];
else
createdDate = null;
String url;
if (urlSet.length >=1)
url = urlSet[0];
else
url = null;
// If we have no modified or created date, it means that the parent has gone away, so we go away too.
if (modifiedDate != null && url != null)
{
// Item has a modified date so we presume it exists.
Date modifiedDateValue;
if (modifiedDate != null)
modifiedDateValue = new Date(new Long(modifiedDate).longValue());
else
modifiedDateValue = null;
Date createdDateValue;
if (createdDate != null)
createdDateValue = new Date(new Long(createdDate).longValue());
else
createdDateValue = null;
// Build version string
String versionToken = modifiedDate;
StringBuilder sb = new StringBuilder();
// Pack the URL to get the data from
pack(sb,url,'+');
// Do the acls. If we get this far, we are guaranteed to have them, but we need to sort.
java.util.Arrays.sort(accessTokens);
java.util.Arrays.sort(denyTokens);
packList(sb,accessTokens,'+');
packList(sb,denyTokens,'+');
packDate(sb,modifiedDateValue);
packDate(sb,createdDateValue);
// The rest of this is unparseable
sb.append(versionToken);
sb.append(pathNameAttributeVersion);
sb.append("_").append(fileBaseUrl);
//
rval[i] = sb.toString();
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Complete version string for '"+documentIdentifier+"': " + rval[i]);
}
else
{
// Can't look up list ID, which means the list is gone, so delete
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Can't get version of '"+documentIdentifier+"' because modified date or attachment url not found");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: List item attachment '"+documentIdentifier+"' is no longer included - removing");
rval[i] = null;
}
}
}
}
else if (dLibSeparatorIndex != -1)
{
// === Library-style identifier ===
if (dLibSeparatorIndex == documentIdentifier.length() - 2)
{
// Library path!
if (checkIncludeLibrary(documentIdentifier.substring(0,documentIdentifier.length()-2),spec))
// This is the path for the library: No versioning
rval[i] = "";
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library specification no longer includes library '"+documentIdentifier+"' - removing");
rval[i] = null;
}
}
else
{
// == Document path ==
// Convert the modified document path to an unmodified one, plus a library path.
String decodedLibPath = documentIdentifier.substring(0,dLibSeparatorIndex);
String decodedDocumentPath = decodedLibPath + documentIdentifier.substring(dLibSeparatorIndex+1);
if (checkIncludeFile(decodedDocumentPath,spec))
{
// This file is included, so calculate a version string. This will include metadata info, so get that first.
MetadataInformation metadataInfo = getMetadataSpecification(decodedDocumentPath,spec);
int lastIndex = decodedLibPath.lastIndexOf("/");
String sitePath = decodedLibPath.substring(0,lastIndex);
String lib = decodedLibPath.substring(lastIndex+1);
// Retrieve the carry-down data we will be using.
// Note well: for sharepoint versions that include document/folder acls, these access tokens will be ignored,
// but they will still be carried down nonetheless, in case someone switches versions on us.
String[] accessTokens = activities.retrieveParentData(documentIdentifier, "accessTokens");
String[] denyTokens = activities.retrieveParentData(documentIdentifier, "denyTokens");
String[] libIDs = activities.retrieveParentData(documentIdentifier, "guids");
String[] libFields = activities.retrieveParentData(documentIdentifier, "fields");
String libID;
if (libIDs.length >= 1)
libID = libIDs[0];
else
libID = null;
if (libID != null)
{
String encodedSitePath = encodePath(sitePath);
String[] sortedMetadataFields = getInterestingFieldSetSorted(metadataInfo,libFields);
// Sort access tokens
java.util.Arrays.sort(accessTokens);
java.util.Arrays.sort(denyTokens);
// Next, get the actual timestamp field for the file.
ArrayList metadataDescription = new ArrayList();
metadataDescription.add("Last_x0020_Modified");
metadataDescription.add("Modified");
metadataDescription.add("Created");
metadataDescription.add("GUID");
// The document path includes the library, with no leading slash, and is decoded.
int cutoff = decodedLibPath.lastIndexOf("/");
String decodedDocumentPathWithoutSite = decodedDocumentPath.substring(cutoff);
Map<String,String> values = proxy.getFieldValues( metadataDescription, encodedSitePath, libID, decodedDocumentPathWithoutSite, dspStsWorks );
String modifiedDate = values.get("Modified");
String createdDate = values.get("Created");
String guid = values.get("GUID");
String modifyDate = values.get("Last_x0020_Modified");
if (modifyDate != null)
{
// Item has a modified date, so we presume it exists
Date modifiedDateValue = DateParser.parseISO8601Date(modifiedDate);
Date createdDateValue = DateParser.parseISO8601Date(createdDate);
// Build version string
String versionToken = modifyDate;
if (supportsItemSecurity)
{
// Do the acls.
if (forcedAcls == null)
{
// Security is off
accessTokens = new String[0];
denyTokens = new String[0];
}
else if (forcedAcls.length > 0)
{
// Security on, forced acls
accessTokens = forcedAcls;
denyTokens = new String[0];
}
else
{
// Security on, is native
accessTokens = proxy.getDocumentACLs( encodedSitePath, encodePath(decodedDocumentPath), activeDirectoryAuthority );
denyTokens = new String[]{defaultAuthorityDenyToken};
}
}
if (accessTokens != null)
{
// Revamped version string on 9/21/2013 to make parseability better
StringBuilder sb = new StringBuilder();
packList(sb,sortedMetadataFields,'+');
packList(sb,accessTokens,'+');
packList(sb,denyTokens,'+');
packDate(sb,modifiedDateValue);
packDate(sb,createdDateValue);
pack(sb,guid,'+');
// The rest of this is unparseable
sb.append(versionToken);
sb.append(pathNameAttributeVersion);
// Added 9/7/07
sb.append("_").append(fileBaseUrl);
//
rval[i] = sb.toString();
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Complete version string for '"+documentIdentifier+"': " + rval[i]);
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Couldn't get access tokens for item '"+decodedDocumentPath+"'; removing document '"+documentIdentifier+"'");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Can't get version of '"+documentIdentifier+"' because it has no modify date");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Can't get version of '"+documentIdentifier+"' because library '"+decodedLibPath+"' does not exist - removing");
rval[i] = null;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Document '"+documentIdentifier+"' is no longer included - removing");
rval[i] = null;
}
}
}
else
{
// === Site-style identifier ===
String sitePath = documentIdentifier.substring(0,documentIdentifier.length()-1);
if (sitePath.length() == 0)
sitePath = "/";
if (checkIncludeSite(sitePath,spec))
// This is the path for the site: No versioning
rval[i] = "";
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site specification no longer includes site '"+documentIdentifier+"' - removing");
rval[i] = null;
}
}
}
else
throw new ManifoldCFException("Invalid document identifier discovered: '"+documentIdentifier+"'");
i++;
}
return rval;
}
protected static void packDate(StringBuilder sb, Date dateValue)
{
if (dateValue != null)
{
sb.append("+");
pack(sb,new Long(dateValue.getTime()).toString(),'+');
}
else
sb.append("-");
}
protected static int unpackDate(String value, int index, Date theDate)
{
if (value.length() > index)
{
if (value.charAt(index++) == '+')
{
StringBuilder sb = new StringBuilder();
index = unpack(sb,value,index,'+');
if (sb.length() > 0)
{
theDate.setTime(new Long(sb.toString()).longValue());
}
}
}
return index;
}
protected String[] getInterestingFieldSetSorted(MetadataInformation metadataInfo, String[] allFields)
{
Set<String> metadataFields = new HashSet<String>();
// Figure out the actual metadata fields we will request
if (metadataInfo.getAllMetadata())
{
for (String field : allFields)
{
metadataFields.add(field);
}
}
else
{
String[] fields = metadataInfo.getMetadataFields();
for (String field : fields)
{
metadataFields.add(field);
}
}
// Convert the hashtable to an array and sort it.
String[] sortedMetadataFields = new String[metadataFields.size()];
int z = 0;
for (String field : metadataFields)
{
sortedMetadataFields[z++] = field;
}
java.util.Arrays.sort(sortedMetadataFields);
return sortedMetadataFields;
}
protected static final String[] attachmentDataNames = new String[]{"createdDate","modifiedDate","accessTokens","denyTokens","url","guids"};
/** Process a set of documents.
* This is the method that should cause each document to be fetched, processed, and the results either added
* to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
* The document specification allows this class to filter what is done based on the job.
*@param documentIdentifiers is the set of document identifiers to process.
*@param activities is the interface this method should use to queue up new document references
* and ingest documents.
*@param spec is the document specification.
*@param scanOnly is an array corresponding to the document identifiers. It is set to true to indicate when the processing
* should only find other references, and should not actually call the ingestion methods.
*/
@Override
public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, DocumentSpecification spec, boolean[] scanOnly)
throws ManifoldCFException, ServiceInterruption
{
getSession();
// Read the forced acls. A null return indicates that security is disabled!!!
// A zero-length return indicates that the native acls should be used.
// All of this is germane to how we ingest the document, so we need to note it in
// the version string completely.
String[] forcedAcls = getAcls(spec);
// Decode the system metadata part of the specification
SystemMetadataDescription sDesc = new SystemMetadataDescription(spec);
int i = 0;
while (i < documentIdentifiers.length)
{
// Make sure the job is still active
activities.checkJobStillActive();
String documentIdentifier = documentIdentifiers[i];
String version = versions[i];
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Processing: '" + documentIdentifier + "'");
if ( documentIdentifier.startsWith("/") )
{
// New document identifier format.
int dListSeparatorIndex = documentIdentifier.indexOf("///");
int dLibSeparatorIndex = documentIdentifier.indexOf("//");
if (dListSeparatorIndex != -1)
{
// === List style identifier ===
if (dListSeparatorIndex == documentIdentifier.length() - 3)
{
// Chained connectors always scan parent nodes, so they don't bother setting a version
String siteListPath = documentIdentifier.substring(0,documentIdentifier.length()-3);
int listCutoff = siteListPath.lastIndexOf( "/" );
String site = siteListPath.substring(0,listCutoff);
String listName = siteListPath.substring( listCutoff + 1 );
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Document identifier is a list: '" + siteListPath + "'" );
String listID = proxy.getListID( encodePath(site), site, listName );
if (listID != null)
{
String encodedSitePath = encodePath(site);
// Get the list's fields
Map<String,String> fieldNames = proxy.getFieldList( encodedSitePath, listID );
if (fieldNames != null)
{
String[] fields = new String[fieldNames.size()];
int j = 0;
for (String field : fieldNames.keySet())
{
fields[j++] = field;
}
String[] accessTokens;
String[] denyTokens;
if (forcedAcls == null)
{
// Security is off
accessTokens = new String[0];
denyTokens = new String[0];
}
else if (forcedAcls.length != 0)
{
// Forced security
accessTokens = forcedAcls;
denyTokens = new String[0];
}
else
{
// Security enabled, native security
accessTokens = proxy.getACLs( encodedSitePath, listID, activeDirectoryAuthority );
denyTokens = new String[]{defaultAuthorityDenyToken};
}
if (accessTokens != null)
{
ListItemStream fs = new ListItemStream( activities, encodedServerLocation, site, siteListPath, spec,
documentIdentifier, accessTokens, denyTokens, listID, fields );
boolean success = proxy.getChildren( fs, encodedSitePath , listID, dspStsWorks );
if (!success)
{
// Site/list no longer exists, so delete entry
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: No list found for list '"+siteListPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Access token lookup failed for list '"+siteListPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Field list lookup failed for list '"+siteListPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: GUID lookup failed for list '"+siteListPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
// == List item or attachment identifier ==
// Get the item part of the path
String decodedListPath = documentIdentifier.substring(0,dListSeparatorIndex);
String itemAndAttachment = documentIdentifier.substring(dListSeparatorIndex+2);
String decodedItemPath = decodedListPath + itemAndAttachment;
// If the item part has a slash, we're looking at an attachment
int attachmentSeparatorIndex = itemAndAttachment.indexOf("//",1);
if (attachmentSeparatorIndex == -1)
{
// == List item identifier ==
// Before we index, we queue up any attachments
int listCutoff = decodedListPath.lastIndexOf( "/" );
String site = decodedListPath.substring(0,listCutoff);
String listName = decodedListPath.substring( listCutoff + 1 );
// Placeholder for metadata specification
ArrayList metadataDescription = new ArrayList();
int startPosition = unpackList(metadataDescription,version,0,'+');
// Acls
ArrayList acls = new ArrayList();
ArrayList denyAcls = new ArrayList();
startPosition = unpackList(acls,version,startPosition,'+');
startPosition = unpackList(denyAcls,version,startPosition,'+');
// Dates
Date modifiedDate = new Date(0L);
startPosition = unpackDate(version,startPosition,modifiedDate);
if (modifiedDate.getTime() == 0L)
modifiedDate = null;
Date createdDate = new Date(0L);
startPosition = unpackDate(version,startPosition,createdDate);
if (createdDate.getTime() == 0L)
createdDate = null;
// ID (for looking up attachments)
StringBuilder idBuffer = new StringBuilder();
startPosition = unpack(idBuffer,version,startPosition,'+');
// List item GUID (for metadata)
StringBuilder guidBuffer = new StringBuilder();
startPosition = unpack(guidBuffer,version,startPosition,'+');
String guid = guidBuffer.toString();
// List item URL
StringBuilder relURLBuffer = new StringBuilder();
startPosition = unpack(relURLBuffer,version,startPosition,'+');
String relURL = relURLBuffer.toString();
// We need the list ID, which we've already fetched, so grab that from the parent data.
String[] listIDs = activities.retrieveParentData(documentIdentifier, "guids");
String listID;
if (listIDs.length >= 1)
listID = listIDs[0];
else
listID = null;
if (listID == null)
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: List '"+decodedListPath+"' no longer exists - deleting item '"+documentIdentifier+"'");
activities.deleteDocument(documentIdentifier);
i++;
continue;
}
// Now, do any queuing that is needed.
if (attachmentsSupported)
{
String itemNumber = idBuffer.toString();
List<NameValue> attachmentNames = proxy.getAttachmentNames( site, listID, itemNumber );
// Now, queue up each attachment as a separate entry
for (NameValue attachmentName : attachmentNames)
{
// For attachments, we use the carry-down feature to get the data where we need it. That's why
// we unpacked the version information early above.
// No check for inclusion; if the list item is included, so is this
String[][] dataValues = new String[attachmentDataNames.length][];
if (createdDate == null)
dataValues[0] = new String[0];
else
dataValues[0] = new String[]{new Long(createdDate.getTime()).toString()};
if (modifiedDate == null)
dataValues[1] = new String[0];
else
dataValues[1] = new String[]{new Long(modifiedDate.getTime()).toString()};
if (acls == null)
dataValues[2] = new String[0];
else
dataValues[2] = (String[])acls.toArray(new String[0]);
if (denyAcls == null)
dataValues[3] = new String[0];
else
dataValues[3] = (String[])denyAcls.toArray(new String[0]);
dataValues[4] = new String[]{attachmentName.getPrettyName()};
dataValues[5] = new String[]{guid};
activities.addDocumentReference(documentIdentifier + "//" + attachmentName.getValue(),
documentIdentifier, null, attachmentDataNames, dataValues);
}
}
if ( !scanOnly[ i ] )
{
// Convert the modified document path to an unmodified one, plus a library path.
String encodedItemPath = encodePath(decodedListPath.substring(0,listCutoff) + "/Lists/" + decodedItemPath.substring(listCutoff+1));
// Generate the URL we are going to use
String itemUrl = fileBaseUrl + relURL; //fileBaseUrl + encodedItemPath;
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Processing list item '"+documentIdentifier+"'; url: '" + itemUrl + "'" );
// Fetch the metadata we will be indexing
Map<String,String> metadataValues = null;
if (metadataDescription.size() > 0)
{
metadataValues = proxy.getFieldValues( metadataDescription, encodePath(site), listID, "/Lists/" + decodedItemPath.substring(listCutoff+1), dspStsWorks );
if (metadataValues == null)
{
// Item has vanished
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Item metadata fetch failure indicated that item is gone: '"+documentIdentifier+"' - removing");
activities.noDocument(documentIdentifier,version);
i++;
continue;
}
}
if (activities.checkLengthIndexable(0L))
{
InputStream is = new ByteArrayInputStream(new byte[0]);
try
{
RepositoryDocument data = new RepositoryDocument();
data.setBinary( is, 0L );
if (modifiedDate != null)
data.setModifiedDate(modifiedDate);
if (createdDate != null)
data.setCreatedDate(createdDate);
setDataACLs(data,acls,denyAcls);
setPathAttribute(data,sDesc,documentIdentifier);
if (metadataValues != null)
{
Iterator<String> iter = metadataValues.keySet().iterator();
while (iter.hasNext())
{
String fieldName = iter.next();
String fieldData = metadataValues.get(fieldName);
data.addField(fieldName,fieldData);
}
}
data.addField("GUID",guid);
try
{
activities.ingestDocumentWithException( documentIdentifier, version, itemUrl , data );
}
catch (IOException e)
{
handleIOException(e,"reading document");
}
}
finally
{
try
{
is.close();
}
catch (IOException e)
{
handleIOException(e,"closing stream");
}
}
}
else
// Document too long (should never happen; length is 0)
activities.noDocument( documentIdentifier, version );
}
}
else
{
// == List item attachment identifier ==
if (!scanOnly[i])
{
// Unpack the version info.
int startPosition = 0;
StringBuilder urlBuffer = new StringBuilder();
ArrayList accessTokens = new ArrayList();
ArrayList denyTokens = new ArrayList();
Date modifiedDate = new Date(0L);
Date createdDate = new Date(0L);
startPosition = unpack(urlBuffer,version,startPosition,'+');
startPosition = unpackList(accessTokens,version,startPosition,'+');
startPosition = unpackList(denyTokens,version,startPosition,'+');
startPosition = unpackDate(version,startPosition,modifiedDate);
startPosition = unpackDate(version,startPosition,createdDate);
if (modifiedDate.getTime() == 0L)
modifiedDate = null;
if (createdDate.getTime() == 0L)
createdDate = null;
// We need the list ID, which we've already fetched, so grab that from the parent data.
String[] guids = activities.retrieveParentData(documentIdentifier, "guids");
String guid;
if (guids.length >= 1)
guid = guids[0];
else
guid = null;
if (guid != null)
{
String url = urlBuffer.toString();
int lastIndex = url.lastIndexOf("/");
guid = guid + ":" + url.substring(lastIndex+1);
// Fetch and index. This also filters documents based on output connector restrictions.
String fileUrl = serverUrl + encodePath(url);
String fetchUrl = fileUrl;
if (!fetchAndIndexFile(activities, documentIdentifier, version, fileUrl, fetchUrl,
accessTokens, denyTokens, createdDate, modifiedDate, null, guid, sDesc))
{
// Document not indexed for whatever reason
activities.noDocument(documentIdentifier,version);
i++;
continue;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Skipping attachment '"+documentIdentifier+"' because no parent guid found");
activities.noDocument(documentIdentifier,version);
i++;
continue;
}
}
}
}
}
else if (dLibSeparatorIndex != -1)
{
// === Library style identifier ===
if (dLibSeparatorIndex == documentIdentifier.length() - 2)
{
// Chained document parents are always rescanned
String siteLibPath = documentIdentifier.substring(0,documentIdentifier.length()-2);
int libCutoff = siteLibPath.lastIndexOf( "/" );
String site = siteLibPath.substring(0,libCutoff);
String libName = siteLibPath.substring( libCutoff + 1 );
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Document identifier is a library: '" + siteLibPath + "'" );
String libID = proxy.getDocLibID( encodePath(site), site, libName );
if (libID != null)
{
String encodedSitePath = encodePath(site);
// Get the lib's fields
Map<String,String> fieldNames = proxy.getFieldList( encodedSitePath, libID );
if (fieldNames != null)
{
String[] fields = new String[fieldNames.size()];
int j = 0;
for (String field : fieldNames.keySet())
{
fields[j++] = field;
}
String[] accessTokens;
String[] denyTokens;
if (forcedAcls == null)
{
// Security is off
accessTokens = new String[0];
denyTokens = new String[0];
}
else if (forcedAcls.length != 0)
{
// Forced security
accessTokens = forcedAcls;
denyTokens = new String[0];
}
else
{
// Security enabled, native security
accessTokens = proxy.getACLs( encodedSitePath, libID, activeDirectoryAuthority );
denyTokens = new String[]{defaultAuthorityDenyToken};
}
if (accessTokens != null)
{
FileStream fs = new FileStream( activities, encodedServerLocation, site, siteLibPath, spec,
documentIdentifier, accessTokens, denyTokens, libID, fields );
boolean success = proxy.getChildren( fs, encodedSitePath , libID, dspStsWorks );
if (!success)
{
// Site/library no longer exists, so delete entry
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: No list found for library '"+siteLibPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Access token lookup failed for library '"+siteLibPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Field list lookup failed for library '"+siteLibPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: GUID lookup failed for library '"+siteLibPath+"' - deleting");
activities.deleteDocument(documentIdentifier);
}
}
else
{
// File/folder identifier
if ( !scanOnly[ i ] )
{
// Convert the modified document path to an unmodified one, plus a library path.
String decodedLibPath = documentIdentifier.substring(0,dLibSeparatorIndex);
String decodedDocumentPath = decodedLibPath + documentIdentifier.substring(dLibSeparatorIndex+1);
String encodedDocumentPath = encodePath(decodedDocumentPath);
int libCutoff = decodedLibPath.lastIndexOf( "/" );
String site = decodedLibPath.substring(0,libCutoff);
String libName = decodedLibPath.substring( libCutoff + 1 );
// Parse what we need out of version string.
// Placeholder for metadata specification
ArrayList metadataDescription = new ArrayList();
int startPosition = unpackList(metadataDescription,version,0,'+');
// Acls
ArrayList acls = new ArrayList();
ArrayList denyAcls = new ArrayList();
startPosition = unpackList(acls,version,startPosition,'+');
startPosition = unpackList(denyAcls,version,startPosition,'+');
// Dates
Date modifiedDate = new Date(0L);
startPosition = unpackDate(version,startPosition,modifiedDate);
if (modifiedDate.getTime() == 0L)
modifiedDate = null;
Date createdDate = new Date(0L);
startPosition = unpackDate(version,startPosition,createdDate);
if (createdDate.getTime() == 0L)
createdDate = null;
// Document GUID (for metadata)
StringBuilder guidBuffer = new StringBuilder();
startPosition = unpack(guidBuffer,version,startPosition,'+');
String guid = guidBuffer.toString();
// Generate the URL we are going to use
String fileUrl = fileBaseUrl + encodedDocumentPath;
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Processing file '"+documentIdentifier+"'; url: '" + fileUrl + "'" );
// First, fetch the metadata we plan to index.
Map<String,String> metadataValues = null;
if (metadataDescription.size() > 0)
{
// Retrieve the library guid from carrydown data
String[] libIDs = activities.retrieveParentData(documentIdentifier, "guids");
String documentLibID;
if (libIDs.length >= 1)
documentLibID = libIDs[0];
else
documentLibID = null;
if (documentLibID == null)
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library '"+decodedLibPath+"' no longer exists - deleting document '"+documentIdentifier+"'");
activities.deleteDocument(documentIdentifier);
i++;
continue;
}
int cutoff = decodedLibPath.lastIndexOf("/");
metadataValues = proxy.getFieldValues( metadataDescription, encodePath(site), documentLibID, decodedDocumentPath.substring(cutoff), dspStsWorks );
if (metadataValues == null)
{
// Document has vanished
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Document metadata fetch failure indicated that document is gone: '"+documentIdentifier+"' - removing");
activities.noDocument(documentIdentifier,version);
i++;
continue;
}
}
// Fetch and index. This also filters documents based on output connector restrictions.
if (!fetchAndIndexFile(activities, documentIdentifier, version, fileUrl, serverUrl + encodedServerLocation + encodedDocumentPath,
acls, denyAcls, createdDate, modifiedDate, metadataValues, guid, sDesc))
{
// Document not indexed for whatever reason
activities.noDocument(documentIdentifier,version);
i++;
continue;
}
}
}
}
else
{
// === Site-style identifier ===
activities.noDocument(documentIdentifier,version);
// Strip off the trailing "/" to get the site name.
String decodedSitePath = documentIdentifier.substring(0,documentIdentifier.length()-1);
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Document identifier is a site: '" + decodedSitePath + "'" );
// Look at subsites
List<NameValue> subsites = proxy.getSites( encodePath(decodedSitePath) );
if (subsites != null)
{
int j = 0;
while (j < subsites.size())
{
NameValue subSiteName = subsites.get(j++);
String newPath = decodedSitePath + "/" + subSiteName.getValue();
String encodedNewPath = encodePath(newPath);
if ( checkIncludeSite(newPath,spec) )
activities.addDocumentReference(newPath + "/");
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: No permissions to access subsites of '"+decodedSitePath+"' - skipping");
}
// Look at libraries
List<NameValue> libraries = proxy.getDocumentLibraries( encodePath(decodedSitePath), decodedSitePath );
if (libraries != null)
{
int j = 0;
while (j < libraries.size())
{
NameValue library = libraries.get(j++);
String newPath = decodedSitePath + "/" + library.getValue();
if (checkIncludeLibrary(newPath,spec))
activities.addDocumentReference(newPath + "//");
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: No permissions to access libraries of '"+decodedSitePath+"' - skipping");
}
// Look at lists
List<NameValue> lists = proxy.getLists( encodePath(decodedSitePath), decodedSitePath );
if (lists != null)
{
int j = 0;
while (j < lists.size())
{
NameValue list = lists.get(j++);
String newPath = decodedSitePath + "/" + list.getValue();
if (checkIncludeList(newPath,spec))
activities.addDocumentReference(newPath + "///");
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: No permissions to access lists of '"+decodedSitePath+"' - skipping");
}
}
}
else
throw new ManifoldCFException("Found illegal document identifier in processDocuments: '"+documentIdentifier+"'");
i++;
}
}
/** Method that fetches and indexes a file fetched from a SharePoint URL, with appropriate error handling
* etc.
*/
protected boolean fetchAndIndexFile(IProcessActivity activities, String documentIdentifier, String version,
String fileUrl, String fetchUrl, ArrayList acls, ArrayList denyAcls, Date createdDate, Date modifiedDate,
Map<String,String> metadataValues, String guid, SystemMetadataDescription sDesc)
throws ManifoldCFException, ServiceInterruption
{
// Before we fetch, confirm that the output connector will accept the document
if (activities.checkURLIndexable(fileUrl))
{
// Also check mime type
String contentType = mapExtensionToMimeType(documentIdentifier);
if (activities.checkMimeTypeIndexable(contentType))
{
// Set stuff up for fetch activity logging
long startFetchTime = System.currentTimeMillis();
try
{
// Read the document into a local temporary file, so I get a reliable length.
File tempFile = File.createTempFile("__shp__",".tmp");
try
{
// Open the output stream
OutputStream os = new FileOutputStream(tempFile);
try
{
// Catch all exceptions having to do with reading the document
try
{
ExecuteMethodThread emt = new ExecuteMethodThread(httpClient, fetchUrl, os);
emt.start();
int returnCode = emt.finishUp();
if (returnCode == 404 || returnCode == 401 || returnCode == 400 || returnCode == 415)
{
// Well, sharepoint thought the document was there, but it really isn't, so delete it.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Document at '"+fileUrl+"' failed to fetch with code "+Integer.toString(returnCode)+", deleting");
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
null,documentIdentifier,"Not found",Integer.toString(returnCode),null);
return false;
}
else if (returnCode != 200)
{
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
null,documentIdentifier,"Error","Http status "+Integer.toString(returnCode),null);
throw new ManifoldCFException("Error fetching document '"+fileUrl+"': "+Integer.toString(returnCode));
}
// Log the normal fetch activity
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Success",null,null);
}
catch (InterruptedException e)
{
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
}
catch (java.net.SocketTimeoutException e)
{
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Error",e.getMessage(),null);
Logging.connectors.warn("SharePoint: SocketTimeoutException thrown: "+e.getMessage(),e);
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to read '"+fileUrl+"', retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
catch (org.apache.http.conn.ConnectTimeoutException e)
{
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Error",e.getMessage(),null);
Logging.connectors.warn("SharePoint: ConnectTimeoutException thrown: "+e.getMessage(),e);
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to read '"+fileUrl+"', retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
catch (InterruptedIOException e)
{
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
}
catch (IllegalArgumentException e)
{
Logging.connectors.error("SharePoint: Illegal argument", e);
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Error",e.getMessage(),null);
throw new ManifoldCFException("SharePoint: Illegal argument: "+e.getMessage(),e);
}
catch (org.apache.http.HttpException e)
{
Logging.connectors.warn("SharePoint: HttpException thrown",e);
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Error",e.getMessage(),null);
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to read '"+fileUrl+"', retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
catch (IOException e)
{
activities.recordActivity(new Long(startFetchTime),ACTIVITY_FETCH,
new Long(tempFile.length()),documentIdentifier,"Error",e.getMessage(),null);
Logging.connectors.warn("SharePoint: IOException thrown: "+e.getMessage(),e);
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to read '"+fileUrl+"', retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
}
finally
{
os.close();
}
// Ingest the document
long documentLength = tempFile.length();
if (activities.checkLengthIndexable(documentLength))
{
InputStream is = new FileInputStream(tempFile);
try
{
RepositoryDocument data = new RepositoryDocument();
data.setBinary( is, documentLength );
data.setFileName(mapToFileName(documentIdentifier));
if (contentType != null)
data.setMimeType(contentType);
setDataACLs(data,acls,denyAcls);
setPathAttribute(data,sDesc,documentIdentifier);
if (modifiedDate != null)
data.setModifiedDate(modifiedDate);
if (createdDate != null)
data.setCreatedDate(createdDate);
if (metadataValues != null)
{
Iterator<String> iter = metadataValues.keySet().iterator();
while (iter.hasNext())
{
String fieldName = iter.next();
String fieldData = metadataValues.get(fieldName);
data.addField(fieldName,fieldData);
}
}
data.addField("GUID",guid);
try
{
activities.ingestDocumentWithException( documentIdentifier, version, fileUrl , data );
}
catch (IOException e)
{
handleIOException(e,"reading document");
}
return true;
}
finally
{
try
{
is.close();
}
catch (java.net.SocketTimeoutException e)
{
// This is not fatal
Logging.connectors.debug("SharePoint: Timeout before read could finish for '"+fileUrl+"': "+e.getMessage(),e);
}
catch (org.apache.http.conn.ConnectTimeoutException e)
{
// This is not fatal
Logging.connectors.debug("SharePoint: Connect timeout before read could finish for '"+fileUrl+"': "+e.getMessage(),e);
}
catch (InterruptedIOException e)
{
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
}
catch (IOException e)
{
// This is not fatal
Logging.connectors.debug("SharePoint: Server closed connection before read could finish for '"+fileUrl+"': "+e.getMessage(),e);
}
}
}
else
{
// Document too long
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Document '"+documentIdentifier+"' was too long, according to output connector");
return false;
}
}
finally
{
tempFile.delete();
}
}
catch (java.net.SocketTimeoutException e)
{
throw new ManifoldCFException("Socket timeout error writing '"+fileUrl+"' to temporary file: "+e.getMessage(),e);
}
catch (org.apache.http.conn.ConnectTimeoutException e)
{
throw new ManifoldCFException("Connect timeout error writing '"+fileUrl+"' to temporary file: "+e.getMessage(),e);
}
catch (InterruptedIOException e)
{
throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
}
catch (IOException e)
{
throw new ManifoldCFException("IO error writing '"+fileUrl+"' to temporary file: "+e.getMessage(),e);
}
}
else
{
// Mime type failed
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Skipping document '"+documentIdentifier+"' because output connector says mime type '"+((contentType==null)?"null":contentType)+"' is not indexable");
return false;
}
}
else
{
// URL failed
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Skipping document '"+documentIdentifier+"' because output connector says URL '"+fileUrl+"' is not indexable");
return false;
}
}
protected static void handleIOException(IOException e, String context)
throws ManifoldCFException, ServiceInterruption
{
if (e instanceof java.net.SocketTimeoutException)
{
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to "+context+", retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
else if (e instanceof org.apache.http.conn.ConnectTimeoutException)
{
long currentTime = System.currentTimeMillis();
throw new ServiceInterruption("SharePoint is down attempting to "+context+", retrying: "+e.getMessage(),e,currentTime + 300000L,
currentTime + 12 * 60 * 60000L,-1,true);
}
else if (e instanceof InterruptedIOException)
throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
else
throw new ManifoldCFException(e.getMessage(),e);
}
/** Map an extension to a mime type */
protected static String mapExtensionToMimeType(String fileName)
{
int slashIndex = fileName.lastIndexOf("/");
if (slashIndex != -1)
fileName = fileName.substring(slashIndex+1);
int dotIndex = fileName.lastIndexOf(".");
if (dotIndex == -1)
return null;
return ExtensionMimeMap.mapToMimeType(fileName.substring(dotIndex+1));
}
/** Map document identifier to file name */
protected static String mapToFileName(String fileName)
{
int slashIndex = fileName.lastIndexOf("/");
if (slashIndex != -1)
fileName = fileName.substring(slashIndex+1);
return fileName;
}
protected static void setDataACLs(RepositoryDocument data, ArrayList acls, ArrayList denyAcls)
{
if (acls != null)
{
String[] actualAcls = new String[acls.size()];
for (int j = 0; j < actualAcls.length; j++)
{
actualAcls[j] = (String)acls.get(j);
}
if (Logging.connectors.isDebugEnabled())
{
StringBuilder sb = new StringBuilder("SharePoint: Acls: [ ");
for (int j = 0; j < actualAcls.length; j++)
{
sb.append(actualAcls[j]).append(" ");
}
sb.append("]");
Logging.connectors.debug( sb.toString() );
}
data.setSecurityACL( RepositoryDocument.SECURITY_TYPE_DOCUMENT, actualAcls );
}
if (denyAcls != null)
{
String[] actualDenyAcls = new String[denyAcls.size()];
for (int j = 0; j < actualDenyAcls.length; j++)
{
actualDenyAcls[j] = (String)denyAcls.get(j);
}
if (Logging.connectors.isDebugEnabled())
{
StringBuilder sb = new StringBuilder("SharePoint: DenyAcls: [ ");
for (int j = 0; j < actualDenyAcls.length; j++)
{
sb.append(actualDenyAcls[j]).append(" ");
}
sb.append("]");
Logging.connectors.debug( sb.toString() );
}
data.setSecurityDenyACL( RepositoryDocument.SECURITY_TYPE_DOCUMENT, actualDenyAcls);
}
}
protected static void setPathAttribute(RepositoryDocument data, SystemMetadataDescription sDesc, String documentIdentifier)
throws ManifoldCFException
{
// Add the path metadata item into the mix, if enabled
String pathAttributeName = sDesc.getPathAttributeName();
if (pathAttributeName != null && pathAttributeName.length() > 0)
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Path attribute name is '"+pathAttributeName+"'");
String pathString = sDesc.getPathAttributeValue(documentIdentifier);
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Path attribute value is '"+pathString+"'");
data.addField(pathAttributeName,pathString);
}
else
Logging.connectors.debug("SharePoint: Path attribute name is null");
}
protected final static String[] fileStreamDataNames = new String[]{"accessTokens", "denyTokens", "guids", "fields"};
protected class FileStream implements IFileStream
{
protected final IProcessActivity activities;
protected final DocumentSpecification spec;
protected final String rootPath;
protected final String sitePath;
protected final String siteLibPath;
// For carry-down
protected final String documentIdentifier;
protected final String[][] dataValues;
public FileStream(IProcessActivity activities, String rootPath, String sitePath, String siteLibPath, DocumentSpecification spec,
String documentIdentifier, String[] accessTokens, String denyTokens[], String libID, String[] fields)
{
this.activities = activities;
this.spec = spec;
this.rootPath = rootPath;
this.sitePath = sitePath;
this.siteLibPath = siteLibPath;
this.documentIdentifier = documentIdentifier;
this.dataValues = new String[fileStreamDataNames.length][];
this.dataValues[0] = accessTokens;
this.dataValues[1] = denyTokens;
this.dataValues[2] = new String[]{libID};
this.dataValues[3] = fields;
}
@Override
public void addFile(String relPath, String displayURL)
throws ManifoldCFException
{
// First, convert the relative path to a full path
if ( !relPath.startsWith("/") )
{
relPath = rootPath + sitePath + "/" + relPath;
}
// Now, strip away what we don't want - namely, the root path. This makes the path relative to the root.
if ( relPath.startsWith(rootPath) )
{
relPath = relPath.substring(rootPath.length());
if ( checkIncludeFile( relPath, spec ) )
{
// Since the processing for a file needs to know the library path, we need a way to signal the cutoff between library and folder levels.
// The way I've chosen to do this is to use a double slash at that point, as a separator.
if (relPath.startsWith(siteLibPath))
{
// Split at the libpath/file boundary
String modifiedPath = siteLibPath + "/" + relPath.substring(siteLibPath.length());
activities.addDocumentReference( modifiedPath, documentIdentifier, null, fileStreamDataNames, dataValues );
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; path is '"+relPath+"', but expected to see something beginning with '"+siteLibPath+"'");
}
}
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; path is '"+relPath+"', but expected to see something beginning with '"+rootPath+"'");
}
}
}
protected final static String[] listItemStreamDataNames = new String[]{"accessTokens", "denyTokens", "guids", "fields", "displayURLs"};
protected class ListItemStream implements IFileStream
{
protected final IProcessActivity activities;
protected final DocumentSpecification spec;
protected final String rootPath;
protected final String sitePath;
protected final String siteListPath;
// For carry-down
protected final String documentIdentifier;
protected final String[][] dataValues;
public ListItemStream(IProcessActivity activities, String rootPath, String sitePath, String siteListPath, DocumentSpecification spec,
String documentIdentifier, String[] accessTokens, String denyTokens[], String listID, String[] fields)
{
this.activities = activities;
this.spec = spec;
this.rootPath = rootPath;
this.sitePath = sitePath;
this.siteListPath = siteListPath;
this.documentIdentifier = documentIdentifier;
this.dataValues = new String[listItemStreamDataNames.length][];
this.dataValues[0] = accessTokens;
this.dataValues[1] = denyTokens;
this.dataValues[2] = new String[]{listID};
this.dataValues[3] = fields;
}
@Override
public void addFile(String relPath, String displayURL)
throws ManifoldCFException
{
// First, convert the relative path to a full path
if ( !relPath.startsWith("/") )
{
relPath = rootPath + sitePath + "/" + relPath;
}
String fullPath = relPath;
// Now, strip away what we don't want - namely, the root path. This makes the path relative to the root.
if ( relPath.startsWith(rootPath) )
{
relPath = relPath.substring(rootPath.length());
if (relPath.startsWith(sitePath))
{
relPath = relPath.substring(sitePath.length());
// Now, strip "Lists" from relPath. If it doesn't start with /Lists/, ignore it.
if (relPath.startsWith("/Lists/"))
{
relPath = sitePath + relPath.substring("/Lists".length());
if ( checkIncludeListItem( relPath, spec ) )
{
if (relPath.startsWith(siteListPath))
{
// Since the processing for a item needs to know the list path, we need a way to signal the cutoff between list and item levels.
// The way I've chosen to do this is to use a triple slash at that point, as a separator.
String modifiedPath = relPath.substring(0,siteListPath.length()) + "//" + relPath.substring(siteListPath.length());
if (displayURL != null)
dataValues[4] = new String[]{displayURL};
else
dataValues[4] = new String[]{fullPath};
activities.addDocumentReference( modifiedPath, documentIdentifier, null, listItemStreamDataNames, dataValues );
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; site path is '"+relPath+"', but expected to see something beginning with '"+siteListPath+"'");
}
}
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; rel path is '"+relPath+"', but expected to see something beginning with '/Lists/'");
}
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; site path is '"+relPath+"', but expected to see something beginning with '"+sitePath+"'");
}
}
else
{
Logging.connectors.warn("SharePoint: Unexpected relPath structure; path is '"+relPath+"', but expected to see something beginning with '"+rootPath+"'");
}
}
}
// UI support methods.
//
// These support methods come in two varieties. The first bunch is involved in setting up connection configuration information. The second bunch
// is involved in presenting and editing document specification information for a job. The two kinds of methods are accordingly treated differently,
// in that the first bunch cannot assume that the current connector object is connected, while the second bunch can. That is why the first bunch
// receives a thread context argument for all UI methods, while the second bunch does not need one (since it has already been applied via the connect()
// method, above).
/** Output the configuration header section.
* This method is called in the head section of the connector's configuration page. Its purpose is to add the required tabs to the list, and to output any
* javascript methods that might be needed by the configuration editing HTML.
*@param threadContext is the local thread context.
*@param out is the output to which any HTML should be sent.
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
*/
@Override
public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out,
Locale locale, ConfigParams parameters, List<String> tabsArray)
throws ManifoldCFException, IOException
{
tabsArray.add(Messages.getString(locale,"SharePointRepository.Server"));
tabsArray.add(Messages.getString(locale,"SharePointRepository.AuthorityType"));
Messages.outputResourceWithVelocity(out,locale,"editConfiguration.js",null);
}
/** Output the configuration body section.
* This method is called in the body section of the connector's configuration page. Its purpose is to present the required form elements for editing.
* The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
* form is "editconnection".
*@param threadContext is the local thread context.
*@param out is the output to which any HTML should be sent.
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@param tabName is the current tab name.
*/
@Override
public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out,
Locale locale, ConfigParams parameters, String tabName)
throws ManifoldCFException, IOException
{
Map<String,Object> velocityContext = new HashMap<String,Object>();
velocityContext.put("TabName",tabName);
fillInServerTab(velocityContext,out,parameters);
fillInAuthorityTypeTab(velocityContext,out,parameters);
Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Server.html",velocityContext);
Messages.outputResourceWithVelocity(out,locale,"editConfiguration_AuthorityType.html",velocityContext);
}
/** Process a configuration post.
* This method is called at the start of the connector's configuration page, whenever there is a possibility that form data for a connection has been
* posted. Its purpose is to gather form information and modify the configuration parameters accordingly.
* The name of the posted form is "editconnection".
*@param threadContext is the local thread context.
*@param variableContext is the set of variables available from the post, including binary file post information.
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
*/
@Override
public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext,
Locale locale, ConfigParams parameters)
throws ManifoldCFException
{
String serverVersion = variableContext.getParameter("serverVersion");
if (serverVersion != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERVERSION,serverVersion);
String serverProtocol = variableContext.getParameter("serverProtocol");
if (serverProtocol != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERPROTOCOL,serverProtocol);
String serverName = variableContext.getParameter("serverName");
if (serverName != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERNAME,serverName);
String serverPort = variableContext.getParameter("serverPort");
if (serverPort != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERPORT,serverPort);
String serverLocation = variableContext.getParameter("serverLocation");
if (serverLocation != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERLOCATION,serverLocation);
String userName = variableContext.getParameter("userName");
if (userName != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERUSERNAME,userName);
String password = variableContext.getParameter("password");
if (password != null)
parameters.setObfuscatedParameter(SharePointConfig.PARAM_SERVERPASSWORD,variableContext.mapKeyToPassword(password));
String keystoreValue = variableContext.getParameter("keystoredata");
if (keystoreValue != null)
parameters.setParameter(SharePointConfig.PARAM_SERVERKEYSTORE,keystoreValue);
String configOp = variableContext.getParameter("configop");
if (configOp != null)
{
if (configOp.equals("Delete"))
{
String alias = variableContext.getParameter("shpkeystorealias");
keystoreValue = parameters.getParameter(SharePointConfig.PARAM_SERVERKEYSTORE);
IKeystoreManager mgr;
if (keystoreValue != null)
mgr = KeystoreManagerFactory.make("",keystoreValue);
else
mgr = KeystoreManagerFactory.make("");
mgr.remove(alias);
parameters.setParameter(SharePointConfig.PARAM_SERVERKEYSTORE,mgr.getString());
}
else if (configOp.equals("Add"))
{
String alias = IDFactory.make(threadContext);
byte[] certificateValue = variableContext.getBinaryBytes("shpcertificate");
keystoreValue = parameters.getParameter(SharePointConfig.PARAM_SERVERKEYSTORE);
IKeystoreManager mgr;
if (keystoreValue != null)
mgr = KeystoreManagerFactory.make("",keystoreValue);
else
mgr = KeystoreManagerFactory.make("");
java.io.InputStream is = new java.io.ByteArrayInputStream(certificateValue);
String certError = null;
try
{
mgr.importCertificate(alias,is);
}
catch (Throwable e)
{
certError = e.getMessage();
}
finally
{
try
{
is.close();
}
catch (IOException e)
{
// Don't report anything
}
}
if (certError != null)
{
// Redirect to error page
return "Illegal certificate: "+certError;
}
parameters.setParameter(SharePointConfig.PARAM_SERVERKEYSTORE,mgr.getString());
}
}
String authorityType = variableContext.getParameter("authorityType");
if (authorityType != null)
parameters.setParameter(SharePointConfig.PARAM_AUTHORITYTYPE,authorityType);
return null;
}
/** View configuration.
* This method is called in the body section of the connector's view configuration page. Its purpose is to present the connection information to the user.
* The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
*@param threadContext is the local thread context.
*@param out is the output to which any HTML should be sent.
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*/
@Override
public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out,
Locale locale, ConfigParams parameters)
throws ManifoldCFException, IOException
{
Map<String,Object> velocityContext = new HashMap<String,Object>();
fillInServerTab(velocityContext,out,parameters);
fillInAuthorityTypeTab(velocityContext,out,parameters);
Messages.outputResourceWithVelocity(out,locale,"viewConfiguration.html",velocityContext);
}
protected static void fillInAuthorityTypeTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters)
throws ManifoldCFException
{
// Default to Active Directory, for backwards compatibility
String authorityType = parameters.getParameter(SharePointConfig.PARAM_AUTHORITYTYPE);
if (authorityType == null)
authorityType = "ActiveDirectory";
velocityContext.put("AUTHORITYTYPE", authorityType);
}
protected static void fillInServerTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters)
throws ManifoldCFException
{
String serverVersion = parameters.getParameter(SharePointConfig.PARAM_SERVERVERSION);
if (serverVersion == null)
serverVersion = "2.0";
String serverProtocol = parameters.getParameter(SharePointConfig.PARAM_SERVERPROTOCOL);
if (serverProtocol == null)
serverProtocol = "http";
String serverName = parameters.getParameter(SharePointConfig.PARAM_SERVERNAME);
if (serverName == null)
serverName = "localhost";
String serverPort = parameters.getParameter(SharePointConfig.PARAM_SERVERPORT);
if (serverPort == null)
serverPort = "";
String serverLocation = parameters.getParameter(SharePointConfig.PARAM_SERVERLOCATION);
if (serverLocation == null)
serverLocation = "";
String userName = parameters.getParameter(SharePointConfig.PARAM_SERVERUSERNAME);
if (userName == null)
userName = "";
String password = parameters.getObfuscatedParameter(SharePointConfig.PARAM_SERVERPASSWORD);
if (password == null)
password = "";
else
password = out.mapPasswordToKey(password);
String keystore = parameters.getParameter(SharePointConfig.PARAM_SERVERKEYSTORE);
IKeystoreManager localKeystore;
if (keystore == null)
localKeystore = KeystoreManagerFactory.make("");
else
localKeystore = KeystoreManagerFactory.make("",keystore);
List<Map<String,String>> certificates = new ArrayList<Map<String,String>>();
String[] contents = localKeystore.getContents();
for (String alias : contents)
{
String description = localKeystore.getDescription(alias);
if (description.length() > 128)
description = description.substring(0,125) + "...";
Map<String,String> certificate = new HashMap<String,String>();
certificate.put("ALIAS", alias);
certificate.put("DESCRIPTION", description);
certificates.add(certificate);
}
// Fill in context
velocityContext.put("SERVERVERSION", serverVersion);
velocityContext.put("SERVERPROTOCOL", serverProtocol);
velocityContext.put("SERVERNAME", serverName);
velocityContext.put("SERVERPORT", serverPort);
velocityContext.put("SERVERLOCATION", serverLocation);
velocityContext.put("USERNAME", userName);
velocityContext.put("PASSWORD", password);
if (keystore != null)
velocityContext.put("KEYSTORE", keystore);
velocityContext.put("CERTIFICATELIST", certificates);
}
/** Output the specification header section.
* This method is called in the head section of a job page which has selected a repository connection of the current type. Its purpose is to add the required tabs
* to the list, and to output any javascript methods that might be needed by the job editing HTML.
*@param out is the output to which any HTML should be sent.
*@param ds is the current document specification for this job.
*@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
*/
@Override
public void outputSpecificationHeader(IHTTPOutput out, Locale locale, DocumentSpecification ds, List<String> tabsArray)
throws ManifoldCFException, IOException
{
tabsArray.add(Messages.getString(locale,"SharePointRepository.Paths"));
tabsArray.add(Messages.getString(locale,"SharePointRepository.Security"));
tabsArray.add(Messages.getString(locale,"SharePointRepository.Metadata"));
Messages.outputResourceWithVelocity(out,locale,"editSpecification.js",null);
}
/** Output the specification body section.
* This method is called in the body section of a job page which has selected a repository connection of the current type. Its purpose is to present the required form elements for editing.
* The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
* form is "editjob".
*@param out is the output to which any HTML should be sent.
*@param ds is the current document specification for this job.
*@param tabName is the current tab name.
*/
@Override
public void outputSpecificationBody(IHTTPOutput out, Locale locale, DocumentSpecification ds, String tabName)
throws ManifoldCFException, IOException
{
Map<String,Object> velocityContext = new HashMap<String,Object>();
velocityContext.put("TabName",tabName);
fillInSecurityTab(velocityContext,out,ds);
fillInPathsTab(velocityContext,out,ds);
fillInMetadataTab(velocityContext,out,ds);
// Now, do the part of the tabs that requires context logic
if (tabName.equals(Messages.getString(locale,"SharePointRepository.Paths")))
fillInTransientPathsInfo(velocityContext);
else if (tabName.equals(Messages.getString(locale,"SharePointRepository.Metadata")))
fillInTransientMetadataInfo(velocityContext);
Messages.outputResourceWithVelocity(out,locale,"editSpecification_Security.html",velocityContext);
Messages.outputResourceWithVelocity(out,locale,"editSpecification_Paths.html",velocityContext);
Messages.outputResourceWithVelocity(out,locale,"editSpecification_Metadata.html",velocityContext);
}
/** Fill in metadata tab */
protected static void fillInMetadataTab(Map<String,Object> velocityContext, IHTTPOutput out, DocumentSpecification ds)
{
// Find the path-value metadata attribute name
String pathNameAttribute = "";
MatchMap matchMap = new MatchMap();
List<Map<String,Object>> metadataRules = new ArrayList<Map<String,Object>>();
for (int i = 0; i < ds.getChildCount(); i++)
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("pathnameattribute"))
{
pathNameAttribute = sn.getAttributeValue("value");
}
else if (sn.getType().equals("pathmap"))
{
String pathMatch = sn.getAttributeValue("match");
String pathReplace = sn.getAttributeValue("replace");
matchMap.appendMatchPair(pathMatch,pathReplace);
}
else if (sn.getType().equals("startpoint"))
{
String site = sn.getAttributeValue("site");
String lib = sn.getAttributeValue("lib");
String path = site + "/" + lib + "/*";
String allmetadata = sn.getAttributeValue("allmetadata");
StringBuilder metadataFieldList = new StringBuilder();
List<String> metadataFieldArray = new ArrayList<String>();
if (allmetadata == null || !allmetadata.equals("true"))
{
for (int j = 0; j < sn.getChildCount(); j++)
{
SpecificationNode node = sn.getChild(j);
if (node.getType().equals("metafield"))
{
if (metadataFieldList.length() > 0)
metadataFieldList.append(", ");
String val = node.getAttributeValue("value");
metadataFieldList.append(val);
metadataFieldArray.add(val);
}
}
allmetadata = "false";
}
if (allmetadata.equals("true") || metadataFieldList.length() > 0)
{
Map<String,Object> item = new HashMap<String,Object>();
item.put("THEPATH",path);
item.put("THEACTION","include");
item.put("ALLFLAG",allmetadata);
item.put("FIELDLIST",metadataFieldArray);
item.put("FIELDS",metadataFieldList.toString());
metadataRules.add(item);
}
}
else if (sn.getType().equals("metadatarule"))
{
String path = sn.getAttributeValue("match");
String action = sn.getAttributeValue("action");
String allmetadata = sn.getAttributeValue("allmetadata");
StringBuilder metadataFieldList = new StringBuilder();
List<String> metadataFieldArray = new ArrayList<String>();
if (action.equals("include"))
{
if (allmetadata == null || !allmetadata.equals("true"))
{
for (int j = 0; j < sn.getChildCount(); j++)
{
SpecificationNode node = sn.getChild(j);
if (node.getType().equals("metafield"))
{
String val = node.getAttributeValue("value");
if (metadataFieldList.length() > 0)
metadataFieldList.append(", ");
metadataFieldList.append(val);
metadataFieldArray.add(val);
}
}
allmetadata="false";
}
}
else
allmetadata = "";
Map<String,Object> item = new HashMap<String,Object>();
item.put("THEPATH",path);
item.put("THEACTION",action);
item.put("ALLFLAG",allmetadata);
item.put("FIELDLIST",metadataFieldArray);
item.put("FIELDS",metadataFieldList.toString());
metadataRules.add(item);
}
}
List<Map<String,String>> mapList = new ArrayList<Map<String,String>>();
for (int i = 0; i < matchMap.getMatchCount(); i++)
{
String matchString = matchMap.getMatchString(i);
String replaceString = matchMap.getReplaceString(i);
Map<String,String> item = new HashMap<String,String>();
item.put("MATCH",matchString);
item.put("REPLACE",replaceString);
mapList.add(item);
}
velocityContext.put("PATHNAMEATTRIBUTE",pathNameAttribute);
velocityContext.put("MAPLIST",mapList);
velocityContext.put("METADATARULES",metadataRules);
}
/** Fill in transient metadata info */
protected void fillInTransientMetadataInfo(Map<String,Object> velocityContext)
{
// The following variables may be in the thread context because postspec.jsp put them there:
// (1) "metapath", which contains the rule path as it currently stands;
// (2) "metapathstate", which describes what the current path represents. Values are "unknown", "site", "library".
// (3) "metapathlibrary" is the library or list path (if this is known yet).
// Once the widget is in the state "unknown", it can only be reset, and cannot be further modified
String metaPathSoFar = (String)currentContext.get("metapath");
String metaPathState = (String)currentContext.get("metapathstate");
String metaPathLibrary = (String)currentContext.get("metapathlibrary");
if (metaPathState == null)
metaPathState = "unknown";
if (metaPathSoFar == null)
{
metaPathSoFar = "/";
metaPathState = "site";
}
String message = null;
List<NameValue> fieldList = null;
if (metaPathLibrary != null)
{
// Look up metadata fields
int index = metaPathLibrary.lastIndexOf("/");
String site = metaPathLibrary.substring(0,index);
String libOrList = metaPathLibrary.substring(index+1);
Map<String,String> metaFieldList = null;
try
{
if (metaPathState.equals("library") || metaPathState.equals("file"))
metaFieldList = getLibFieldList(site,libOrList);
else if (metaPathState.equals("list"))
metaFieldList = getListFieldList(site,libOrList);
}
catch (ManifoldCFException e)
{
e.printStackTrace();
message = e.getMessage();
}
catch (ServiceInterruption e)
{
message = "SharePoint unavailable: "+e.getMessage();
}
if (metaFieldList != null)
{
String[] fields = new String[metaFieldList.size()];
int j = 0;
Iterator<String> iter = metaFieldList.keySet().iterator();
while (iter.hasNext())
{
fields[j++] = iter.next();
}
java.util.Arrays.sort(fields);
fieldList = new ArrayList<NameValue>();
for (String field : fields)
{
fieldList.add(new NameValue(field,metaFieldList.get(field)));
}
}
}
// Grab next site list and lib list
List<NameValue> childSiteList = null;
List<NameValue> childLibList = null;
List<NameValue> childListList = null;
if (message == null && metaPathState.equals("site"))
{
try
{
String queryPath = metaPathSoFar;
if (queryPath.equals("/"))
queryPath = "";
childSiteList = getSites(queryPath);
if (childSiteList == null)
{
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
// Illegal path - state becomes "unknown".
metaPathState = "unknown";
metaPathLibrary = null;
}
childLibList = getDocLibsBySite(queryPath);
if (childLibList == null)
{
// Illegal path - state becomes "unknown"
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
metaPathState = "unknown";
metaPathLibrary = null;
}
childListList = getListsBySite(queryPath);
if (childListList == null)
{
// Illegal path - state becomes "unknown"
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
metaPathState = "unknown";
metaPathLibrary = null;
}
}
catch (ManifoldCFException e)
{
Logging.connectors.warn(e.getMessage(),e);
message = e.getMessage();
}
catch (ServiceInterruption e)
{
message = "SharePoint unavailable: "+e.getMessage();
}
}
if (metaPathSoFar != null)
velocityContext.put("METAPATHSOFAR",metaPathSoFar);
if (metaPathState != null)
velocityContext.put("METAPATHSTATE",metaPathState);
if (metaPathLibrary != null)
velocityContext.put("METAPATHLIBRARY",metaPathLibrary);
if (message != null)
velocityContext.put("METAMESSAGE",message);
if (fieldList != null)
velocityContext.put("METAFIELDLIST",fieldList);
if (childSiteList != null)
velocityContext.put("METACHILDSITELIST",childSiteList);
if (childLibList != null)
velocityContext.put("METACHILDLIBLIST",childLibList);
if (childListList != null)
velocityContext.put("METACHILDLISTLIST",childListList);
}
/** Fill in paths tab */
protected static void fillInPathsTab(Map<String,Object> velocityContext, IHTTPOutput out, DocumentSpecification ds)
{
List<Map<String,String>> rules = new ArrayList<Map<String,String>>();
for (int i = 0; i < ds.getChildCount(); i++)
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("startpoint"))
{
String site = sn.getAttributeValue("site");
String lib = sn.getAttributeValue("lib");
String siteLib = site + "/" + lib + "/";
// Go through all the file/folder rules for the startpoint, and generate new "rules" corresponding to each.
for (int j = 0; j < sn.getChildCount(); j++)
{
SpecificationNode node = sn.getChild(j);
if (node.getType().equals("include") || node.getType().equals("exclude"))
{
String matchPart = node.getAttributeValue("match");
String ruleType = node.getAttributeValue("type");
String theFlavor = node.getType();
String thePath = siteLib + matchPart;
Map<String,String> item = new HashMap<String,String>();
item.put("THEPATH",thePath);
item.put("THETYPE","file");
item.put("THEACTION",theFlavor);
rules.add(item);
if (ruleType.equals("file") && !matchPart.startsWith("*"))
{
thePath = siteLib + "*/" + matchPart;
item = new HashMap<String,String>();
item.put("THEPATH",thePath);
item.put("THETYPE","file");
item.put("THEACTION",theFlavor);
rules.add(item);
}
}
}
}
else if (sn.getType().equals("pathrule"))
{
String match = sn.getAttributeValue("match");
String type = sn.getAttributeValue("type");
String action = sn.getAttributeValue("action");
Map<String,String> item = new HashMap<String,String>();
item.put("THEPATH",match);
item.put("THETYPE",type);
item.put("THEACTION",action);
rules.add(item);
}
}
velocityContext.put("RULES",rules);
}
/** Fill in the transient portion of the Paths tab */
protected void fillInTransientPathsInfo(Map<String,Object> velocityContext)
{
// The following variables may be in the thread context because postspec.jsp put them there:
// (1) "specpath", which contains the rule path as it currently stands;
// (2) "specpathstate", which describes what the current path represents. Values are "unknown", "site", "library", "list".
// Once the widget is in the state "unknown", it can only be reset, and cannot be further modified
// specsitepath may be in the thread context, put there by postspec.jsp
String pathSoFar = (String)currentContext.get("specpath");
String pathState = (String)currentContext.get("specpathstate");
String pathLibrary = (String)currentContext.get("specpathlibrary");
if (pathState == null)
{
pathState = "unknown";
pathLibrary = null;
}
if (pathSoFar == null)
{
pathSoFar = "/";
pathState = "site";
pathLibrary = null;
}
// Grab next site list and lib list
List<NameValue> childSiteList = null;
List<NameValue> childLibList = null;
List<NameValue> childListList = null;
String message = null;
if (pathState.equals("site"))
{
try
{
String queryPath = pathSoFar;
if (queryPath.equals("/"))
queryPath = "";
childSiteList = getSites(queryPath);
if (childSiteList == null)
{
// Illegal path - state becomes "unknown".
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
pathState = "unknown";
pathLibrary = null;
}
childLibList = getDocLibsBySite(queryPath);
if (childLibList == null)
{
// Illegal path - state becomes "unknown"
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
pathState = "unknown";
pathLibrary = null;
}
childListList = getListsBySite(queryPath);
if (childListList == null)
{
// Illegal path - state becomes "unknown"
if (queryPath.length() == 0)
throw new ManifoldCFException("Root site is unreachable, or user has no permissions");
pathState = "unknown";
pathLibrary = null;
}
}
catch (ManifoldCFException e)
{
Logging.connectors.warn(e.getMessage(),e);
message = e.getMessage();
}
catch (ServiceInterruption e)
{
message = "SharePoint unavailable: "+e.getMessage();
}
}
if (pathSoFar != null)
velocityContext.put("PATHSOFAR",pathSoFar);
if (pathState != null)
velocityContext.put("PATHSTATE",pathState);
if (pathLibrary != null)
velocityContext.put("PATHLIBRARY",pathLibrary);
if (message != null)
velocityContext.put("MESSAGE",message);
if (childSiteList != null)
velocityContext.put("CHILDSITELIST",childSiteList);
if (childLibList != null)
velocityContext.put("CHILDLIBLIST",childLibList);
if (childListList != null)
velocityContext.put("CHILDLISTLIST",childListList);
}
/** Fill in security tab */
protected static void fillInSecurityTab(Map<String,Object> velocityContext, IHTTPOutput out, DocumentSpecification ds)
{
// Security tab
String security = "on";
List<String> accessTokens = new ArrayList<String>();
for (int i = 0; i < ds.getChildCount(); i++)
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("security"))
{
security = sn.getAttributeValue("value");
}
else if (sn.getType().equals("access"))
{
String token = sn.getAttributeValue("token");
accessTokens.add(token);
}
}
velocityContext.put("SECURITY",security);
velocityContext.put("ACCESSTOKENS",accessTokens);
}
/** Process a specification post.
* This method is called at the start of job's edit or view page, whenever there is a possibility that form data for a connection has been
* posted. Its purpose is to gather form information and modify the document specification accordingly.
* The name of the posted form is "editjob".
*@param variableContext contains the post data, including binary file-upload information.
*@param ds is the current document specification for this job.
*@return null if all is well, or a string error message if there is an error that should prevent saving of the job (and cause a redirection to an error page).
*/
@Override
public String processSpecificationPost(IPostParameters variableContext, Locale locale, DocumentSpecification ds)
throws ManifoldCFException
{
// Remove old-style rules, but only if the information would not be lost
if (variableContext.getParameter("specpathcount") != null && variableContext.getParameter("metapathcount") != null)
{
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("startpoint"))
ds.removeChild(i);
else
i++;
}
}
String x = variableContext.getParameter("specpathcount");
if (x != null)
{
// Delete all path rule entries first
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("pathrule"))
ds.removeChild(i);
else
i++;
}
// Find out how many children were sent
int pathCount = Integer.parseInt(x);
// Gather up these
i = 0;
while (i < pathCount)
{
String pathDescription = "_"+Integer.toString(i);
String pathOpName = "specop"+pathDescription;
x = variableContext.getParameter(pathOpName);
if (x != null && x.equals("Delete"))
{
// Skip to the next
i++;
continue;
}
// Get the stored information for this rule.
String path = variableContext.getParameter("specpath"+pathDescription);
String type = variableContext.getParameter("spectype"+pathDescription);
String action = variableContext.getParameter("specflav"+pathDescription);
SpecificationNode node = new SpecificationNode("pathrule");
node.setAttribute("match",path);
node.setAttribute("action",action);
node.setAttribute("type",type);
// If there was an insert operation, do it now
if (x != null && x.equals("Insert Here"))
{
// The global parameters are what are used to create the rule
path = variableContext.getParameter("specpath");
type = variableContext.getParameter("spectype");
action = variableContext.getParameter("specflavor");
SpecificationNode sn = new SpecificationNode("pathrule");
sn.setAttribute("match",path);
sn.setAttribute("action",action);
sn.setAttribute("type",type);
ds.addChild(ds.getChildCount(),sn);
}
ds.addChild(ds.getChildCount(),node);
i++;
}
// See if there's a global path rule operation
String op = variableContext.getParameter("specop");
if (op != null)
{
if (op.equals("Add"))
{
String match = variableContext.getParameter("specpath");
String action = variableContext.getParameter("specflavor");
String type = variableContext.getParameter("spectype");
SpecificationNode node = new SpecificationNode("pathrule");
node.setAttribute("match",match);
node.setAttribute("action",action);
node.setAttribute("type",type);
ds.addChild(ds.getChildCount(),node);
}
}
// See if there's a global pathbuilder operation
String pathop = variableContext.getParameter("specpathop");
if (pathop != null)
{
if (pathop.equals("Reset"))
{
currentContext.save("specpath","/");
currentContext.save("specpathstate","site");
currentContext.save("specpathlibrary",null);
}
else if (pathop.equals("AppendSite"))
{
String path = variableContext.getParameter("specpath");
String addon = variableContext.getParameter("specsite");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
}
currentContext.save("specpath",path);
currentContext.save("specpathstate","site");
currentContext.save("specpathlibrary",null);
}
else if (pathop.equals("AppendLibrary"))
{
String path = variableContext.getParameter("specpath");
String addon = variableContext.getParameter("speclibrary");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
currentContext.save("specpathstate","library");
currentContext.save("specpathlibrary",path);
}
currentContext.save("specpath",path);
}
else if (pathop.equals("AppendList"))
{
String path = variableContext.getParameter("specpath");
String addon = variableContext.getParameter("speclist");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
currentContext.save("specpathstate","list");
currentContext.save("specpathlibrary",path);
}
currentContext.save("specpath",path);
}
else if (pathop.equals("AppendText"))
{
String path = variableContext.getParameter("specpath");
String library = variableContext.getParameter("specpathlibrary");
String addon = variableContext.getParameter("specmatch");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
currentContext.save("specpathstate","unknown");
}
currentContext.save("specpath",path);
currentContext.save("specpathlibrary",library);
}
else if (pathop.equals("Remove"))
{
// Strip off end
String path = variableContext.getParameter("specpath");
int index = path.lastIndexOf("/");
path = path.substring(0,index);
if (path.length() == 0)
path = "/";
currentContext.save("specpath",path);
// Now, adjust state.
String pathState = variableContext.getParameter("specpathstate");
if (pathState.equals("library") || pathState.equals("list"))
pathState = "site";
currentContext.save("specpathstate",pathState);
}
}
}
x = variableContext.getParameter("metapathcount");
if (x != null)
{
// Delete all metadata rule entries first
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("metadatarule"))
ds.removeChild(i);
else
i++;
}
// Find out how many children were sent
int pathCount = Integer.parseInt(x);
// Gather up these
i = 0;
while (i < pathCount)
{
String pathDescription = "_"+Integer.toString(i);
String pathOpName = "metaop"+pathDescription;
x = variableContext.getParameter(pathOpName);
if (x != null && x.equals("Delete"))
{
// Skip to the next
i++;
continue;
}
// Get the stored information for this rule.
String path = variableContext.getParameter("metapath"+pathDescription);
String action = variableContext.getParameter("metaflav"+pathDescription);
String allmetadata = variableContext.getParameter("metaall"+pathDescription);
String[] metadataFields = variableContext.getParameterValues("metafields"+pathDescription);
SpecificationNode node = new SpecificationNode("metadatarule");
node.setAttribute("match",path);
node.setAttribute("action",action);
if (action.equals("include"))
{
if (allmetadata != null)
node.setAttribute("allmetadata",allmetadata);
if (metadataFields != null)
{
int j = 0;
while (j < metadataFields.length)
{
SpecificationNode sn = new SpecificationNode("metafield");
sn.setAttribute("value",metadataFields[j]);
node.addChild(j++,sn);
}
}
}
if (x != null && x.equals("Insert Here"))
{
// Insert the new global rule information now
path = variableContext.getParameter("metapath");
action = variableContext.getParameter("metaflavor");
allmetadata = variableContext.getParameter("metaall");
metadataFields = variableContext.getParameterValues("metafields");
SpecificationNode sn = new SpecificationNode("metadatarule");
sn.setAttribute("match",path);
sn.setAttribute("action",action);
if (action.equals("include"))
{
if (allmetadata != null)
node.setAttribute("allmetadata",allmetadata);
if (metadataFields != null)
{
int j = 0;
while (j < metadataFields.length)
{
SpecificationNode node2 = new SpecificationNode("metafield");
node2.setAttribute("value",metadataFields[j]);
sn.addChild(j++,node2);
}
}
}
ds.addChild(ds.getChildCount(),sn);
}
ds.addChild(ds.getChildCount(),node);
i++;
}
// See if there's a global path rule operation
String op = variableContext.getParameter("metaop");
if (op != null)
{
if (op.equals("Add"))
{
String match = variableContext.getParameter("metapath");
String action = variableContext.getParameter("metaflavor");
SpecificationNode node = new SpecificationNode("metadatarule");
node.setAttribute("match",match);
node.setAttribute("action",action);
if (action.equals("include"))
{
String allmetadata = variableContext.getParameter("metaall");
String[] metadataFields = variableContext.getParameterValues("metafields");
if (allmetadata != null)
node.setAttribute("allmetadata",allmetadata);
if (metadataFields != null)
{
int j = 0;
while (j < metadataFields.length)
{
SpecificationNode sn = new SpecificationNode("metafield");
sn.setAttribute("value",metadataFields[j]);
node.addChild(j++,sn);
}
}
}
ds.addChild(ds.getChildCount(),node);
}
}
// See if there's a global pathbuilder operation
String pathop = variableContext.getParameter("metapathop");
if (pathop != null)
{
if (pathop.equals("Reset"))
{
currentContext.save("metapath","/");
currentContext.save("metapathstate","site");
currentContext.save("metapathlibrary",null);
}
else if (pathop.equals("AppendSite"))
{
String path = variableContext.getParameter("metapath");
String addon = variableContext.getParameter("metasite");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
}
currentContext.save("metapath",path);
currentContext.save("metapathstate","site");
currentContext.save("metapathlibrary",null);
}
else if (pathop.equals("AppendLibrary"))
{
String path = variableContext.getParameter("metapath");
String addon = variableContext.getParameter("metalibrary");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
currentContext.save("metapathstate","library");
currentContext.save("metapathlibrary",path);
}
currentContext.save("metapath",path);
}
else if (pathop.equals("AppendList"))
{
String path = variableContext.getParameter("metapath");
String addon = variableContext.getParameter("metalist");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
currentContext.save("metapathstate","list");
currentContext.save("metapathlibrary",path);
// Automatically add on wildcard for list item part of the match
path += "/*";
}
currentContext.save("metapath",path);
}
else if (pathop.equals("AppendText"))
{
String path = variableContext.getParameter("metapath");
String library = variableContext.getParameter("metapathlibrary");
String addon = variableContext.getParameter("metamatch");
if (addon != null && addon.length() > 0)
{
if (path.equals("/"))
path = path + addon;
else
path = path + "/" + addon;
if (library != null)
currentContext.save("metapathstate","file");
else
currentContext.save("metapathstate","unknown");
}
currentContext.save("metapath",path);
currentContext.save("metapathlibrary",library);
}
else if (pathop.equals("Remove"))
{
String pathState = variableContext.getParameter("metapathstate");
String path;
if (pathState.equals("file"))
{
pathState = "library";
path = variableContext.getParameter("metapathlibrary");
}
else if (pathState.equals("list") || pathState.equals("library"))
{
pathState = "site";
path = variableContext.getParameter("metapathlibrary");
int index = path.lastIndexOf("/");
path = path.substring(0,index);
if (path.length() == 0)
path = "/";
currentContext.save("metapathlibrary",null);
}
else
{
path = variableContext.getParameter("metapath");
int index = path.lastIndexOf("/");
path = path.substring(0,index);
if (path.length() == 0)
path = "/";
}
currentContext.save("metapathstate",pathState);
currentContext.save("metapath",path);
}
}
}
String xc = variableContext.getParameter("specsecurity");
if (xc != null)
{
// Delete all security entries first
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("security"))
ds.removeChild(i);
else
i++;
}
SpecificationNode node = new SpecificationNode("security");
node.setAttribute("value",xc);
ds.addChild(ds.getChildCount(),node);
}
xc = variableContext.getParameter("tokencount");
if (xc != null)
{
// Delete all file specs first
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("access"))
ds.removeChild(i);
else
i++;
}
int accessCount = Integer.parseInt(xc);
i = 0;
while (i < accessCount)
{
String accessDescription = "_"+Integer.toString(i);
String accessOpName = "accessop"+accessDescription;
xc = variableContext.getParameter(accessOpName);
if (xc != null && xc.equals("Delete"))
{
// Next row
i++;
continue;
}
// Get the stuff we need
String accessSpec = variableContext.getParameter("spectoken"+accessDescription);
SpecificationNode node = new SpecificationNode("access");
node.setAttribute("token",accessSpec);
ds.addChild(ds.getChildCount(),node);
i++;
}
String op = variableContext.getParameter("accessop");
if (op != null && op.equals("Add"))
{
String accessspec = variableContext.getParameter("spectoken");
SpecificationNode node = new SpecificationNode("access");
node.setAttribute("token",accessspec);
ds.addChild(ds.getChildCount(),node);
}
}
xc = variableContext.getParameter("specpathnameattribute");
if (xc != null)
{
// Delete old one
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("pathnameattribute"))
ds.removeChild(i);
else
i++;
}
if (xc.length() > 0)
{
SpecificationNode node = new SpecificationNode("pathnameattribute");
node.setAttribute("value",xc);
ds.addChild(ds.getChildCount(),node);
}
}
xc = variableContext.getParameter("specmappingcount");
if (xc != null)
{
// Delete old spec
int i = 0;
while (i < ds.getChildCount())
{
SpecificationNode sn = ds.getChild(i);
if (sn.getType().equals("pathmap"))
ds.removeChild(i);
else
i++;
}
// Now, go through the data and assemble a new list.
int mappingCount = Integer.parseInt(xc);
// Gather up these
i = 0;
while (i < mappingCount)
{
String pathDescription = "_"+Integer.toString(i);
String pathOpName = "specmappingop"+pathDescription;
xc = variableContext.getParameter(pathOpName);
if (xc != null && xc.equals("Delete"))
{
// Skip to the next
i++;
continue;
}
// Inserts won't happen until the very end
String match = variableContext.getParameter("specmatch"+pathDescription);
String replace = variableContext.getParameter("specreplace"+pathDescription);
SpecificationNode node = new SpecificationNode("pathmap");
node.setAttribute("match",match);
node.setAttribute("replace",replace);
ds.addChild(ds.getChildCount(),node);
i++;
}
// Check for add
xc = variableContext.getParameter("specmappingop");
if (xc != null && xc.equals("Add"))
{
String match = variableContext.getParameter("specmatch");
String replace = variableContext.getParameter("specreplace");
SpecificationNode node = new SpecificationNode("pathmap");
node.setAttribute("match",match);
node.setAttribute("replace",replace);
ds.addChild(ds.getChildCount(),node);
}
}
return null;
}
/** View specification.
* This method is called in the body section of a job's view page. Its purpose is to present the document specification information to the user.
* The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
*@param out is the output to which any HTML should be sent.
*@param ds is the current document specification for this job.
*/
@Override
public void viewSpecification(IHTTPOutput out, Locale locale, DocumentSpecification ds)
throws ManifoldCFException, IOException
{
Map<String,Object> velocityContext = new HashMap<String,Object>();
fillInSecurityTab(velocityContext,out,ds);
fillInPathsTab(velocityContext,out,ds);
fillInMetadataTab(velocityContext,out,ds);
Messages.outputResourceWithVelocity(out,locale,"viewSpecification.html",velocityContext);
}
protected static class ExecuteMethodThread extends Thread
{
protected final HttpClient httpClient;
protected final String url;
protected final OutputStream os;
protected Throwable exception = null;
protected int returnCode = 0;
public ExecuteMethodThread( HttpClient httpClient, String url, OutputStream os )
{
super();
setDaemon(true);
this.httpClient = httpClient;
this.url = url;
this.os = os;
}
public void run()
{
try
{
HttpGet method = new HttpGet( url );
// Try block to insure that the connection gets cleaned up
try
{
// Begin the fetch
HttpResponse response = httpClient.execute(method);
returnCode = response.getStatusLine().getStatusCode();
if (returnCode == 200)
{
// Process the data
HttpEntity entity = response.getEntity();
if (entity != null)
{
InputStream is = entity.getContent();
// Figure out what to do with the data.
byte[] transferBuffer = new byte[65536];
while (true)
{
int amt = is.read(transferBuffer);
if (amt == -1)
break;
os.write(transferBuffer,0,amt);
}
}
}
}
finally
{
// Consumes and closes the stream, releasing the connection
method.abort();
}
}
catch (Throwable e)
{
this.exception = e;
}
}
public int finishUp()
throws InterruptedException, IOException, org.apache.http.HttpException
{
join();
if (exception != null)
{
if (exception instanceof IOException)
throw (IOException)exception;
else if (exception instanceof Error)
throw (Error)exception;
else if (exception instanceof org.apache.http.HttpException)
throw (org.apache.http.HttpException)exception;
else if (exception instanceof RuntimeException)
throw (RuntimeException)exception;
else
throw new RuntimeException("Unexpected exception type thrown: "+exception.getClass().getName());
}
return returnCode;
}
}
/**
* Gets a list of field names of the given document library or list.
* @param parentSite - parent site path
* @param docLibrary name
* @return list of the fields
*/
public Map<String,String> getLibFieldList( String parentSite, String docLibrary )
throws ServiceInterruption, ManifoldCFException
{
getSession();
return proxy.getFieldList( encodePath(parentSite), proxy.getDocLibID( encodePath(parentSite), parentSite, docLibrary ) );
}
/**
* Gets a list of field names of the given document library or list.
* @param parentSite - parent site path
* @param docLibrary name
* @return list of the fields
*/
public Map<String,String> getListFieldList( String parentSite, String listName )
throws ServiceInterruption, ManifoldCFException
{
getSession();
return proxy.getFieldList( encodePath(parentSite), proxy.getListID( encodePath(parentSite), parentSite, listName ) );
}
/**
* Gets a list of sites/subsites of the given parent site
* @param parentSite the unencoded parent site path to search for subsites, empty for root.
* @return list of the sites
*/
public List<NameValue> getSites( String parentSite )
throws ServiceInterruption, ManifoldCFException
{
getSession();
return proxy.getSites( encodePath(parentSite) );
}
/**
* Gets a list of document libraries of the given parent site
* @param parentSite the unencoded parent site to search for libraries, empty for root.
* @return list of the libraries
*/
public List<NameValue> getDocLibsBySite( String parentSite )
throws ManifoldCFException, ServiceInterruption
{
getSession();
return proxy.getDocumentLibraries( encodePath(parentSite), parentSite );
}
/**
* Gets a list of lists of the given parent site
* @param parentSite the unencoded parent site to search for lists, empty for root.
* @return list of the lists
*/
public List<NameValue> getListsBySite( String parentSite )
throws ManifoldCFException, ServiceInterruption
{
getSession();
return proxy.getLists( encodePath(parentSite), parentSite );
}
// Protected static methods
/** Check if a library should be included, given a document specification.
*@param libraryPath is the unencoded canonical library name (including site path from root site), without any starting slash.
*@param documentSpecification is the specification.
*@return true if it should be included.
*/
protected boolean checkIncludeLibrary( String libraryPath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include library '" + libraryPath + "'" );
// Scan the specification, looking for the old-style "startpoint" matches and the new-style "libraryrule" matches.
int i = 0;
while (i < documentSpecification.getChildCount())
{
SpecificationNode sn = documentSpecification.getChild(i++);
if ( sn.getType().equals("startpoint") )
{
// Old style rule!
String site = sn.getAttributeValue( "site" );
String lib = sn.getAttributeValue( "lib" );
// Both site and lib are unencoded. See if they match the library path
String pathStart = site + "/" + lib;
// Old-style matches have a preceding "/" when there's no subsite...
if (libraryPath.equals(pathStart))
{
// Hey, the startpoint rule matches! It's an implicit inclusion, so we don't need to do anything else except return.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library path '"+libraryPath+"' matched old-style startpoint with site '"+site+"' and library '"+lib+"' - including");
return true;
}
}
else if (sn.getType().equals("pathrule"))
{
// New-style rule.
// Here's the trick: We do what the first matching rule tells us to do.
String pathMatch = sn.getAttributeValue("match");
String action = sn.getAttributeValue("action");
String ruleType = sn.getAttributeValue("type");
// First, find out if we match EXACTLY.
if (checkMatch(libraryPath,0,pathMatch))
{
// If this is true, the type also has to match if the rule is to apply.
if (ruleType.equals("library"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library '"+libraryPath+"' exactly matched rule path '"+pathMatch+"'");
if (action.equals("include"))
{
// For include rules, partial match is good enough to proceed.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Including library '"+libraryPath+"'");
return true;
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Excluding library '"+libraryPath+"'");
return false;
}
}
else if (ruleType.equals("file") && checkPartialPathMatch(libraryPath,0,pathMatch,1) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library '"+libraryPath+"' partially matched file rule path '"+pathMatch+"' - including");
return true;
}
else if (ruleType.equals("folder") && checkPartialPathMatch(libraryPath,0,pathMatch,1) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Library '"+libraryPath+"' partially matched folder rule path '"+pathMatch+"' - including");
return true;
}
}
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Not including library '"+libraryPath+"' because no matching rule");
return false;
}
/** Check if a list should be included, given a document specification.
*@param listPath is the unencoded canonical list name (including site path from root site), without any starting slash.
*@param documentSpecification is the specification.
*@return true if it should be included.
*/
protected boolean checkIncludeList( String listPath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include list '" + listPath + "'" );
// Scan the specification, looking for new-style "pathrule" matches.
int i = 0;
while (i < documentSpecification.getChildCount())
{
SpecificationNode sn = documentSpecification.getChild(i++);
if (sn.getType().equals("pathrule"))
{
// New-style rule.
// Here's the trick: We do what the first matching rule tells us to do.
String pathMatch = sn.getAttributeValue("match");
String action = sn.getAttributeValue("action");
String ruleType = sn.getAttributeValue("type");
// First, find out if we match EXACTLY.
if (checkMatch(listPath,0,pathMatch))
{
// If this is true, the type also has to match if the rule is to apply.
if (ruleType.equals("list"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: List '"+listPath+"' exactly matched rule path '"+pathMatch+"'");
if (action.equals("include"))
{
// For include rules, partial match is good enough to proceed.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Including list '"+listPath+"'");
return true;
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Excluding list '"+listPath+"'");
return false;
}
}
}
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Not including list '"+listPath+"' because no matching rule");
return false;
}
/** Check if a site should be included, given a document specification.
*@param sitePath is the unencoded canonical site path name from the root site level, without any starting slash.
*@param documentSpecification is the specification.
*@return true if it should be included.
*/
protected boolean checkIncludeSite( String sitePath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include site '" + sitePath + "'" );
// Scan the specification, looking for the old-style "startpoint" matches and the new-style "libraryrule" matches.
int i = 0;
while (i < documentSpecification.getChildCount())
{
SpecificationNode sn = documentSpecification.getChild(i++);
if ( sn.getType().equals("startpoint") )
{
// Old style rule!
String site = sn.getAttributeValue( "site" );
// Both site and lib are unencoded. See if they match part of the site path.
// Note well: We want a complete subsection match! That is, what's left in the path after the match must
// either start with "/" or be empty.
if (!site.startsWith("/"))
site = "/" + site;
// Old-style matches have a preceding "/" when there's no subsite...
if (site.startsWith(sitePath))
{
if (sitePath.length() == 1 || site.length() == sitePath.length() || site.charAt(sitePath.length()) == '/')
{
// Hey, the startpoint rule matches! It's an implicit inclusion, so we don't need to do anything else except return.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site path '"+sitePath+"' matched old-style startpoint with site '"+site+"' - including");
return true;
}
}
}
else if (sn.getType().equals("pathrule"))
{
// New-style rule.
String pathMatch = sn.getAttributeValue("match");
String action = sn.getAttributeValue("action");
String ruleType = sn.getAttributeValue("type");
// First, find out if we match EXACTLY.
if (checkMatch(sitePath,0,pathMatch))
{
// If this is true, the type also has to match if the rule is to apply.
if (ruleType.equals("site"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' exactly matched rule path '"+pathMatch+"'");
if (action.equals("include"))
{
// For include rules, partial match is good enough to proceed.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Including site '"+sitePath+"'");
return true;
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Excluding site '"+sitePath+"'");
return false;
}
}
else if (ruleType.equals("library") && checkPartialPathMatch(sitePath,0,pathMatch,1) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' partially matched library rule path '"+pathMatch+"' - including");
return true;
}
else if (ruleType.equals("list") && checkPartialPathMatch(sitePath,0,pathMatch,1) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' partially matched list rule path '"+pathMatch+"' - including");
return true;
}
else if (ruleType.equals("site") && checkPartialPathMatch(sitePath,0,pathMatch,0) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' partially matched site rule path '"+pathMatch+"' - including");
return true;
}
else if (ruleType.equals("file") && checkPartialPathMatch(sitePath,0,pathMatch,2) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' partially matched file rule path '"+pathMatch+"' - including");
return true;
}
else if (ruleType.equals("folder") && checkPartialPathMatch(sitePath,0,pathMatch,2) && action.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Site '"+sitePath+"' partially matched folder rule path '"+pathMatch+"' - including");
return true;
}
}
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Not including site '"+sitePath+"' because no matching rule");
return false;
}
/** Get a file or item's metadata specification, given a path and a document specification.
*@param filePath is the unencoded path to a file or item, including sites and library/list, beneath the root site.
*@param documentSpecification is the document specification.
*@return the metadata description appropriate to the file.
*/
protected MetadataInformation getMetadataSpecification( String filePath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Finding metadata to include for document/item '" + filePath + "'." );
MetadataInformation rval = new MetadataInformation();
// Scan the specification, looking for the old-style "startpoint" matches and the new-style "metadatarule" matches.
int i = 0;
while (i < documentSpecification.getChildCount())
{
SpecificationNode sn = documentSpecification.getChild(i++);
if ( sn.getType().equals("startpoint") )
{
// Old style rule!
String site = sn.getAttributeValue( "site" );
String lib = sn.getAttributeValue( "lib" );
// Both site and lib are unencoded. See if they match the first part of the filepath
String pathStart = site + "/" + lib + "/";
// Old-style matches have a preceding "/" when there's no subsite...
if (filePath.startsWith(pathStart))
{
// Hey, the startpoint rule matches! It's an implicit inclusion, so this is where we get the metadata from (and then return)
String allmetadata = sn.getAttributeValue("allmetadata");
if (allmetadata != null && allmetadata.equals("true"))
rval.setAllMetadata();
else
{
// Scan children looking for metadata nodes
int j = 0;
while (j < sn.getChildCount())
{
SpecificationNode node = sn.getChild(j++);
if (node.getType().equals("metafield"))
rval.addMetadataField(node.getAttributeValue("value"));
}
}
return rval;
}
}
else if (sn.getType().equals("metadatarule"))
{
// New-style rule.
// Here's the trick: We do what the first matching rule tells us to do.
String pathMatch = sn.getAttributeValue("match");
// First, find out if we match...
if (checkMatch(filePath,0,pathMatch))
{
// The rule "fired". Now, do what it tells us to.
String action = sn.getAttributeValue("action");
if (action.equals("include"))
{
// Include: Process the metadata specification, then return
String allMetadata = sn.getAttributeValue("allmetadata");
if (allMetadata != null && allMetadata.equals("true"))
rval.setAllMetadata();
else
{
// Scan children looking for metadata nodes
int j = 0;
while (j < sn.getChildCount())
{
SpecificationNode node = sn.getChild(j++);
if (node.getType().equals("metafield"))
rval.addMetadataField(node.getAttributeValue("value"));
}
}
}
return rval;
}
}
}
return rval;
}
/** Check if a file should be included.
*@param filePath is the path to the file, including sites and library, beneath the root site.
*@param documentSpecification is the document specification.
*@return true if file should be included.
*/
protected boolean checkIncludeFile( String filePath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include document '" + filePath + "'" );
// Break up the file/folder part of the path
int lastSlash = filePath.lastIndexOf("/");
String pathPart = filePath.substring(0,lastSlash);
String filePart = filePath.substring(lastSlash+1);
// Scan the spec rules looking for a library match, and extract the information if found.
// We need to understand both the old-style rules (startpoints), and the new style (matchrules)
int i = 0;
while (i < documentSpecification.getChildCount())
{
SpecificationNode sn = documentSpecification.getChild(i++);
if ( sn.getType().equals("startpoint") )
{
// Old style rule!
String site = sn.getAttributeValue( "site" );
String lib = sn.getAttributeValue( "lib" );
// Both site and lib are unencoded. The string we are matching starts with "/" if the site is empty.
String pathMatch = site + "/" + lib + "/";
if (filePath.startsWith(pathMatch))
{
// Hey, it matched!
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File path '"+filePath+"' matched old-style startpoint with site '"+site+"' and library '"+lib+"'");
int restOfPathIndex = pathMatch.length();
// We need to walk through the subrules and see whether it's in or out.
int j = 0;
while (j < sn.getChildCount())
{
SpecificationNode node = sn.getChild(j++);
String flavor = node.getType();
if (flavor.equals("include") || flavor.equals("exclude"))
{
String match = node.getAttributeValue("match");
String type = node.getAttributeValue("type");
String sourceMatch;
int sourceIndex;
if ( type.equals("file") )
{
sourceMatch = filePart;
sourceIndex = 0;
}
else
{
sourceMatch = pathPart;
sourceIndex = restOfPathIndex;
}
if ( checkMatch(sourceMatch,sourceIndex,match) )
{
// Our file path matched the rule.
if (flavor.equals("include"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File path '"+filePath+"' matched old-style inclusion rule '"+match+"' - including");
return true;
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File path '"+filePath+"' matched old-style exclusion rule '"+match+"' - excluding");
return false;
}
}
}
// Didn't match any of the file rules; therefore exclude.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File path '"+filePath+"' did not match any old-style inclusion/exclusion rules - excluding");
return false;
}
}
else if (sn.getType().equals("pathrule"))
{
// New style rule!
String pathMatch = sn.getAttributeValue("match");
String action = sn.getAttributeValue("action");
String ruleType = sn.getAttributeValue("type");
// Find out if we match EXACTLY. There are no "partial matches" for files.
if (checkMatch(filePath,0,pathMatch))
{
// If this is true, the type also has to match if the rule is to apply.
if (ruleType.equals("file"))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File '"+filePath+"' exactly matched rule path '"+pathMatch+"'");
if (action.equals("include"))
{
// For include rules, partial match is good enough to proceed.
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Including file '"+filePath+"'");
return true;
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: Excluding file '"+filePath+"'");
return false;
}
}
}
}
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: File path '"+filePath+"' does not match any rules - excluding");
return false;
}
/** Check if a list item attachment should be included.
*@param attachmentPath is the path to the attachment, including sites and list name, beneath the root site.
*@param documentSpecification is the document specification.
*@return true if file should be included.
*/
protected boolean checkIncludeListItemAttachment( String attachmentPath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include list item attachment '" + attachmentPath + "'" );
// There are no attachment rules, so they are always included
return true;
}
/** Check if a list item should be included.
*@param itemPath is the path to the item, including sites and list name, beneath the root site.
*@param documentSpecification is the document specification.
*@return true if file should be included.
*/
protected boolean checkIncludeListItem( String itemPath, DocumentSpecification documentSpecification )
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug( "SharePoint: Checking whether to include list item '" + itemPath + "'" );
// There are no item rules, so they are always included
return true;
}
/** Match a sub-path. The sub-path must match the complete starting part of the full path, in a path
* sense. The returned value should point into the file name beyond the end of the matched path, or
* be -1 if there is no match.
*@param subPath is the sub path.
*@param fullPath is the full path.
*@return the index of the start of the remaining part of the full path, or -1.
*/
protected static int matchSubPath( String subPath, String fullPath )
{
if ( subPath.length() > fullPath.length() )
return -1;
if ( fullPath.startsWith( subPath ) == false )
return -1;
int rval = subPath.length();
if ( fullPath.length() == rval )
return rval;
char x = fullPath.charAt( rval );
if ( x == '/' )
rval++;
return rval;
}
/** Check for a partial path match between two strings with wildcards.
* Match allowance also must be made for the minimum path components in the rest of the path.
*/
protected static boolean checkPartialPathMatch( String sourceMatch, int sourceIndex, String match, int requiredExtraPathSections )
{
// The partial match must be of a complete path, with at least a specified number of trailing path components possible in what remains.
// Path components can include everything but the "/" character itself.
//
// The match string is the one containing the wildcards. Both the "*" wildcard and the "?" wildcard will match a "/", which is intended but is why this
// matcher is a little tricky to write.
//
// Note also that it is OK to return "true" more than strictly necessary, but it is never OK to return "false" incorrectly.
// This is a partial path match. That means that we don't have to completely use up the match string, but what's left on the match string after the source
// string is used up MUST either be capable of being null, or be capable of starting with a "/"integral path sections, and MUST include at least n of these sections.
//
boolean caseSensitive = true;
if (!sourceMatch.endsWith("/"))
sourceMatch = sourceMatch + "/";
return processPartialPathCheck( caseSensitive, sourceMatch, sourceIndex, match, 0, requiredExtraPathSections );
}
/** Recursive worker method for checkPartialPathMatch. Returns 'true' if there is a path that consumes the source string entirely,
* and leaves the remainder of the match string able to match the required followup.
*@param caseSensitive is true if file names are case sensitive.
*@param sourceMatch is the source string (w/o wildcards)
*@param sourceIndex is the current point in the source string.
*@param match is the match string (w/wildcards)
*@param matchIndex is the current point in the match string.
*@return true if there is a match.
*/
protected static boolean processPartialPathCheck(boolean caseSensitive, String sourceMatch, int sourceIndex, String match, int matchIndex,
int requiredExtraPathSections)
{
// Match up through the next * we encounter
while ( true )
{
// If we've reached the end of the source, verify that it's a match.
if ( sourceMatch.length() == sourceIndex)
{
// The "correct" way to code this is to recursively attempt to generate all different paths that correspond to the required extra sections. However,
// that's computationally very nasty. In practice, we'll simply distinguish between "some" and "none".
// If we've reached the end of the match string too, then it passes (or fails, if we need extra sections)
if (match.length() == matchIndex)
return (requiredExtraPathSections == 0);
// We can match a path separator, so we win
return true;
}
// If we have reached the end of the match (but not the source), match fails
if ( match.length() == matchIndex )
return false;
char x = sourceMatch.charAt( sourceIndex );
char y = match.charAt( matchIndex );
if ( !caseSensitive )
{
if ( x >= 'A' && x <= 'Z' )
x -= 'A'-'a';
if ( y >= 'A' && y <= 'Z' )
y -= 'A'-'a';
}
if ( y == '*' )
{
// Wildcard!
// We will recurse at this point.
// Basically, we want to combine the results for leaving the "*" in the match string
// at this point and advancing the source index, with skipping the "*" and leaving the source
// string alone.
return processPartialPathCheck( caseSensitive, sourceMatch, sourceIndex + 1, match, matchIndex, requiredExtraPathSections ) ||
processPartialPathCheck( caseSensitive, sourceMatch, sourceIndex, match, matchIndex + 1, requiredExtraPathSections );
}
if ( y == '?' || x == y )
{
sourceIndex++;
matchIndex++;
}
else
return false;
}
}
/** Check a match between two strings with wildcards.
*@param sourceMatch is the expanded string (no wildcards)
*@param sourceIndex is the starting point in the expanded string.
*@param match is the wildcard-based string.
*@return true if there is a match.
*/
protected static boolean checkMatch( String sourceMatch, int sourceIndex, String match )
{
// Note: The java regex stuff looks pretty heavyweight for this purpose.
// I've opted to try and do a simple recursive version myself, which is not compiled.
// Basically, the match proceeds by recursive descent through the string, so that all *'s cause
// recursion.
boolean caseSensitive = true;
return processCheck( caseSensitive, sourceMatch, sourceIndex, match, 0 );
}
/** Recursive worker method for checkMatch. Returns 'true' if there is a path that consumes both
* strings in their entirety in a matched way.
*@param caseSensitive is true if file names are case sensitive.
*@param sourceMatch is the source string (w/o wildcards)
*@param sourceIndex is the current point in the source string.
*@param match is the match string (w/wildcards)
*@param matchIndex is the current point in the match string.
*@return true if there is a match.
*/
protected static boolean processCheck(boolean caseSensitive, String sourceMatch, int sourceIndex, String match, int matchIndex )
{
// Match up through the next * we encounter
while ( true )
{
// If we've reached the end, it's a match.
if ( sourceMatch.length() == sourceIndex && match.length() == matchIndex )
return true;
// If one has reached the end but the other hasn't, no match
if ( match.length() == matchIndex )
return false;
if ( sourceMatch.length() == sourceIndex )
{
if ( match.charAt(matchIndex) != '*' )
return false;
matchIndex++;
continue;
}
char x = sourceMatch.charAt( sourceIndex );
char y = match.charAt( matchIndex );
if ( !caseSensitive )
{
if ( x >= 'A' && x <= 'Z' )
x -= 'A'-'a';
if ( y >= 'A' && y <= 'Z' )
y -= 'A'-'a';
}
if ( y == '*' )
{
// Wildcard!
// We will recurse at this point.
// Basically, we want to combine the results for leaving the "*" in the match string
// at this point and advancing the source index, with skipping the "*" and leaving the source
// string alone.
return processCheck( caseSensitive, sourceMatch, sourceIndex + 1, match, matchIndex ) ||
processCheck( caseSensitive, sourceMatch, sourceIndex, match, matchIndex + 1 );
}
if ( y == '?' || x == y )
{
sourceIndex++;
matchIndex++;
}
else
return false;
}
}
/** Grab forced acl out of document specification.
*@param spec is the document specification.
*@return the acls.
*/
protected static String[] getAcls(DocumentSpecification spec)
{
HashMap map = new HashMap();
int i = 0;
boolean securityOn = true;
while (i < spec.getChildCount())
{
SpecificationNode sn = spec.getChild(i++);
if (sn.getType().equals("access"))
{
String token = sn.getAttributeValue("token");
map.put(token,token);
}
else if (sn.getType().equals("security"))
{
String value = sn.getAttributeValue("value");
if (value.equals("on"))
securityOn = true;
else if (value.equals("off"))
securityOn = false;
}
}
if (!securityOn)
return null;
String[] rval = new String[map.size()];
Iterator iter = map.keySet().iterator();
i = 0;
while (iter.hasNext())
{
rval[i++] = (String)iter.next();
}
return rval;
}
/** Decode a path item.
*/
public static String pathItemDecode(String pathItem)
{
return URLDecoder.decode(pathItem.replaceAll("\\%20","+"));
}
/** Encode a path item.
*/
public static String pathItemEncode(String pathItem)
{
String output = URLEncoder.encode(pathItem);
return output.replaceAll("\\+","%20");
}
/** Given a path that is /-separated, and otherwise encoded, decode properly to convert to
* unencoded form.
*/
public static String decodePath(String relPath)
{
StringBuilder sb = new StringBuilder();
String[] pathEntries = relPath.split("/");
int k = 0;
boolean isFirst = true;
while (k < pathEntries.length)
{
if (isFirst)
isFirst = false;
else
sb.append("/");
sb.append(pathItemDecode(pathEntries[k++]));
}
return sb.toString();
}
/** Given a path that is /-separated, and otherwise unencoded, encode properly for an actual
* URI
*/
public static String encodePath(String relPath)
{
StringBuilder sb = new StringBuilder();
String[] pathEntries = relPath.split("/");
int k = 0;
boolean isFirst = true;
while (k < pathEntries.length)
{
if (isFirst)
isFirst = false;
else
sb.append("/");
sb.append(pathItemEncode(pathEntries[k++]));
}
return sb.toString();
}
/** Metadata information gleaned from document paths and specification.
*/
protected static class MetadataInformation
{
protected boolean allMetadata = false;
protected HashMap metadataFields = new HashMap();
/** Constructor */
public MetadataInformation()
{
}
/** Set "all metadata" */
public void setAllMetadata()
{
allMetadata = true;
}
/** Add a metadata field */
public void addMetadataField(String fieldName)
{
metadataFields.put(fieldName,fieldName);
}
/** Get whether "all metadata" is to be used */
public boolean getAllMetadata()
{
return allMetadata;
}
/** Get the set of metadata fields to use */
public String[] getMetadataFields()
{
String[] rval = new String[metadataFields.size()];
Iterator iter = metadataFields.keySet().iterator();
int i = 0;
while (iter.hasNext())
{
rval[i++] = (String)iter.next();
}
return rval;
}
}
/** Class that tracks paths associated with id's, and the name
* of the metadata attribute to use for the path.
*/
protected class SystemMetadataDescription
{
// The path attribute name
protected String pathAttributeName;
// The path name map
protected MatchMap matchMap = new MatchMap();
/** Constructor */
public SystemMetadataDescription(DocumentSpecification spec)
throws ManifoldCFException
{
pathAttributeName = null;
int i = 0;
while (i < spec.getChildCount())
{
SpecificationNode n = spec.getChild(i++);
if (n.getType().equals("pathnameattribute"))
pathAttributeName = n.getAttributeValue("value");
else if (n.getType().equals("pathmap"))
{
String pathMatch = n.getAttributeValue("match");
String pathReplace = n.getAttributeValue("replace");
matchMap.appendMatchPair(pathMatch,pathReplace);
}
}
}
/** Get the path attribute name.
*@return the path attribute name, or null if none specified.
*/
public String getPathAttributeName()
{
return pathAttributeName;
}
/** Given an identifier, get the translated string that goes into the metadata.
*/
public String getPathAttributeValue(String documentIdentifier)
throws ManifoldCFException
{
String path = getPathString(documentIdentifier);
return matchMap.translate(path);
}
/** For a given id, get the portion of its path which the mapping and ingestion
* should go against. Effectively this should include the whole identifer, so this
* is easy to calculate.
*/
public String getPathString(String documentIdentifier)
throws ManifoldCFException
{
// There will be a "//" somewhere in the string. Remove it!
int dslashIndex = documentIdentifier.indexOf("//");
if (dslashIndex == -1)
return documentIdentifier;
return documentIdentifier.substring(0,dslashIndex) + documentIdentifier.substring(dslashIndex+1);
}
}
}