/**
* Licensed to Jasig under one or more contributor license
* agreements. See the NOTICE file distributed with this work
* for additional information regarding copyright ownership.
* Jasig licenses this file to you under the Apache License,
* Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a
* copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jasig.portal.io.xml;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.StartElement;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stax.StAXSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
import org.apache.commons.compress.archivers.jar.JarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.io.CloseShieldInputStream;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tools.ant.DirectoryScanner;
import org.jasig.portal.concurrency.CallableWithoutResult;
import org.jasig.portal.utils.AntPatternFileFilter;
import org.jasig.portal.utils.ConcurrentDirectoryScanner;
import org.jasig.portal.utils.PeriodicFlushingBufferedWriter;
import org.jasig.portal.utils.ResourceUtils;
import org.jasig.portal.utils.SafeFilenameUtils;
import org.jasig.portal.xml.StaxUtils;
import org.jasig.portal.xml.XmlUtilities;
import org.jasig.portal.xml.XmlUtilitiesImpl;
import org.jasig.portal.xml.stream.BufferedXMLEventReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ResourceLoaderAware;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.dao.support.DataAccessUtils;
import org.springframework.oxm.Marshaller;
import org.springframework.oxm.Unmarshaller;
import org.springframework.oxm.XmlMappingException;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
import com.google.common.io.InputSupplier;
/**
* Pulls together {@link IPortalDataType}, {@link IDataUpgrader}, and {@link IDataImporter}
* implementations to handle data upgrade, import, export and removal operations.
*
* @author Eric Dalquist
* @version $Revision$
*/
@Service("portalDataHandlerService")
public class JaxbPortalDataHandlerService implements IPortalDataHandlerService, ResourceLoaderAware {
/**
* Tracks the base import directory to allow for easier to read logging when importing
*/
private static final ThreadLocal<String> IMPORT_BASE_DIR = new ThreadLocal<String>();
private static final String REPORT_FORMAT = "%s,%s,%.2fms\n";
private static final MediaType MT_JAVA_ARCHIVE = MediaType.application("java-archive");
private static final MediaType MT_CPIO = MediaType.application("x-cpio");
private static final MediaType MT_AR = MediaType.application("x-archive");
private static final MediaType MT_TAR = MediaType.application("x-tar");
private static final MediaType MT_BZIP2 = MediaType.application("x-bzip2");
private static final MediaType MT_GZIP = MediaType.application("x-gzip");
private static final MediaType MT_PACK200 = MediaType.application("x-java-pack200");
private static final MediaType MT_XZ = MediaType.application("x-xz");
protected final Logger logger = LoggerFactory.getLogger(getClass());
// Order in which data must be imported
private List<PortalDataKey> dataKeyImportOrder = Collections.emptyList();
// Map to lookup the associated IPortalDataType for each known PortalDataKey
private Map<PortalDataKey, IPortalDataType> dataKeyTypes = Collections.emptyMap();
// Ant path matcher patterns that a file must match when scanning directories (unless a pattern is explicitly specified)
private Set<String> dataFileIncludes = Collections.emptySet();
private Set<String> dataFileExcludes = ImmutableSet.copyOf(DirectoryScanner.getDefaultExcludes());
// Data upgraders mapped by PortalDataKey
private Map<PortalDataKey, IDataUpgrader> portalDataUpgraders = Collections.emptyMap();
// Data importers mapped by PortalDataKey
private Map<PortalDataKey, IDataImporter<Object>> portalDataImporters = Collections.emptyMap();
// ExportAll data types
private Set<IPortalDataType> exportAllPortalDataTypes = null;
// All portal data types available for export
private Set<IPortalDataType> exportPortalDataTypes = Collections.emptySet();
// Data exporters mapped by IPortalDateType#getTypeId()
private Map<String, IDataExporter<Object>> portalDataExporters = Collections.emptyMap();
// All portal data types available for delete
private Set<IPortalDataType> deletePortalDataTypes = Collections.emptySet();
// Data deleters mapped by IPortalDateType#getTypeId()
private Map<String, IDataDeleter<Object>> portalDataDeleters = Collections.emptyMap();
private org.jasig.portal.utils.DirectoryScanner directoryScanner;
private ExecutorService importExportThreadPool;
private XmlUtilities xmlUtilities;
private ResourceLoader resourceLoader;
private long maxWait = -1;
private TimeUnit maxWaitTimeUnit = TimeUnit.MILLISECONDS;
@Autowired
public void setXmlUtilities(XmlUtilities xmlUtilities) {
this.xmlUtilities = xmlUtilities;
}
@Autowired
public void setImportExportThreadPool(@Qualifier("importExportThreadPool") ExecutorService importExportThreadPool) {
this.importExportThreadPool = importExportThreadPool;
this.directoryScanner = new ConcurrentDirectoryScanner(this.importExportThreadPool);
}
/**
* Maximum time to wait for an import, export, or delete to execute.
*/
public void setMaxWait(long maxWait) {
this.maxWait = maxWait;
}
/**
* {@link TimeUnit} for {@link #setMaxWait(long)} value.
*/
public void setMaxWaitTimeUnit(TimeUnit maxWaitTimeUnit) {
this.maxWaitTimeUnit = maxWaitTimeUnit;
}
/**
* Order in which data types should be imported.
*/
@javax.annotation.Resource(name="dataTypeImportOrder")
public void setDataTypeImportOrder(List<IPortalDataType> dataTypeImportOrder) {
final ArrayList<PortalDataKey> dataKeyImportOrder = new ArrayList<PortalDataKey>(dataTypeImportOrder.size() * 2);
final Map<PortalDataKey, IPortalDataType> dataKeyTypes = new LinkedHashMap<PortalDataKey, IPortalDataType>(dataTypeImportOrder.size() * 2);
for (final IPortalDataType portalDataType : dataTypeImportOrder) {
final List<PortalDataKey> supportedDataKeys = portalDataType.getDataKeyImportOrder();
for (final PortalDataKey portalDataKey : supportedDataKeys) {
dataKeyImportOrder.add(portalDataKey);
dataKeyTypes.put(portalDataKey, portalDataType);
}
}
dataKeyImportOrder.trimToSize();
this.dataKeyImportOrder = Collections.unmodifiableList(dataKeyImportOrder);
this.dataKeyTypes = Collections.unmodifiableMap(dataKeyTypes);
}
/**
* Ant path matching patterns that files must match to be included
*/
@javax.annotation.Resource(name="dataFileIncludes")
public void setDataFileIncludes(Set<String> dataFileIncludes) {
this.dataFileIncludes = dataFileIncludes;
}
/**
* Ant path matching patterns that exclude matched files. Defaults to {@link DirectoryScanner#addDefaultExcludes()}
*/
public void setDataFileExcludes(Set<String> dataFileExcludes) {
this.dataFileExcludes = dataFileExcludes;
}
/**
* {@link IDataImporter} implementations to delegate import operations to.
*/
@SuppressWarnings("unchecked")
@Autowired(required=false)
public void setDataImporters(Collection<IDataImporter<? extends Object>> dataImporters) {
final Map<PortalDataKey, IDataImporter<Object>> dataImportersMap = new LinkedHashMap<PortalDataKey, IDataImporter<Object>>();
for (final IDataImporter<?> dataImporter : dataImporters) {
try {
final Set<PortalDataKey> importDataKeys = dataImporter.getImportDataKeys();
for (final PortalDataKey importDataKey : importDataKeys) {
this.logger.debug("Registering IDataImporter for '{}' - {}",
new Object[]{importDataKey, dataImporter});
final IDataImporter<Object> existing =
dataImportersMap.put(importDataKey, (IDataImporter<Object>) dataImporter);
if (existing != null) {
this.logger.warn("Duplicate IDataImporter PortalDataKey for {} Replacing {} with {}",
new Object[]{importDataKey, existing, dataImporter});
}
}
} catch (Exception exception) {
logger.error("Failed to register data importer {}.", dataImporter, exception);
}
}
this.portalDataImporters = Collections.unmodifiableMap(dataImportersMap);
}
/**
* {@link IDataExporter} implementations to delegate export operations to.
*/
@SuppressWarnings("unchecked")
@Autowired(required=false)
public void setDataExporters(Collection<IDataExporter<? extends Object>> dataExporters) {
final Map<String, IDataExporter<Object>> dataExportersMap = new LinkedHashMap<String, IDataExporter<Object>>();
final Set<IPortalDataType> portalDataTypes = new LinkedHashSet<IPortalDataType>();
for (final IDataExporter<?> dataExporter : dataExporters) {
try {
final IPortalDataType portalDataType = dataExporter.getPortalDataType();
final String typeId = portalDataType.getTypeId();
this.logger.debug("Registering IDataExporter for '{}' - {}",
new Object[]{typeId, dataExporter});
final IDataExporter<Object> existing =
dataExportersMap.put(typeId, (IDataExporter<Object>) dataExporter);
if (existing != null) {
this.logger.warn("Duplicate IDataExporter typeId for {} Replacing {} with {}",
new Object[]{typeId, existing, dataExporter});
}
portalDataTypes.add(portalDataType);
} catch (Exception exception) {
logger.error("Failed to register data exporter {}.", dataExporter, exception);
}
}
this.portalDataExporters = Collections.unmodifiableMap(dataExportersMap);
this.exportPortalDataTypes = Collections.unmodifiableSet(portalDataTypes);
}
/**
* Optional set of all portal data types to export. If not specified all available portal data types
* will be listed.
*/
@javax.annotation.Resource(name="exportAllPortalDataTypes")
public void setExportAllPortalDataTypes(Set<IPortalDataType> exportAllPortalDataTypes) {
this.exportAllPortalDataTypes = ImmutableSet.copyOf(exportAllPortalDataTypes);
}
/**
* {@link IDataDeleter} implementations to delegate delete operations to.
*/
@SuppressWarnings("unchecked")
@Autowired(required=false)
public void setDataDeleters(Collection<IDataDeleter<? extends Object>> dataDeleters) {
final Map<String, IDataDeleter<Object>> dataDeletersMap = new LinkedHashMap<String, IDataDeleter<Object>>();
final Set<IPortalDataType> portalDataTypes = new LinkedHashSet<IPortalDataType>();
for (final IDataDeleter<?> dataDeleter : dataDeleters) {
try {
final IPortalDataType portalDataType = dataDeleter.getPortalDataType();
final String typeId = portalDataType.getTypeId();
this.logger.debug("Registering IDataDeleter for '{}' - {}",
new Object[]{typeId, dataDeleter});
final IDataDeleter<Object> existing =
dataDeletersMap.put(typeId, (IDataDeleter<Object>) dataDeleter);
if (existing != null) {
this.logger.warn("Duplicate IDataDeleter typeId for {} Replacing {} with {}",
new Object[]{typeId, existing, dataDeleter});
}
portalDataTypes.add(portalDataType);
} catch (Exception exception) {
logger.error("Failed to register data deleter {}.", dataDeleter, exception);
}
}
this.portalDataDeleters = Collections.unmodifiableMap(dataDeletersMap);
this.deletePortalDataTypes = Collections.unmodifiableSet(portalDataTypes);
}
/**
* {@link IDataUpgrader} implementations to delegate upgrade operations to.
*/
@Autowired(required=false)
public void setDataUpgraders(Collection<IDataUpgrader> dataUpgraders) {
final Map<PortalDataKey, IDataUpgrader> dataUpgraderMap = new LinkedHashMap<PortalDataKey, IDataUpgrader>();
for (final IDataUpgrader dataUpgrader : dataUpgraders) {
try {
final Set<PortalDataKey> upgradeDataKeys = dataUpgrader.getSourceDataTypes();
for (final PortalDataKey upgradeDataKey : upgradeDataKeys) {
this.logger.debug("Registering IDataUpgrader for '{}' - {}", upgradeDataKey, dataUpgrader);
final IDataUpgrader existing = dataUpgraderMap.put(upgradeDataKey, dataUpgrader);
if (existing != null) {
this.logger.warn("Duplicate IDataUpgrader PortalDataKey for {} Replacing {} with {}",
new Object[]{upgradeDataKey, existing, dataUpgrader});
}
}
} catch (Exception exception) {
logger.error("Failed to register data upgrader {}.", dataUpgrader, exception);
}
}
this.portalDataUpgraders = Collections.unmodifiableMap(dataUpgraderMap);
}
@Override
public void setResourceLoader(ResourceLoader resourceLoader) {
this.resourceLoader = resourceLoader;
}
@Override
public void importDataArchive(Resource archive, BatchImportOptions options) {
try {
importDataArchive(archive, archive.getInputStream(), options);
}
catch (IOException e) {
throw new RuntimeException("Could not load InputStream for resource: " + archive, e);
}
}
protected void importDataArchive(Resource archive, InputStream resourceStream, BatchImportOptions options) {
BufferedInputStream bufferedResourceStream = null;
try {
//Make sure the stream is buffered
if (resourceStream instanceof BufferedInputStream) {
bufferedResourceStream = (BufferedInputStream)resourceStream;
}
else {
bufferedResourceStream = new BufferedInputStream(resourceStream);
}
//Buffer up to 100MB, bad things will happen if we bust this buffer.
//TODO see if there is a buffered stream that will write to a file once the buffer fills up
bufferedResourceStream.mark(100*1024*1024);
final MediaType type = getMediaType(bufferedResourceStream, archive.getFilename());
if (MT_JAVA_ARCHIVE.equals(type)) {
final ArchiveInputStream archiveStream = new JarArchiveInputStream(bufferedResourceStream);
importDataArchive(archive, archiveStream, options);
}
else if (MediaType.APPLICATION_ZIP.equals(type)) {
final ArchiveInputStream archiveStream = new ZipArchiveInputStream(bufferedResourceStream);
importDataArchive(archive, archiveStream, options);
}
else if (MT_CPIO.equals(type)) {
final ArchiveInputStream archiveStream = new CpioArchiveInputStream(bufferedResourceStream);
importDataArchive(archive, archiveStream, options);
}
else if (MT_AR.equals(type)) {
final ArchiveInputStream archiveStream = new ArArchiveInputStream(bufferedResourceStream);
importDataArchive(archive, archiveStream, options);
}
else if (MT_TAR.equals(type)) {
final ArchiveInputStream archiveStream = new TarArchiveInputStream(bufferedResourceStream);
importDataArchive(archive, archiveStream, options);
}
else if (MT_BZIP2.equals(type)) {
final CompressorInputStream compressedStream = new BZip2CompressorInputStream(bufferedResourceStream);
importDataArchive(archive, compressedStream, options);
}
else if (MT_GZIP.equals(type)) {
final CompressorInputStream compressedStream = new GzipCompressorInputStream(bufferedResourceStream);
importDataArchive(archive, compressedStream, options);
}
else if (MT_PACK200.equals(type)) {
final CompressorInputStream compressedStream = new Pack200CompressorInputStream(bufferedResourceStream);
importDataArchive(archive, compressedStream, options);
}
else if (MT_XZ.equals(type)) {
final CompressorInputStream compressedStream = new XZCompressorInputStream(bufferedResourceStream);
importDataArchive(archive, compressedStream, options);
}
else {
throw new RuntimeException("Unrecognized archive media type: " + type);
}
}
catch (IOException e) {
throw new RuntimeException("Could not load InputStream for resource: " + archive, e);
}
finally {
IOUtils.closeQuietly(bufferedResourceStream);
}
}
/**
* Extracts the archive resource and then runs the batch-import process on it.
*/
protected void importDataArchive(final Resource resource, final ArchiveInputStream resourceStream, BatchImportOptions options) {
final File tempDir = Files.createTempDir();
try {
ArchiveEntry archiveEntry;
while ((archiveEntry = resourceStream.getNextEntry()) != null) {
final File entryFile = new File(tempDir, archiveEntry.getName());
if (archiveEntry.isDirectory()) {
entryFile.mkdirs();
}
else {
entryFile.getParentFile().mkdirs();
Files.copy(new InputSupplier<InputStream>() {
@Override
public InputStream getInput() throws IOException {
return new CloseShieldInputStream(resourceStream);
}
}, entryFile);
}
}
importData(tempDir, null, options);
}
catch (IOException e) {
throw new RuntimeException("Failed to extract data from '" +resource + "' to '" + tempDir + "' for batch import.", e);
}
finally {
FileUtils.deleteQuietly(tempDir);
}
}
protected void importXmlData(final Resource resource,
final BufferedInputStream resourceStream,
final PortalDataKey portalDataKey) {
try {
final String resourceUri = ResourceUtils.getResourceUri(resource);
this.importData(new StreamSource(resourceStream, resourceUri), portalDataKey);
}
finally {
IOUtils.closeQuietly(resourceStream);
}
}
protected MediaType getMediaType(BufferedInputStream inputStream, String fileName) throws IOException {
final TikaInputStream tikaInputStreamStream = TikaInputStream.get(new CloseShieldInputStream(inputStream));
try {
final Detector detector = new DefaultDetector();
final Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, fileName);
final MediaType type = detector.detect(tikaInputStreamStream, metadata);
logger.debug("Determined '{}' for '{}'", type, fileName);
return type;
}
catch (IOException e) {
logger.warn("Failed to determine media type for '" + fileName + "' assuming XML", e);
return null;
}
finally {
IOUtils.closeQuietly(tikaInputStreamStream);
//Reset the buffered stream to make up for anything read by the detector
inputStream.reset();
}
}
@Override
public void importData(File directory, String pattern, final BatchImportOptions options) {
if (!directory.exists()) {
throw new IllegalArgumentException("The specified directory '" + directory + "' does not exist");
}
//Create the file filter to use when searching for files to import
final FileFilter fileFilter;
if (pattern != null) {
fileFilter = new AntPatternFileFilter(true, false, pattern, this.dataFileExcludes);
}
else {
fileFilter = new AntPatternFileFilter(true, false, this.dataFileIncludes, this.dataFileExcludes);
}
//Determine the parent directory to log to
final File logDirectory = determineLogDirectory(options, "import");
//Setup reporting file
final File importReport = new File(logDirectory, "data-import.txt");
final PrintWriter reportWriter;
try {
reportWriter = new PrintWriter(new PeriodicFlushingBufferedWriter(500, new FileWriter(importReport)));
}
catch (IOException e) {
throw new RuntimeException("Failed to create FileWriter for: " + importReport, e);
}
//Convert directory to URI String to provide better logging output
final URI directoryUri = directory.toURI();
final String directoryUriStr = directoryUri.toString();
IMPORT_BASE_DIR.set(directoryUriStr);
try {
//Scan the specified directory for files to import
logger.info("Scanning for files to Import from: {}", directory);
final PortalDataKeyFileProcessor fileProcessor = new PortalDataKeyFileProcessor(this.dataKeyTypes, options);
this.directoryScanner.scanDirectoryNoResults(directory, fileFilter, fileProcessor);
final long resourceCount = fileProcessor.getResourceCount();
logger.info("Found {} files to Import from: {}", resourceCount, directory);
//See if the import should fail on error
final boolean failOnError = options != null ? options.isFailOnError() : true;
//Map of files to import, grouped by type
final ConcurrentMap<PortalDataKey, Queue<Resource>> dataToImport = fileProcessor.getDataToImport();
//Import the data files
for (final PortalDataKey portalDataKey : this.dataKeyImportOrder) {
final Queue<Resource> files = dataToImport.remove(portalDataKey);
if (files == null) {
continue;
}
final Queue<ImportFuture<?>> importFutures = new LinkedList<ImportFuture<?>>();
final List<FutureHolder<?>> failedFutures = new LinkedList<FutureHolder<?>>();
final int fileCount = files.size();
logger.info("Importing {} files of type {}", fileCount, portalDataKey);
reportWriter.println(portalDataKey + "," + fileCount);
while (!files.isEmpty()) {
final Resource file = files.poll();
//Check for completed futures on every iteration, needed to fail as fast as possible on an import exception
final List<FutureHolder<?>> newFailed = waitForFutures(importFutures, reportWriter, logDirectory, false);
failedFutures.addAll(newFailed);
final AtomicLong importTime = new AtomicLong(-1);
//Create import task
final Callable<Object> task = new CallableWithoutResult() {
@Override
protected void callWithoutResult() {
IMPORT_BASE_DIR.set(directoryUriStr);
importTime.set(System.nanoTime());
try {
importData(file, portalDataKey);
}
finally {
importTime.set(System.nanoTime() - importTime.get());
IMPORT_BASE_DIR.remove();
}
}
};
//Submit the import task
final Future<?> importFuture = this.importExportThreadPool.submit(task);
//Add the future for tracking
importFutures.offer(new ImportFuture(importFuture, file, portalDataKey, importTime));
}
//Wait for all of the imports on of this type to complete
final List<FutureHolder<?>> newFailed = waitForFutures(importFutures, reportWriter, logDirectory, true);
failedFutures.addAll(newFailed);
if (failOnError && !failedFutures.isEmpty()) {
throw new RuntimeException(failedFutures.size() + " " + portalDataKey + " entities failed to import.\n\n" +
"\tPer entity exception logs and a full report can be found in " + logDirectory + "\n");
}
reportWriter.flush();
}
if (!dataToImport.isEmpty()) {
throw new IllegalStateException("The following PortalDataKeys are not listed in the dataTypeImportOrder List: " + dataToImport.keySet());
}
logger.info("For a detailed report on the data import see " + importReport);
}
catch (InterruptedException e) {
throw new RuntimeException("Interrupted while waiting for entities to import", e);
}
finally {
IOUtils.closeQuietly(reportWriter);
IMPORT_BASE_DIR.remove();
}
}
/**
* Determine directory to log import/export reports to
*/
private File determineLogDirectory(final BatchOptions options, String operation) {
File logDirectoryParent = options != null ? options.getLogDirectoryParent() : null;
if (logDirectoryParent == null) {
logDirectoryParent = Files.createTempDir();
}
File logDirectory = new File(logDirectoryParent, "data-" + operation + "-reports");
try {
logDirectory = logDirectory.getCanonicalFile();
FileUtils.deleteDirectory(logDirectory);
}
catch (IOException e) {
throw new RuntimeException("Failed to clean data-" + operation + " log directory: " + logDirectory, e);
}
logDirectory.mkdirs();
return logDirectory;
}
/* (non-Javadoc)
* @see org.jasig.portal.io.xml.IDataImportExportService#importData(java.lang.String)
*/
@Override
public void importData(String resourceLocation) {
final Resource resource = this.resourceLoader.getResource(resourceLocation);
this.importData(resource);
}
@Override
public void importData(final Resource resource) {
this.importData(resource, null);
}
@Override
public void importData(Source source) {
this.importData(source, null);
}
@Override
public final void importData(final Source source, PortalDataKey portalDataKey) {
//Get a StAX reader for the source to determine info about the data to import
final BufferedXMLEventReader bufferedXmlEventReader = createSourceXmlEventReader(source);
//If no PortalDataKey was passed build it from the source
if (portalDataKey == null) {
final StartElement rootElement = StaxUtils.getRootElement(bufferedXmlEventReader);
portalDataKey = new PortalDataKey(rootElement);
bufferedXmlEventReader.reset();
}
final String systemId = source.getSystemId();
//Post Process the PortalDataKey to see if more complex import operations are needed
final IPortalDataType portalDataType = this.dataKeyTypes.get(portalDataKey);
if (portalDataType == null) {
throw new RuntimeException("No IPortalDataType configured for " + portalDataKey + ", the resource will be ignored: " + getPartialSystemId(systemId));
}
final Set<PortalDataKey> postProcessedPortalDataKeys = portalDataType.postProcessPortalDataKey(systemId, portalDataKey, bufferedXmlEventReader);
bufferedXmlEventReader.reset();
//If only a single result from post processing import
if (postProcessedPortalDataKeys.size() == 1) {
this.importOrUpgradeData(systemId, DataAccessUtils.singleResult(postProcessedPortalDataKeys), bufferedXmlEventReader);
}
//If multiple results from post processing ordering is needed
else {
//Iterate over the data key order list to run the imports in the correct order
for (final PortalDataKey orderedPortalDataKey : this.dataKeyImportOrder) {
if (postProcessedPortalDataKeys.contains(orderedPortalDataKey)) {
//Reset the to start of the XML document for each import/upgrade call
bufferedXmlEventReader.reset();
this.importOrUpgradeData(systemId, orderedPortalDataKey, bufferedXmlEventReader);
}
}
}
}
/**
* @param portalDataKey Optional PortalDataKey to use, useful for batch imports where post-processing of keys has already take place
*/
protected final void importData(final Resource resource, final PortalDataKey portalDataKey) {
final InputStream resourceStream;
try {
resourceStream = resource.getInputStream();
}
catch (IOException e) {
throw new RuntimeException("Could not load InputStream for resource: " + resource, e);
}
try {
final String resourceUri = ResourceUtils.getResourceUri(resource);
this.importData(new StreamSource(resourceStream, resourceUri), portalDataKey);
}
finally {
IOUtils.closeQuietly(resourceStream);
}
}
protected String getPartialSystemId(String systemId) {
final String directoryUriStr = IMPORT_BASE_DIR.get();
if (directoryUriStr == null) {
return systemId;
}
if (systemId.startsWith(directoryUriStr)) {
return systemId.substring(directoryUriStr.length());
}
return systemId;
}
/**
* Run the import/update process on the data
*/
protected final void importOrUpgradeData(String systemId, PortalDataKey portalDataKey, XMLEventReader xmlEventReader) {
//See if there is a registered importer for the data, if so import
final IDataImporter<Object> dataImporterExporter = this.portalDataImporters.get(portalDataKey);
if (dataImporterExporter != null) {
this.logger.debug("Importing: {}", getPartialSystemId(systemId));
final Object data = unmarshallData(xmlEventReader, dataImporterExporter);
dataImporterExporter.importData(data);
this.logger.info("Imported : {}", getPartialSystemId(systemId));
return;
}
//No importer, see if there is an upgrader, if so upgrade
final IDataUpgrader dataUpgrader = this.portalDataUpgraders.get(portalDataKey);
if (dataUpgrader != null) {
this.logger.debug("Upgrading: {}", getPartialSystemId(systemId));
//Convert the StAX stream to a DOM node, due to poor JDK support for StAX with XSLT
final Node sourceNode;
try {
sourceNode = xmlUtilities.convertToDom(xmlEventReader);
}
catch (XMLStreamException e) {
throw new RuntimeException("Failed to create StAXSource from original XML reader", e);
}
final DOMSource source = new DOMSource(sourceNode);
final DOMResult result = new DOMResult();
final boolean doImport = dataUpgrader.upgradeData(source, result);
if (doImport) {
//If the upgrader didn't handle the import as well wrap the result DOM in a new Source and start the import process over again
final org.w3c.dom.Node node = result.getNode();
final PortalDataKey upgradedPortalDataKey = new PortalDataKey(node);
if (this.logger.isTraceEnabled()) {
this.logger.trace("Upgraded: " + getPartialSystemId(systemId) + " to " + upgradedPortalDataKey +
"\n\nSource XML: \n" + XmlUtilitiesImpl.toString(source.getNode()) +
"\n\nResult XML: \n" + XmlUtilitiesImpl.toString(node));
}
else {
this.logger.info("Upgraded: {} to {}", getPartialSystemId(systemId), upgradedPortalDataKey);
}
final DOMSource upgradedSource = new DOMSource(node, systemId);
this.importData(upgradedSource, upgradedPortalDataKey);
}
else {
this.logger.info("Upgraded and Imported: {}", getPartialSystemId(systemId));
}
return;
}
//No importer or upgrader found, fail
throw new IllegalArgumentException("Provided data " + portalDataKey + " has no registered importer or upgrader support: " + systemId);
}
protected Object unmarshallData(final XMLEventReader bufferedXmlEventReader, final IDataImporter<Object> dataImporterExporter) {
final Unmarshaller unmarshaller = dataImporterExporter.getUnmarshaller();
try {
final StAXSource source = new StAXSource(bufferedXmlEventReader);
return unmarshaller.unmarshal(source);
}
catch (XmlMappingException e) {
throw new RuntimeException("Failed to map provided XML to portal data", e);
}
catch (IOException e) {
throw new RuntimeException("Failed to read the provided XML data", e);
}
catch (XMLStreamException e) {
throw new RuntimeException("Failed to create StAX Source to read XML data", e);
}
}
protected BufferedXMLEventReader createSourceXmlEventReader(final Source source) {
//If it is a StAXSource see if we can do better handling of it
if (source instanceof StAXSource) {
final StAXSource staxSource = (StAXSource)source;
XMLEventReader xmlEventReader = staxSource.getXMLEventReader();
if (xmlEventReader != null) {
if (xmlEventReader instanceof BufferedXMLEventReader) {
final BufferedXMLEventReader bufferedXMLEventReader = (BufferedXMLEventReader)xmlEventReader;
bufferedXMLEventReader.reset();
bufferedXMLEventReader.mark(-1);
return bufferedXMLEventReader;
}
return new BufferedXMLEventReader(xmlEventReader, -1);
}
}
final XMLInputFactory xmlInputFactory = this.xmlUtilities.getXmlInputFactory();
final XMLEventReader xmlEventReader;
try {
xmlEventReader = xmlInputFactory.createXMLEventReader(source);
}
catch (XMLStreamException e) {
throw new RuntimeException("Failed to create XML Event Reader for data Source", e);
}
return new BufferedXMLEventReader(xmlEventReader, -1);
}
@Override
public Iterable<IPortalDataType> getExportPortalDataTypes() {
return this.exportPortalDataTypes;
}
@Override
public Iterable<IPortalDataType> getDeletePortalDataTypes() {
return this.deletePortalDataTypes;
}
@Override
public Iterable<? extends IPortalData> getPortalData(String typeId) {
final IDataExporter<Object> dataImporterExporter = getPortalDataExporter(typeId);
return dataImporterExporter.getPortalData();
}
@Override
public String exportData(String typeId, String dataId, Result result) {
final IDataExporter<Object> portalDataExporter = this.getPortalDataExporter(typeId);
final Object data = portalDataExporter.exportData(dataId);
if (data == null) {
return null;
}
final Marshaller marshaller = portalDataExporter.getMarshaller();
try {
marshaller.marshal(data, result);
return portalDataExporter.getFileName(data);
}
catch (XmlMappingException e) {
throw new RuntimeException("Failed to map provided portal data to XML", e);
}
catch (IOException e) {
throw new RuntimeException("Failed to write the provided XML data", e);
}
}
@Override
public boolean exportData(String typeId, String dataId, File directory) {
directory.mkdirs();
final File exportTempFile;
try {
exportTempFile = File.createTempFile(
SafeFilenameUtils.makeSafeFilename(StringUtils.rightPad(dataId, 2, '-') + "-"),
SafeFilenameUtils.makeSafeFilename("." + typeId),
directory);
}
catch (IOException e) {
throw new RuntimeException("Could not create temp file to export " + typeId + " " + dataId, e);
}
try {
final String fileName = this.exportData(typeId, dataId, new StreamResult(exportTempFile));
if (fileName == null) {
logger.info("Skipped: type={} id={}", typeId, dataId);
return false;
}
final File destFile = new File(directory, fileName + "." + typeId + ".xml");
if (destFile.exists()) {
logger.warn("Exporting " + typeId + " " + dataId + " but destination file already exists, it will be overwritten: " + destFile);
destFile.delete();
}
FileUtils.moveFile(exportTempFile, destFile);
logger.info("Exported: {}", destFile);
return true;
}
catch (Exception e) {
if (e instanceof RuntimeException) {
throw (RuntimeException)e;
}
throw new RuntimeException("Failed to export " + typeId + " " + dataId, e);
}
finally {
FileUtils.deleteQuietly(exportTempFile);
}
}
@Override
public void exportAllDataOfType(Set<String> typeIds, File directory, BatchExportOptions options) {
final Queue<ExportFuture<?>> exportFutures = new ConcurrentLinkedQueue<ExportFuture<?>>();
final boolean failOnError = options != null ? options.isFailOnError() : true;
//Determine the parent directory to log to
final File logDirectory = determineLogDirectory(options, "export");
//Setup reporting file
final File exportReport = new File(logDirectory, "data-export.txt");
final PrintWriter reportWriter;
try {
reportWriter = new PrintWriter(new BufferedWriter(new FileWriter(exportReport)));
}
catch (IOException e) {
throw new RuntimeException("Failed to create FileWriter for: " + exportReport, e);
}
try {
for (final String typeId : typeIds) {
final List<FutureHolder<?>> failedFutures = new LinkedList<FutureHolder<?>>();
final File typeDir = new File(directory, typeId);
logger.info("Adding all data of type {} to export queue: {}", typeId, typeDir);
reportWriter.println(typeId + "," + typeDir);
final Iterable<? extends IPortalData> dataForType = this.getPortalData(typeId);
for (final IPortalData data : dataForType) {
final String dataId = data.getDataId();
//Check for completed futures on every iteration, needed to fail as fast as possible on an import exception
final List<FutureHolder<?>> newFailed = waitForFutures(exportFutures, reportWriter, logDirectory, false);
failedFutures.addAll(newFailed);
final AtomicLong exportTime = new AtomicLong(-1);
//Create export task
Callable<Object> task = new CallableWithoutResult() {
@Override
protected void callWithoutResult() {
exportTime.set(System.nanoTime());
try {
exportData(typeId, dataId, typeDir);
}
finally {
exportTime.set(System.nanoTime() - exportTime.get());
}
}
};
//Submit the export task
final Future<?> exportFuture = this.importExportThreadPool.submit(task);
//Add the future for tracking
final ExportFuture futureHolder = new ExportFuture(exportFuture, typeId, dataId, exportTime);
exportFutures.offer(futureHolder);
}
final List<FutureHolder<?>> newFailed = waitForFutures(exportFutures, reportWriter, logDirectory, true);
failedFutures.addAll(newFailed);
reportWriter.flush();
if (failOnError && !failedFutures.isEmpty()) {
throw new RuntimeException(failedFutures.size() + " " + typeId + " entities failed to export.\n" +
"\tPer entity exception logs and a full report can be found in " + logDirectory);
}
}
}
catch (InterruptedException e) {
throw new RuntimeException("Interrupted while waiting for entities to export", e);
}
finally {
IOUtils.closeQuietly(reportWriter);
}
}
@Override
public void exportAllData(File directory, BatchExportOptions options) {
final Set<IPortalDataType> portalDataTypes;
if (this.exportAllPortalDataTypes != null) {
portalDataTypes = this.exportAllPortalDataTypes;
}
else {
portalDataTypes = this.exportPortalDataTypes;
}
final Set<String> typeIds = new LinkedHashSet<String>();
for (final IPortalDataType portalDataType : portalDataTypes) {
typeIds.add(portalDataType.getTypeId());
}
this.exportAllDataOfType(typeIds, directory, options);
}
protected IDataExporter<Object> getPortalDataExporter(String typeId) {
final IDataExporter<Object> dataExporter = this.portalDataExporters.get(typeId);
if (dataExporter == null) {
throw new IllegalArgumentException("No IDataExporter exists for: " + typeId);
}
return dataExporter;
}
@Override
public void deleteData(String typeId, String dataId) {
final IDataDeleter<Object> dataDeleter = this.portalDataDeleters.get(typeId);
if (dataDeleter == null) {
throw new IllegalArgumentException("No IDataDeleter exists for: " + typeId);
}
final Object data = dataDeleter.deleteData(dataId);
if (data != null) {
logger.info("Deleted data " + dataId + " of type " + typeId);
}
else {
logger.info("No data " + dataId + " of type " + typeId + " exists to delete");
}
}
/**
* Used by batch import and export to wait for queued tasks to complete. Handles fail-fast behavior
* if any of the tasks threw and exception by canceling all queued futures and logging a summary of
* the failures. All completed futures are removed from the queue.
*
* @param futures Queued futures to check for completeness
* @param wait If true it will wait for all futures to complete, if false only check for completed futures
* @return a list of futures that either threw exceptions or timed out
*/
protected List<FutureHolder<?>> waitForFutures(
final Queue<? extends FutureHolder<?>> futures,
final PrintWriter reportWriter, final File reportDirectory,
final boolean wait) throws InterruptedException {
final List<FutureHolder<?>> failedFutures = new LinkedList<FutureHolder<?>>();
for (Iterator<? extends FutureHolder<?>> futuresItr = futures.iterator(); futuresItr.hasNext();) {
final FutureHolder<?> futureHolder = futuresItr.next();
//If waiting, or if not waiting but the future is already done do the get
final Future<?> future = futureHolder.getFuture();
if (wait || (!wait && future.isDone())) {
futuresItr.remove();
try {
//Don't bother doing a get() on canceled futures
if (!future.isCancelled()) {
if (this.maxWait > 0) {
future.get(this.maxWait, this.maxWaitTimeUnit);
}
else {
future.get();
}
reportWriter.printf(REPORT_FORMAT, "SUCCESS", futureHolder.getDescription(), futureHolder.getExecutionTimeMillis());
}
}
catch (CancellationException e) {
//Ignore cancellation exceptions
}
catch (ExecutionException e) {
logger.error("Failed: " + futureHolder);
futureHolder.setError(e);
failedFutures.add(futureHolder);
reportWriter.printf(REPORT_FORMAT, "FAIL", futureHolder.getDescription(), futureHolder.getExecutionTimeMillis());
try {
final String dataReportName = SafeFilenameUtils.makeSafeFilename(futureHolder.getDataType() + "_" + futureHolder.getDataName() + ".txt");
final File dataReportFile = new File(reportDirectory, dataReportName);
final PrintWriter dataReportWriter = new PrintWriter(new BufferedWriter(new FileWriter(dataReportFile)));
try {
dataReportWriter.println("FAIL: " + futureHolder.getDataType() + " - " + futureHolder.getDataName());
dataReportWriter.println("--------------------------------------------------------------------------------");
e.getCause().printStackTrace(dataReportWriter);
}
finally {
IOUtils.closeQuietly(dataReportWriter);
}
}
catch (Exception re) {
logger.warn("Failed to write error report for failed " + futureHolder + ", logging root failure here", e.getCause());
}
}
catch (TimeoutException e) {
logger.warn("Failed: " + futureHolder);
futureHolder.setError(e);
failedFutures.add(futureHolder);
future.cancel(true);
reportWriter.printf(REPORT_FORMAT, "TIMEOUT", futureHolder.getDescription(), futureHolder.getExecutionTimeMillis());
}
}
}
return failedFutures;
}
private static abstract class FutureHolder<T> {
private final Future<T> future;
private final AtomicLong time;
private Exception error;
public FutureHolder(Future<T> future, AtomicLong time) {
this.future = future;
this.time = time;
}
public Future<T> getFuture() {
return this.future;
}
public double getExecutionTimeMillis() {
final long t = time.get();
if (!future.isDone()) {
return System.nanoTime() - t;
}
return t / 1000000.0;
}
public Exception getError() {
return error;
}
public void setError(Exception error) {
this.error = error;
}
public abstract String getDescription();
public abstract String getDataType();
public abstract String getDataName();
}
private static class ImportFuture<T> extends FutureHolder<T> {
private final Resource resource;
private final PortalDataKey dataKey;
public ImportFuture(Future<T> future, Resource resource, PortalDataKey dataKey, AtomicLong importTime) {
super(future, importTime);
this.resource = resource;
this.dataKey = dataKey;
}
@Override
public String getDescription() {
return this.resource.getDescription();
}
@Override
public String getDataType() {
return dataKey.getName().getLocalPart();
}
@Override
public String getDataName() {
return this.resource.getFilename();
}
@Override
public String toString() {
return "importing " + this.getDescription();
}
}
private static class ExportFuture<T> extends FutureHolder<T> {
private final String typeId;
private final String dataId;
public ExportFuture(Future<T> future, String typeId, String dataId, AtomicLong exportTime) {
super(future, exportTime);
this.typeId = typeId;
this.dataId = dataId;
}
@Override
public String getDescription() {
return "type=" + this.typeId + ", dataId=" + this.dataId;
}
@Override
public String getDataType() {
return this.typeId;
}
@Override
public String getDataName() {
return this.dataId;
}
@Override
public String toString() {
return "exporting " + this.getDescription();
}
}
}