/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.oodt.cas.filemgr.datatransfer;
//APACHE Imports
import org.apache.commons.io.FileUtils;
//OODT imports
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.mime.MimeTypesFactory;
//JDK imports
import java.io.IOException;
import java.io.File;
import java.util.List;
import java.util.Iterator;
import java.util.Vector;
import java.util.logging.Logger;
import java.util.logging.Level;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.net.URISyntaxException;
/**
* @author mattmann
* @author bfoster
* @version $Revision$
*
* <p>
* An implementation of the {@link DataTransfer} interface that moves
* products that are available via URIs on the same machine, through an
* NFS mounted disk, or via the locally mounted file repository.
* </p>
*
*/
public class LocalDataTransferer implements DataTransfer {
/* our log stream */
private static final Logger LOG = Logger.getLogger(LocalDataTransferer.class
.getName());
/* file manager client */
private XmlRpcFileManagerClient client = null;
/**
* <p>
* Default Constructor
* </p>
*/
public LocalDataTransferer() {
}
/*
* (non-Javadoc)
*
* @see
* org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl
* (java.net.URL)
*/
public void setFileManagerUrl(URL url) {
try {
client = new XmlRpcFileManagerClient(url);
LOG.log(Level.INFO, "Local Data Transfer to: ["
+ client.getFileManagerUrl().toString() + "] enabled");
} catch (ConnectionException e) {
e.printStackTrace();
}
}
/*
* (non-Javadoc)
*
* @see
* org.apache.oodt.cas.datatransfer.DataTransfer#transferProduct(org.apache
* .oodt.cas.data.structs.Product)
*/
public void transferProduct(Product product) throws DataTransferException,
IOException {
// check whether or not it's a set of files, or it's actually a dir
// structure
if (product.getProductStructure().equals(Product.STRUCTURE_HIERARCHICAL)) {
try {
moveDirToProductRepo(product);
} catch (URISyntaxException e) {
LOG.log(
Level.WARNING,
"URI Syntax Exception when moving dir "
+ ((Reference) product.getProductReferences().get(0))
.getOrigReference() + ": Message: "
+ e.getMessage());
throw new DataTransferException(e);
}
} else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
try {
moveFilesToProductRepo(product);
} catch (URISyntaxException e) {
LOG.log(
Level.WARNING,
"URI Syntax Exception when moving files: Message: "
+ e.getMessage());
throw new DataTransferException(e);
}
} else {
throw new DataTransferException(
"Cannot transfer product on unknown ProductStructure: "
+ product.getProductStructure());
}
}
/*
* (non-Javadoc)
*
* @see
* org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#retrieveProduct(org.
* apache.oodt.cas.filemgr.structs.Product, java.io.File)
*/
public void retrieveProduct(Product product, File directory)
throws DataTransferException, IOException {
// check whether or not it's a set of files, or it's actually a dir
// structure
if (product.getProductStructure().equals(Product.STRUCTURE_HIERARCHICAL)) {
try {
copyDirToDir(product, directory);
} catch (URISyntaxException e) {
LOG.log(
Level.WARNING,
"URI Syntax Exception when moving dir "
+ ((Reference) product.getProductReferences().get(0))
.getDataStoreReference() + ": Message: "
+ e.getMessage());
throw new DataTransferException(e);
}
} else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
try {
copyFilesToDir(product, directory);
} catch (URISyntaxException e) {
LOG.log(
Level.WARNING,
"URI Syntax Exception when moving files: Message: "
+ e.getMessage());
throw new DataTransferException(e);
}
} else {
throw new DataTransferException(
"Cannot transfer product on unknown ProductStructure: "
+ product.getProductStructure());
}
}
@Override
public void deleteProduct(Product product) throws DataTransferException, IOException {
for (Reference ref : product.getProductReferences()) {
File dataFile = new File(URI.create(ref.getDataStoreReference()).toURL().getPath());
if (!dataFile.delete()) {
throw new IOException(String.format("Failed to delete file %s - delete returned false",
dataFile));
}
}
}
/**
* @param args
*/
public static void main(String[] args) throws DataTransferException,
IOException, URISyntaxException {
String usage = "LocalFileTransfer --productName <name> --productRepo <repo> [--dir <dirRef>] [--files <origRef 1>...<origRef N>]\n";
MimeTypes mimeTypeRepo;
try {
mimeTypeRepo = MimeTypesFactory
.create(System
.getProperty("org.apache.oodt.cas.filemgr.mime.type.repository"));
} catch (MimeTypeException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
String productName = null;
String productRepo = null;
String transferType = null;
Reference dirReference = null;
List<Reference> fileReferences = null;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--dir")) {
transferType = "dir";
dirReference = new Reference();
dirReference.setOrigReference(new File(new URI(args[++i])).toURI()
.toString());
LOG.log(Level.FINER,
"LocalFileTransfer.main: Generated orig reference: "
+ dirReference.getOrigReference());
} else if (args[i].equals("--files")) {
transferType = "files";
fileReferences = new Vector<Reference>();
for (int j = i + 1; j < args.length; j++) {
LOG.log(Level.FINER, "LocalFileTransfer.main: Adding file ref: "
+ args[j]);
fileReferences.add(new Reference(args[j], null,
new File(args[j]).length(), mimeTypeRepo
.getMimeType(args[j])));
}
} else if (args[i].equals("--productName")) {
productName = args[++i];
} else if (args[i].equals("--productRepo")) {
productRepo = args[++i];
}
}
if (transferType == null
|| (transferType != null && ((transferType.equals("dir") && dirReference == null)
|| (transferType.equals("files") && fileReferences == null)
|| (transferType != null && !(transferType.equals("dir") || transferType
.equals("files"))) || productName == null || productRepo == null))) {
System.err.println(usage);
System.exit(1);
}
// construct a new Product
Product p = new Product();
p.setProductName(productName);
if (transferType.equals("dir")) {
p.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
dirReference.setDataStoreReference(new File(new URI(productRepo))
.toURI().toURL().toExternalForm()
+ URLEncoder.encode(p.getProductName(), "UTF-8") + "/");
p.getProductReferences().add(dirReference);
/* we'll do a simple versioning scheme ourselves: no versioning! */
p.getProductReferences().addAll(
VersioningUtils.getReferencesFromDir(new File(new URI(
dirReference.getOrigReference()))));
VersioningUtils.createBasicDataStoreRefsHierarchical(p
.getProductReferences());
} else if (transferType.equals("files")) {
p.setProductStructure("Flat");
p.getProductReferences().addAll(fileReferences);
VersioningUtils.createBasicDataStoreRefsFlat(productName, productRepo,
p.getProductReferences());
}
DataTransfer transfer = new LocalDataTransferer();
transfer.transferProduct(p);
}
private void copyDirToDir(Product product, File directory)
throws IOException, URISyntaxException {
Reference dirRef = (Reference) product.getProductReferences().get(0);
LOG.log(
Level.INFO,
"LocalDataTransferer: Staging Directory: "
+ dirRef.getDataStoreReference() + " into directory "
+ directory.getAbsolutePath());
for (Iterator<Reference> i = product.getProductReferences().iterator(); i
.hasNext();) {
Reference r = i.next();
File fileRef = new File(new URI(r.getDataStoreReference()));
if (fileRef.isFile()) {
copyFile(r, directory);
} else if (fileRef.isDirectory()
&& (fileRef.list() != null && fileRef.list().length == 0)) {
// if it's a directory and it doesn't exist yet, we should
// create it
// just in case there's no files in it
File dest = new File(directory, fileRef.getName());
if (!new File(new URI(dest.getAbsolutePath())).exists()) {
LOG.log(Level.FINER, "Directory: [" + dest.getAbsolutePath()
+ "] doesn't exist: creating it");
try {
FileUtils.forceMkdir(new File(new URI(dest.getAbsolutePath())));
}
catch(IOException e){
LOG.log(
Level.WARNING,
"Unable to create directory: ["
+ dest.getAbsolutePath()
+ "] in local data transferer");
}
}
}
}
}
private void moveDirToProductRepo(Product product) throws IOException,
URISyntaxException {
Reference dirRef = (Reference) product.getProductReferences().get(0);
LOG.log(
Level.INFO,
"LocalDataTransferer: Moving Directory: "
+ dirRef.getOrigReference() + " to "
+ dirRef.getDataStoreReference());
// notify the file manager that we started
quietNotifyTransferProduct(product);
for (Iterator<Reference> i = product.getProductReferences().iterator(); i
.hasNext();) {
Reference r = i.next();
File fileRef = new File(new URI(r.getOrigReference()));
if (fileRef.isFile()) {
moveFile(r, false);
} else if (fileRef.isDirectory()
&& (fileRef.list() != null && fileRef.list().length == 0)) {
// if it's a directory and it doesn't exist yet, we should
// create it
// just in case there's no files in it
if (!new File(new URI(r.getDataStoreReference())).exists()) {
LOG.log(Level.FINER, "Directory: [" + r.getDataStoreReference()
+ "] doesn't exist: creating it");
try {
FileUtils.forceMkdir(new File(new URI(r.getDataStoreReference())));
}
catch(IOException e){
LOG.log(
Level.WARNING,
"Unable to create directory: ["
+ r.getDataStoreReference()
+ "] in local data transferer");
}
}
}
}
// notify the file manager that we're done
quietNotifyProductTransferComplete(product);
}
private void moveFilesToProductRepo(Product product) throws IOException,
URISyntaxException {
List<Reference> refs = product.getProductReferences();
// notify the file manager that we started
quietNotifyTransferProduct(product);
for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
Reference r = (Reference) i.next();
moveFile(r, true);
}
// notify the file manager that we're done
quietNotifyProductTransferComplete(product);
}
private void copyFilesToDir(Product product, File directory)
throws IOException, URISyntaxException {
List<Reference> refs = product.getProductReferences();
for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
Reference r = (Reference) i.next();
copyFile(r, directory);
}
}
private void moveFile(Reference r, boolean log) throws IOException,
URISyntaxException {
if (log) {
LOG.log(Level.INFO,
"LocalDataTransfer: Moving File: " + r.getOrigReference()
+ " to " + r.getDataStoreReference());
}
File srcFileRef = new File(new URI(r.getOrigReference()));
File destFileRef = new File(new URI(r.getDataStoreReference()));
FileUtils.copyFile(srcFileRef, destFileRef);
}
private void copyFile(Reference r, File directory) throws IOException,
URISyntaxException {
File srcFileRef = new File(new URI(r.getDataStoreReference()));
LOG.log(Level.INFO,
"LocalDataTransfer: Copying File: " + r.getDataStoreReference()
+ " to file:" + directory.getAbsolutePath() + "/"
+ srcFileRef.getName());
FileUtils.copyFile(srcFileRef, new File(directory, srcFileRef.getName()));
}
private void quietNotifyTransferProduct(Product p) {
if (client == null) {
LOG.log(Level.WARNING,
"File Manager service not defined: this transfer will not be tracked");
return;
}
try {
client.transferringProduct(p);
} catch (DataTransferException e) {
e.printStackTrace();
LOG.log(Level.WARNING,
"Error notifying file manager of product transfer initiation for product: ["
+ p.getProductId() + "]: Message: " + e.getMessage());
return;
}
}
private void quietNotifyProductTransferComplete(Product p) {
if (client == null) {
LOG.log(Level.WARNING,
"File Manager service not defined: this transfer will not be tracked");
return;
}
try {
client.removeProductTransferStatus(p);
} catch (DataTransferException e) {
e.printStackTrace();
LOG.log(Level.WARNING,
"Error notifying file manager of product transfer completion for product: ["
+ p.getProductId() + "]: Message: " + e.getMessage());
return;
}
}
}