/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.oodt.cas.filemgr.tools;
// JDK imports
import java.io.File;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
// OODT imports
import org.apache.oodt.cas.filemgr.catalog.Catalog;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.metadata.Metadata;
/**
* @author mattmann
* @version $Revision$
*
* <p>
* A Tool to export from a source file manager catalog and import into a dest
* file manager catalog.
* </p>
*
*/
public class ExpImpCatalog {
/* the client to the source catalog to export */
private XmlRpcFileManagerClient sourceClient = null;
/* the client to the dest catalog to import into */
private XmlRpcFileManagerClient destClient = null;
/* a source catalog I/F to export from (if no fm client is desired) */
private Catalog srcCatalog = null;
/* a dest catalog I/F to import into (if no fm client is desired) */
private Catalog destCatalog = null;
/* whether or not we should ensure a product doesn't exist before copying */
private boolean ensureUnique = false;
/* our log stream */
private static final Logger LOG = Logger.getLogger(ExpImpCatalog.class
.getName());
/**
* Default Constructor.
*
* @param sUrl
* The url to the source file manager to export from.
* @param dUrl
* The url to the dest file manager to import into.
* @param unique
* Whether or not the import tool should ensure that the product
* from the source does not exist in the dest.
*/
public ExpImpCatalog(URL sUrl, URL dUrl, boolean unique) {
try {
sourceClient = new XmlRpcFileManagerClient(sUrl);
} catch (ConnectionException e) {
LOG.log(Level.WARNING, "Unable to connect to source filemgr: ["
+ sUrl + "]");
throw new RuntimeException(e);
}
try {
destClient = new XmlRpcFileManagerClient(dUrl);
} catch (ConnectionException e) {
LOG.log(Level.WARNING, "Unable to connect to dest filemgr: ["
+ dUrl + "]");
throw new RuntimeException(e);
}
this.ensureUnique = unique;
}
public ExpImpCatalog(String sPropFilePath, String dPropFilePath,
boolean unique) throws InstantiationException {
this.ensureUnique = unique;
LOG.log(Level.INFO, "Constructing tool using catalog interfaces");
// first load the source prop file
try {
System.getProperties().load(
new File(sPropFilePath).toURL().openStream());
} catch (Exception e) {
throw new InstantiationException(e.getMessage());
}
// now construct the source catalog
String srcCatFactoryStr = System.getProperty("filemgr.catalog.factory");
LOG.log(Level.INFO, "source catalog factory: [" + srcCatFactoryStr
+ "]");
this.srcCatalog = GenericFileManagerObjectFactory
.getCatalogServiceFromFactory(srcCatFactoryStr);
// first load the dest prop file
try {
System.getProperties().load(
new File(dPropFilePath).toURL().openStream());
} catch (Exception e) {
throw new InstantiationException(e.getMessage());
}
String destCatFactoryStr = System
.getProperty("filemgr.catalog.factory");
LOG
.log(Level.INFO, "dest catalog factory: [" + destCatFactoryStr
+ "]");
this.destCatalog = GenericFileManagerObjectFactory
.getCatalogServiceFromFactory(destCatFactoryStr);
}
public void doExpImport(List sourceProductTypes) throws Exception {
if (this.sourceClient != null && this.destClient != null) {
// do validation of source/dest types
// otherwise, user is on their own discretion
List destProductTypes = destClient.getProductTypes();
if (!typesExist(sourceProductTypes, destProductTypes)) {
throw new Exception(
"The source product types must be present in the dest file manager!");
} else {
LOG
.log(Level.INFO,
"Source types and Dest types match: beginning processing");
}
} else
LOG.log(Level.INFO,
"Skipping type validation: catalog i/f impls being used.");
// we'll use the get product page method for each product type
// paginate through products using source product type
for (Iterator i = sourceProductTypes.iterator(); i.hasNext();) {
ProductType type = (ProductType) i.next();
try {
exportTypeToDest(type);
} catch (Exception e) {
LOG.log(Level.WARNING, "Error exporting product type: ["
+ type.getName() + "] from source to dest: Message: "
+ e.getMessage(), e);
throw e;
}
}
}
public void doExpImport() throws Exception {
if (sourceClient == null)
throw new RuntimeException(
"Cannot request exp/imp of all product types if no filemgr url specified!");
List sourceProductTypes = sourceClient.getProductTypes();
doExpImport(sourceProductTypes);
}
private void exportTypeToDest(ProductType type) throws Exception {
ProductPage page = null;
if (this.srcCatalog != null) {
page = srcCatalog.getFirstPage(type);
} else {
page = sourceClient.getFirstPage(type);
}
if (page == null)
return;
exportProductsToDest(page.getPageProducts(), type);
while (!page.isLastPage()) {
if (this.srcCatalog != null) {
page = srcCatalog.getNextPage(type, page);
} else
page = sourceClient.getNextPage(type, page);
if (page == null)
break;
exportProductsToDest(page.getPageProducts(), type);
}
}
private void exportProductsToDest(List products, ProductType type)
throws Exception {
if (products != null && products.size() > 0) {
for (Iterator i = products.iterator(); i.hasNext();) {
Product p = (Product) i.next();
if (ensureUnique) {
boolean hasProduct = safeHasProductTypeByName(p
.getProductName());
if (hasProduct) {
LOG.log(Level.INFO, "Skipping product: ["
+ p.getProductName()
+ "]: ensure unique enabled: "
+ "product exists in dest catalog");
continue;
}
}
p.setProductType(type);
if (sourceClient != null) {
p
.setProductReferences(sourceClient
.getProductReferences(p));
} else
p.setProductReferences(srcCatalog.getProductReferences(p));
Metadata met = null;
if (sourceClient != null) {
met = sourceClient.getMetadata(p);
} else {
met = srcCatalog.getMetadata(p);
}
LOG
.log(
Level.INFO,
"Source Product: ["
+ p.getProductName()
+ "]: Met Extraction and "
+ "Reference Extraction successful: writing to dest file manager");
// remove the default CAS fields for metadata
met.removeMetadata("CAS.ProductId");
met.removeMetadata("CAS.ProductReceivedTime");
met.removeMetadata("CAS.ProductName");
Product destProduct = new Product();
// copy through
destProduct.setProductName(p.getProductName());
destProduct.setProductStructure(p.getProductStructure());
destProduct.setProductType((destClient != null) ? destClient
.getProductTypeById(type.getProductTypeId()) : type);
destProduct.setTransferStatus(p.getTransferStatus());
LOG.log(Level.INFO, "Cataloging Product: ["
+ p.getProductName() + "]");
String destProductId = null;
if (destCatalog != null) {
destCatalog.addProduct(destProduct);
destProductId = destProduct.getProductId();
} else
destProductId = destClient.catalogProduct(destProduct);
LOG.log(Level.INFO, "Catalog successful: dest product id: ["
+ destProductId + "]");
destProduct.setProductId(destProductId);
LOG.log(Level.INFO, "Adding references for dest product: ["
+ destProductId + "]");
destProduct.setProductReferences(p.getProductReferences());
if (destCatalog != null) {
destCatalog.addProductReferences(destProduct);
} else
destClient.addProductReferences(destProduct);
LOG.log(Level.INFO,
"Reference addition successful for dest product: ["
+ destProductId + "]");
LOG.log(Level.INFO, "Adding metadata for dest product: ["
+ destProductId + "]");
if (destCatalog != null) {
destCatalog.addMetadata(met, destProduct);
} else
destClient.addMetadata(destProduct, met);
LOG.log(Level.INFO,
"Met addition successful for dest product: ["
+ destProductId + "]");
LOG.log(Level.INFO, "Successful import of product: ["
+ p.getProductName() + "] into dest file manager");
}
}
}
/**
* @return Returns the ensureUnique.
*/
public boolean isEnsureUnique() {
return ensureUnique;
}
/**
* @param ensureUnique
* The ensureUnique to set.
*/
public void setEnsureUnique(boolean ensureUnique) {
this.ensureUnique = ensureUnique;
}
/**
* @param args
*/
public static void main(String[] args) throws Exception {
String sourceUrl = null, destUrl = null, srcCatPropFile = null, destCatPropFile = null;
boolean unique = false;
List types = null;
String usage = "ExpImpCatalog [options] \n" + "--source <url>\n"
+ "--dest <url>\n " + "--unique\n"
+ "[--types <comma separate list of product type names>]\n"
+ "[--sourceCatProps <file> --destCatProps <file>]\n";
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--source")) {
sourceUrl = args[++i];
} else if (args[i].equals("--dest")) {
destUrl = args[++i];
} else if (args[i].equals("--unique")) {
unique = true;
} else if (args[i].equals("--types")) {
String[] typesAndIdsEnc = args[++i].split(",");
types = new Vector(typesAndIdsEnc.length);
for (int j = 0; j < typesAndIdsEnc.length; j++) {
String[] typeIdToks = typesAndIdsEnc[j].split("\\|");
ProductType type = new ProductType();
type.setName(typeIdToks[0]);
type.setProductTypeId(typeIdToks[1]);
types.add(type);
}
} else if (args[i].equals("--sourceCatProps")) {
srcCatPropFile = args[++i];
} else if (args[i].equals("--destCatProps")) {
destCatPropFile = args[++i];
}
}
if (((sourceUrl == null || destUrl == null) && (srcCatPropFile == null || destCatPropFile == null))
|| (sourceUrl != null && destUrl != null && (srcCatPropFile != null || destCatPropFile != null))
|| ((srcCatPropFile != null && destCatPropFile == null) || (destCatPropFile != null && srcCatPropFile == null))) {
System.err.println(usage);
System.exit(1);
}
ExpImpCatalog tool = null;
if (srcCatPropFile != null) {
tool = new ExpImpCatalog(srcCatPropFile, destCatPropFile, unique);
} else
tool = new ExpImpCatalog(new URL(sourceUrl), new URL(destUrl),
unique);
if (types != null && types.size() > 0) {
tool.doExpImport(types);
} else
tool.doExpImport();
}
private boolean typesExist(List sourceList, List destList) {
if (sourceList == null
|| (sourceList != null && sourceList.size() == 0)) {
return false;
}
if (destList == null || (destList != null && destList.size() == 0)) {
return false;
}
// iterate through the source types and try and find the type in the
// destList
for (Iterator i = sourceList.iterator(); i.hasNext();) {
ProductType type = (ProductType) i.next();
if (!typeInList(type, destList)) {
LOG.log(Level.WARNING, "Source type: [" + type.getName()
+ "] not present in dest file manager");
return false;
}
}
return true;
}
private boolean typeInList(ProductType type, List typeList) {
if (typeList == null || (typeList != null && typeList.size() == 0)) {
return false;
}
for (Iterator i = typeList.iterator(); i.hasNext();) {
ProductType destType = (ProductType) i.next();
if (destType.getProductTypeId().equals(type.getProductTypeId())
&& destType.getName().equals(type.getName())) {
return true;
}
}
return false;
}
private boolean safeHasProductTypeByName(String productName) {
if (destCatalog != null) {
try {
return (destCatalog.getProductByName(productName) != null);
} catch (CatalogException e) {
e.printStackTrace();
LOG
.log(Level.WARNING,
"Exceptiong checking for product type by name: ["
+ productName + "]: Message: "
+ e.getMessage());
return false;
}
} else {
try {
return destClient.hasProduct(productName);
} catch (CatalogException e) {
e.printStackTrace();
LOG
.log(Level.WARNING,
"Exceptiong checking for product type by name: ["
+ productName + "]: Message: "
+ e.getMessage());
return false;
}
}
}
}