package scigest.core;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
import thredds.catalog.InvDatasetImpl;
import ucar.nc2.Attribute;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
/**
*
* This class controls how a top level thredds catalog is generated.
*
*
* @author Feiyi Wang
*
*/
public class ParentDataset {
private File curDir;
private String[] files;
private List<Attribute> attrs;
private List<Variable> vars;
private InvDatasetImpl pds; // parent dataset
private Configuration config;
private Document doc;
private Element toplevel;
private String toplevelDSName;
private List<Element> elist;
private Namespace invcat;
ParentDataset(File curDir, String[] files) {
this.config = ConfigReader.load();
this.curDir = curDir;
this.files = files;
this.setVars(null);
this.setAttrs(null);
this.pds = null;
this.elist = new ArrayList<Element>();
this.toplevelDSName = config.getString("dataset.rootid") + curDir.getName();
this.doc = new Document();
Namespace xlink = Namespace.getNamespace("xlink", "http://www.w3.org/1999/xlink");
this.invcat = Namespace.getNamespace("",
"http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0");
Namespace xsi = Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
Element el = new Element("catalog", invcat);
el.addNamespaceDeclaration(invcat);
el.addNamespaceDeclaration(xlink);
el.addNamespaceDeclaration(xsi);
String location = "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0 "
+ "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.1.0.2.xsd";
org.jdom.Attribute attr = new org.jdom.Attribute("schemaLocation", location, xsi);
el.setAttribute(attr);
el.setAttribute("name", "TDS Configuration File");
el.setAttribute("version", "1.0.1");
this.doc.addContent(el);
}
private Element getProperty(String name, String value) {
Element el = new Element("property", invcat);
el.setAttribute("name", name);
el.setAttribute("value", value);
return el;
}
private Element httpService() {
Element el = new Element("service", invcat);
el.setAttribute("name", "HTTPServer");
el.setAttribute("serviceType", "HTTPServer");
el.setAttribute("base", "/thredds/fileServer/");
el.addContent(getProperty("requires_authorization", "true"));
el.addContent(getProperty("application", "Web Browser"));
el.addContent(getProperty("application", "Web Script"));
return el;
}
private Element inheritVariables() {
Element el = new Element("variables", invcat);
el.setAttribute("vocabulary", "CF-1.0");
for (Variable var: vars) {
Element e2 = new Element("variable", invcat);
e2.setAttribute("name", var.getShortName());
e2.setAttribute("vocabulary_name", var.getName());
String unit = var.getUnitsString();
if (null == unit)
unit = "";
e2.setAttribute("units", unit);
e2.setText(var.getDescription());
el.addContent(e2);
}
return el;
}
private Element inheritMeta() {
Element el = new Element("metadata", invcat);
el.setAttribute("inherited", "true");
// add service into metadata is a bad idea
// this will have validation error, not fatal, but errors
// el.addContent(httpService());
el.addContent(new Element("dataType", invcat).setText("GRID"));
el.addContent(new Element("dataFormat", invcat).setText("NetCDF"));
el.addContent(inheritVariables());
return el;
}
private Element toplevelDataset() {
Element el = new Element("dataset", invcat);
el.setAttribute("name", this.toplevelDSName);
el.setAttribute("ID", this.toplevelDSName + ".v1");
el.setAttribute("restrictAccess", "esg-user");
el.addContent(getProperty("dataset_id", this.toplevelDSName));
el.addContent(getProperty("dataset_version", "1"));
el.addContent(getProperty("project", config.getString("project.name")));
el.addContent(getProperty("model", config.getString("project.model")));
el.addContent(getProperty("institute", config.getString("project.institute")));
el.addContent(getProperty("time_frequency", config.getString("project.tf")));
this.toplevel = el;
return el;
}
public void threddsGen() {
// start a new catalog
Element root = doc.getRootElement();
root.addContent(httpService());
root.addContent(getProperty("catalog_version", "2"));
root.addContent(toplevelDataset());
for (String file : files) {
elist.add(childDataset(curDir, file));
}
// now can do metadata
this.toplevel.addContent(inheritMeta());
for (Element e: elist) {
this.toplevel.addContent(e);
}
String outfname = config.getString("project.name") + "_" + curDir.getName() + ".xml";
// intend to persist to file, \n to make sure the line behavior
CatalogPublisher.catalogList.add(outfname +"\n");
String fullpath = null;
if (config.containsKey("outdir")) {
fullpath = config.getString("outdir") + "/" + outfname;
} else {
fullpath = curDir.getPath() + outfname;
}
System.out.printf("\t Generating catalog file [%s] \n", fullpath);
OutputStream out;
try {
out = new FileOutputStream(new File(fullpath));
XMLOutputter outp = new XMLOutputter(Format.getPrettyFormat());
outp.output(doc, out);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void setAttrs(List<Attribute> attrs) {
this.attrs = attrs;
}
public List<Attribute> getAttrs() {
return attrs;
}
public void setVars(List<Variable> vars) {
this.vars = vars;
}
public List<Variable> getVars() {
return vars;
}
public InvDatasetImpl getParentDataset() {
return pds;
}
/**
*
* This method fills in child dataset information
*
* @param curDir
* @param filename
* @return
*/
public Element childDataset(File curDir, String filename) {
String fullPath = filename;
String fileBaseName = FilenameUtils.getName(filename);
if (null != attrs) {
// no need to scan
}
else {
// scan
NetcdfFile ncfile = null;
try {
ncfile = NetcdfFile.open(fullPath);
attrs = ncfile.getGlobalAttributes();
vars = ncfile.getVariables();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (null != ncfile) try {
ncfile.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
}
// in either case, we now produce a child dataset
Element el = new Element("dataset", invcat);
String name = toplevelDSName + "." + fileBaseName;
String project = config.getString("project.name");
el.setAttribute("name", name);
el.setAttribute("ID", "ornl." + project + "." + curDir.getName() + "." + fileBaseName);
el.setAttribute("urlPath",
"esg_dataroot/" + project + "/" + curDir.getName() + "/" + fileBaseName);
el.setAttribute("restrictAccess", "esg-user");
el.setAttribute("serviceName", "HTTPServer");
//el.addContent(httpService());
// calc dataset, should be factored out
File f = new File(filename);
Long fsize = FileUtils.sizeOf(f);
Long lastmod = f.lastModified();
Date lastdate = new Date(lastmod);
el.addContent(getDataSize(fsize));
el.addContent(getProperty("file_id", name));
el.addContent(getProperty("size", fsize.toString()));
el.addContent(getProperty("mod_time", lastmod.toString()));
return el;
}
private Element getDataSize(Long fsize) {
Element el = new Element("dataSize", invcat);
el.setAttribute("units", "bytes");
el.setText(fsize.toString());
return el;
}
}