/*
* Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.ncml;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.Attribute;
import ucar.nc2.dataset.*;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.IO;
import ucar.nc2.util.URLnaming;
import thredds.catalog.XMLEntityResolver;
import org.jdom.*;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
import ucar.unidata.util.StringUtil;
import java.io.*;
import java.net.*;
import java.util.*;
/**
* Read NcML and create NetcdfDataset.
*
* @author caron
* @see <a href="http://www.unidata.ucar.edu/software/netcdf/ncml/">http://www.unidata.ucar.edu/software/netcdf/ncml/</a>
*/
public class NcMLReader {
static public final Namespace ncNS = Namespace.getNamespace("nc", XMLEntityResolver.NJ22_NAMESPACE);
static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(NcMLReader.class);
private static boolean debugURL = false, debugXML = false, showParsedXML = false;
private static boolean debugOpen = false, debugConstruct = false, debugCmd = false;
private static boolean debugAggDetail = false;
static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
debugURL = debugFlag.isSet("NcML/debugURL");
debugXML = debugFlag.isSet("NcML/debugXML");
showParsedXML = debugFlag.isSet("NcML/showParsedXML");
debugCmd = debugFlag.isSet("NcML/debugCmd");
debugOpen = debugFlag.isSet("NcML/debugOpen");
debugConstruct = debugFlag.isSet("NcML/debugConstruct");
debugAggDetail = debugFlag.isSet("NcML/debugAggDetail");
}
private static boolean validate = false;
/**
* Use NCML to modify a dataset, getting the NcML document as a resource stream.
* Uses ClassLoader.getResourceAsStream(ncmlResourceLocation), so the NcML can be inside of a jar file, for example.
*
* @param ncDataset modify this dataset
* @param ncmlResourceLocation resource location of NcML
* @param cancelTask allow user to cancel task; may be null
* @throws IOException on read error
*/
static public void wrapNcMLresource(NetcdfDataset ncDataset, String ncmlResourceLocation, CancelTask cancelTask) throws IOException {
ClassLoader cl = ncDataset.getClass().getClassLoader();
InputStream is = cl.getResourceAsStream(ncmlResourceLocation);
if (is == null)
throw new FileNotFoundException(ncmlResourceLocation);
if (debugXML) {
System.out.println(" NetcdfDataset URL = <" + ncmlResourceLocation + ">");
InputStream is2 = cl.getResourceAsStream(ncmlResourceLocation);
System.out.println(" contents=\n" + IO.readContents(is2));
}
org.jdom.Document doc;
try {
SAXBuilder builder = new SAXBuilder(validate);
if (debugURL) System.out.println(" NetcdfDataset URL = <" + ncmlResourceLocation + ">");
doc = builder.build(is);
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
if (debugXML) System.out.println(" SAXBuilder done");
if (showParsedXML) {
XMLOutputter xmlOut = new XMLOutputter();
System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
}
Element netcdfElem = doc.getRootElement();
NcMLReader reader = new NcMLReader();
reader.readNetcdf(ncDataset.getLocation(), ncDataset, ncDataset, netcdfElem, cancelTask);
if (debugOpen) System.out.println("***NcMLReader.wrapNcML result= \n" + ncDataset);
}
/**
* Use NCML to modify the dataset, getting NcML from a URL
*
* @param ncDataset modify this dataset
* @param ncmlLocation URL location of NcML
* @param cancelTask allow user to cancel task; may be null
* @throws IOException on read error
*/
static public void wrapNcML(NetcdfDataset ncDataset, String ncmlLocation, CancelTask cancelTask) throws IOException {
org.jdom.Document doc;
try {
SAXBuilder builder = new SAXBuilder(validate);
if (debugURL) System.out.println(" NetcdfDataset URL = <" + ncmlLocation + ">");
doc = builder.build(ncmlLocation);
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
if (debugXML) System.out.println(" SAXBuilder done");
if (showParsedXML) {
XMLOutputter xmlOut = new XMLOutputter();
System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
}
Element netcdfElem = doc.getRootElement();
NcMLReader reader = new NcMLReader();
reader.readNetcdf(ncmlLocation, ncDataset, ncDataset, netcdfElem, cancelTask);
if (debugOpen) System.out.println("***NcMLReader.wrapNcML result= \n" + ncDataset);
}
/**
* Use NCML to modify the referenced dataset, create a new dataset with the merged info
* Used to wrap each dataset of an aggregation before its aggregated
*
* @param ref referenced dataset
* @param parentElem parent element - usually the aggregation element of the ncml
* @return new dataset with the merged info
* @throws IOException on read error
*/
static public NetcdfDataset mergeNcML(NetcdfFile ref, Element parentElem) throws IOException {
NetcdfDataset targetDS = new NetcdfDataset(ref, null); // no enhance
NcMLReader reader = new NcMLReader();
reader.readGroup(targetDS, targetDS, null, null, parentElem);
targetDS.finish();
return targetDS;
}
/**
* Use NCML to directly modify the dataset
*
* @param targetDS referenced dataset
* @param parentElem parent element - usually the aggregation element of the ncml
* @return new dataset with the merged info
* @throws IOException on read error
*/
static public NetcdfDataset mergeNcMLdirect(NetcdfDataset targetDS, Element parentElem) throws IOException {
NcMLReader reader = new NcMLReader();
reader.readGroup(targetDS, targetDS, null, null, parentElem);
targetDS.finish();
return targetDS;
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
* Read an NcML file from a URL location, and construct a NetcdfDataset.
*
* @param ncmlLocation the URL location string of the NcML document
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(String ncmlLocation, CancelTask cancelTask) throws IOException {
return readNcML(ncmlLocation, (String) null, cancelTask);
}
/**
* Read an NcML file from a URL location, and construct a NetcdfDataset.
*
* @param ncmlLocation the URL location string of the NcML document
* @param referencedDatasetUri if null (usual case) get this from NcML, otherwise use URI as the location of the referenced dataset.
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(String ncmlLocation, String referencedDatasetUri, CancelTask cancelTask) throws IOException {
URL url = new URL(ncmlLocation);
if (debugURL) {
System.out.println(" NcMLReader open " + ncmlLocation);
System.out.println(" URL = " + url.toString());
System.out.println(" external form = " + url.toExternalForm());
System.out.println(" protocol = " + url.getProtocol());
System.out.println(" host = " + url.getHost());
System.out.println(" path = " + url.getPath());
System.out.println(" file = " + url.getFile());
}
org.jdom.Document doc;
try {
SAXBuilder builder = new SAXBuilder(validate);
if (debugURL) System.out.println(" NetcdfDataset URL = <" + url + ">");
doc = builder.build(url);
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
if (debugXML) System.out.println(" SAXBuilder done");
if (showParsedXML) {
XMLOutputter xmlOut = new XMLOutputter();
System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
}
Element netcdfElem = doc.getRootElement();
if (referencedDatasetUri == null) {
// the ncml probably refers to another dataset, but doesnt have to
referencedDatasetUri = netcdfElem.getAttributeValue("location");
if (referencedDatasetUri == null)
referencedDatasetUri = netcdfElem.getAttributeValue("uri");
}
NcMLReader reader = new NcMLReader();
NetcdfDataset ncd = reader.readNcML(ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
if (debugOpen) System.out.println("***NcMLReader.readNcML result= \n" + ncd);
return ncd;
}
/**
* Read NcML doc from an InputStream, and construct a NetcdfDataset.
*
* @param ins the InputStream containing the NcML document
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(InputStream ins, CancelTask cancelTask) throws IOException {
org.jdom.Document doc;
try {
SAXBuilder builder = new SAXBuilder(validate);
doc = builder.build(ins);
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
if (debugXML) System.out.println(" SAXBuilder done");
if (showParsedXML) {
XMLOutputter xmlOut = new XMLOutputter();
System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
}
Element netcdfElem = doc.getRootElement();
NetcdfDataset ncd = readNcML(null, netcdfElem, cancelTask);
if (debugOpen) System.out.println("***NcMLReader.readNcML (stream) result= \n" + ncd);
return ncd;
}
/**
* Read NcML doc from a Reader, and construct a NetcdfDataset.
*
* @param r the Reader containing the NcML document
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(Reader r, CancelTask cancelTask) throws IOException {
return readNcML(r, null, cancelTask);
}
/**
* Read NcML doc from a Reader, and construct a NetcdfDataset.
* eg: NcMLReader.readNcML(new StringReader(ncml), location, null);
*
* @param r the Reader containing the NcML document
* @param ncmlLocation the URL location string of the NcML document, used to resolve reletive path of the referenced dataset,
* or may be just a unique name for caching purposes.
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(Reader r, String ncmlLocation, CancelTask cancelTask) throws IOException {
org.jdom.Document doc;
try {
SAXBuilder builder = new SAXBuilder(validate);
doc = builder.build(r);
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
if (debugXML) System.out.println(" SAXBuilder done");
if (showParsedXML) {
XMLOutputter xmlOut = new XMLOutputter();
System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
}
Element netcdfElem = doc.getRootElement();
NetcdfDataset ncd = readNcML(ncmlLocation, netcdfElem, cancelTask);
if (debugOpen) System.out.println("***NcMLReader.readNcML (stream) result= \n" + ncd);
return ncd;
}
/**
* Read NcML from a JDOM Document, and construct a NetcdfDataset.
*
* @param ncmlLocation the URL location string of the NcML document, used to resolve reletive path of the referenced dataset,
* or may be just a unique name for caching purposes.
* @param netcdfElem the JDOM Document's root (netcdf) element
* @param cancelTask allow user to cancel the task; may be null
* @return the resulting NetcdfDataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
static public NetcdfDataset readNcML(String ncmlLocation, Element netcdfElem, CancelTask cancelTask) throws IOException {
// the ncml probably refers to another dataset, but doesnt have to
String referencedDatasetUri = netcdfElem.getAttributeValue("location");
if (referencedDatasetUri == null)
referencedDatasetUri = netcdfElem.getAttributeValue("uri");
NcMLReader reader = new NcMLReader();
return reader.readNcML(ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
}
//////////////////////////////////////////////////////////////////////////////////////
private String location;
private boolean explicit = false;
private Formatter errlog = new Formatter();
/**
* This sets up the target dataset and the referenced dataset.
*
* @param ncmlLocation the URL location string of the NcML document, used to resolve reletive path of the referenced dataset,
* or may be just a unique name for caching purposes.
* @param referencedDatasetUri refers to this dataset (may be null)
* @param netcdfElem JDOM netcdf element
* @param cancelTask allow user to cancel the task; may be null
* @return NetcdfDataset the constructed dataset
* @throws IOException on read error, or bad referencedDatasetUri URI
*/
private NetcdfDataset readNcML(String ncmlLocation, String referencedDatasetUri,
Element netcdfElem, CancelTask cancelTask) throws IOException {
// augment URI.resolve(), by also dealing with base file: URIs
referencedDatasetUri = URLnaming.resolve(ncmlLocation, referencedDatasetUri);
// common error causing infinite regression
if ((referencedDatasetUri != null) && referencedDatasetUri.equals(ncmlLocation))
throw new IllegalArgumentException("NcML location attribute refers to the NcML document itself" + referencedDatasetUri);
// they can specify the iosp to use - but must be file based
String iospS = netcdfElem.getAttributeValue("iosp");
String iospParam = netcdfElem.getAttributeValue("iospParam");
String bufferSizeS = netcdfElem.getAttributeValue("buffer_size");
int buffer_size = -1;
if (bufferSizeS != null)
buffer_size = Integer.parseInt(bufferSizeS);
// open the referenced dataset - do NOT use acquire, and dont enhance
// LOOK : shouldnt enhance be controlled by enhance attribute on the netcdf element ?
NetcdfDataset refds = null;
if (referencedDatasetUri != null) {
if (iospS != null) {
NetcdfFile ncfile;
try {
ncfile = new NcMLNetcdfFile(iospS, iospParam, referencedDatasetUri, buffer_size, cancelTask);
} catch (Exception e) {
throw new IOException(e);
}
refds = new NetcdfDataset(ncfile, false);
} else {
// String location, boolean enhance, int buffer_size, ucar.nc2.util.CancelTask cancelTask, Object spiObject) throws IOException {
// (String location, EnumSet<Enhance> enhanceMode, int buffer_size, ucar.nc2.util.CancelTask cancelTask, Object spiObject) throws IOException {
refds = NetcdfDataset.openDataset(referencedDatasetUri, null, buffer_size, cancelTask, iospParam);
// refds.setEnhanceProcessed(false); // hasnt had enhance applied to it yet - wait till ncml mods have been applied
}
}
// explicit means all of the metadata is specified in the XML, and the referenced dataset is used only for data access
Element elemE = netcdfElem.getChild("explicit", ncNS);
explicit = (elemE != null);
// general idea is that we just modify the referenced dataset
// the exception is when explicit is specified, then we keep them seperate.
// refds != null refds == null
// explicit refds!=new new (ref=new)
// readMetadata modify (new=ref) new (ref=new)
//
NetcdfDataset targetDS;
if (explicit || (refds == null)) {
targetDS = new NetcdfDataset();
if (refds == null)
refds = targetDS;
else
targetDS.setReferencedFile(refds); // gotta set so it gets closed !!
} else { // modify the referenced dataset directly
targetDS = refds;
}
// continue processing here
readNetcdf(ncmlLocation, targetDS, refds, netcdfElem, cancelTask);
return targetDS;
}
// need access to protected constructor
private static class NcMLNetcdfFile extends NetcdfFile {
NcMLNetcdfFile(String iospClassName, String iospParam, String location, int buffer_size, ucar.nc2.util.CancelTask cancelTask)
throws IOException, IllegalAccessException, ClassNotFoundException, InstantiationException {
super(iospClassName, iospParam, location, buffer_size, cancelTask);
}
}
///////// Heres where the parsing work starts
/**
* parse a netcdf JDOM Element, and add contents to the targetDS NetcdfDataset.
* <p/>
* This is a bit tricky, because it handles several cases
* When targetDS == refds, we are just modifying targetDS.
* When targetDS != refds, we keep them seperate, and copy from refds to newds.
* <p/>
* The user may be defining new elements or modifying old ones. The only way to tell is by seeing
* if the elements already exist.
*
* @param ncmlLocation NcML URL location, or may be just a unique name for caching purposes.
* @param targetDS add the info to this one, never null
* @param refds the referenced dataset; may equal newds, never null
* @param netcdfElem JDOM netcdf element
* @param cancelTask allow user to cancel the task; may be null
* @throws IOException on read error
*/
public void readNetcdf(String ncmlLocation, NetcdfDataset targetDS, NetcdfFile refds, Element netcdfElem, CancelTask cancelTask) throws IOException {
this.location = ncmlLocation; // log messages need this
if (debugOpen)
System.out.println("NcMLReader.readNetcdf ncml= " + ncmlLocation + " referencedDatasetUri= " + refds.getLocation());
// detect incorrect namespace
Namespace use = netcdfElem.getNamespace();
if (!use.equals(ncNS)) {
throw new IllegalArgumentException("Incorrect namespace specified in NcML= " + use.getURI() + "\n must be=" + ncNS.getURI());
}
if (ncmlLocation != null) targetDS.setLocation(ncmlLocation);
targetDS.setId(netcdfElem.getAttributeValue("id"));
targetDS.setTitle(netcdfElem.getAttributeValue("title"));
// aggregation first
Element aggElem = netcdfElem.getChild("aggregation", ncNS);
if (aggElem != null) {
Aggregation agg = readAgg(aggElem, ncmlLocation, targetDS, cancelTask);
targetDS.setAggregation(agg);
agg.finish(cancelTask);
}
// the root group
readGroup(targetDS, refds, null, null, netcdfElem);
String errors = errlog.toString();
if (errors.length() > 0)
throw new IllegalArgumentException("NcML had fatal errors:" + errors);
// transfer from groups to global containers
targetDS.finish();
// enhance means do scale/offset and/or add CoordSystems
Set<NetcdfDataset.Enhance> mode = NetcdfDataset.parseEnhanceMode(netcdfElem.getAttributeValue("enhance"));
//if (mode == null)
// mode = NetcdfDataset.getEnhanceDefault();
targetDS.enhance(mode);
// optionally add record structure to netcdf-3
String addRecords = netcdfElem.getAttributeValue("addRecords");
if ((addRecords != null) && addRecords.equalsIgnoreCase("true"))
targetDS.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
}
/* public void merge(NetcdfDataset targetDS, Element parentElem) throws IOException {
// the root group
readGroup(targetDS, targetDS, null, null, parentElem);
// transfer from groups to global containers
targetDS.finish();
} */
////////////////////////////////////////////////////////////////////////
/**
* Read an NcML attribute element.
*
* @param parent Group or Variable
* @param refParent Group or Variable in reference dataset
* @param attElem ncml attribute element
*/
private void readAtt(Object parent, Object refParent, Element attElem) {
String name = attElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Attribute name is required (%s)%n", attElem);
return;
}
String nameInFile = attElem.getAttributeValue("orgName");
boolean newName = (nameInFile != null) && !nameInFile.equals(name);
if (nameInFile == null)
nameInFile = name;
else if (null == findAttribute(refParent, nameInFile)) { // has to exists
errlog.format("NcML attribute orgName '%s' doesnt exist. att=%s in=%s%n", nameInFile, name, parent);
return;
}
// see if its new
ucar.nc2.Attribute att = findAttribute(refParent, nameInFile);
if (att == null) { // new
if (debugConstruct) System.out.println(" add new att = " + name);
try {
ucar.ma2.Array values = readAttributeValues(attElem);
addAttribute(parent, new ucar.nc2.Attribute(name, values));
} catch (RuntimeException e) {
errlog.format("NcML new Attribute Exception: %s att=%s in=%s%n", e.getMessage(), name, parent);
}
} else { // already exists
if (debugConstruct) System.out.println(" modify existing att = " + name);
boolean hasValue = attElem.getAttribute("value") != null;
if (hasValue) {
try {
ucar.ma2.Array values = readAttributeValues(attElem);
addAttribute(parent, new ucar.nc2.Attribute(name, values));
} catch (RuntimeException e) {
errlog.format("NcML existing Attribute Exception: %s att=%s in=%s%n", e.getMessage(), name, parent);
return;
}
} else { // use the old values
addAttribute(parent, new ucar.nc2.Attribute(name, att.getValues()));
}
// remove the old one ??
if (newName && !explicit) {
removeAttribute(parent, att);
if (debugConstruct) System.out.println(" remove old att = " + nameInFile);
}
}
}
/**
* Parse the values element
*
* @param s JDOM element to parse
* @return Array with parsed values
* @throws IllegalArgumentException if string values not parsable to specified data type
*/
public static ucar.ma2.Array readAttributeValues(Element s) throws IllegalArgumentException {
String valString = s.getAttributeValue("value");
if (valString == null) throw new IllegalArgumentException("No value specified");
valString = StringUtil.unquoteXmlAttribute(valString);
String type = s.getAttributeValue("type");
DataType dtype = (type == null) ? DataType.STRING : DataType.getType(type);
if (dtype == DataType.CHAR) dtype = DataType.STRING;
String sep = s.getAttributeValue("separator");
if ((sep == null) && (dtype == DataType.STRING)) {
List<String> list = new ArrayList<String>();
list.add(valString);
return Array.makeArray(dtype, list);
}
if (sep == null) sep = " "; // default whitespace separated
List<String> stringValues = new ArrayList<String>();
StringTokenizer tokn = new StringTokenizer(valString, sep);
while (tokn.hasMoreTokens())
stringValues.add(tokn.nextToken());
return Array.makeArray(dtype, stringValues);
}
private ucar.nc2.Attribute findAttribute(Object parent, String name) {
if (parent == null)
return null;
if (parent instanceof Group)
return ((Group) parent).findAttribute(name);
else if (parent instanceof Variable)
return ((Variable) parent).findAttribute(name);
return null;
}
private void addAttribute(Object parent, ucar.nc2.Attribute att) {
if (parent instanceof Group)
((Group) parent).addAttribute(att);
else if (parent instanceof Variable)
((Variable) parent).addAttribute(att);
}
private void removeAttribute(Object parent, Attribute att) {
if (parent instanceof Group)
((Group) parent).remove(att);
else if (parent instanceof Variable)
((Variable) parent).remove(att);
}
/**
* Read an NcML dimension element.
*
* @param g put dimension into this group
* @param refg parent Group in referenced dataset
* @param dimElem ncml dimension element
*/
private void readDim(Group g, Group refg, Element dimElem) {
String name = dimElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Dimension name is required (%s)%n", dimElem);
return;
}
String nameInFile = dimElem.getAttributeValue("orgName");
if (nameInFile == null) nameInFile = name;
// see if it already exists
Dimension dim = (refg == null) ? null : refg.findDimension(nameInFile);
if (dim == null) { // nope - create it
String lengthS = dimElem.getAttributeValue("length");
String isUnlimitedS = dimElem.getAttributeValue("isUnlimited");
String isSharedS = dimElem.getAttributeValue("isShared");
String isUnknownS = dimElem.getAttributeValue("isVariableLength");
boolean isUnlimited = (isUnlimitedS != null) && isUnlimitedS.equalsIgnoreCase("true");
boolean isUnknown = (isUnknownS != null) && isUnknownS.equalsIgnoreCase("true");
boolean isShared = true;
if ((isSharedS != null) && isSharedS.equalsIgnoreCase("false"))
isShared = false;
int len = Integer.parseInt(lengthS);
if (isUnknown)
len = Dimension.VLEN.getLength();
if (debugConstruct) System.out.println(" add new dim = " + name);
g.addDimension(new Dimension(name, len, isShared, isUnlimited, isUnknown));
} else { // yes - modify it
dim.setName(name);
String lengthS = dimElem.getAttributeValue("length");
String isUnlimitedS = dimElem.getAttributeValue("isUnlimited");
String isSharedS = dimElem.getAttributeValue("isShared");
String isUnknownS = dimElem.getAttributeValue("isVariableLength");
if (isUnlimitedS != null)
dim.setUnlimited(isUnlimitedS.equalsIgnoreCase("true"));
if (isSharedS != null)
dim.setShared(!isSharedS.equalsIgnoreCase("false"));
if (isUnknownS != null)
dim.setVariableLength(isUnknownS.equalsIgnoreCase("true"));
if ((lengthS != null) && !dim.isVariableLength()) {
int len = Integer.parseInt(lengthS);
dim.setLength(len);
}
if (debugConstruct) System.out.println(" modify existing dim = " + name);
if (g != refg) // explicit, copy to new
g.addDimension(dim);
}
}
/**
* Read the NcML group element, and nested elements.
*
* @param newds new dataset
* @param refds referenced dataset
* @param parent Group
* @param refParent parent Group in referenced dataset
* @param groupElem ncml group element
*/
private void readGroup(NetcdfDataset newds, NetcdfFile refds, Group parent, Group refParent, Element groupElem) throws IOException {
Group g, refg = null;
if (parent == null) { // this is the <netcdf> element
g = newds.getRootGroup();
refg = refds.getRootGroup();
if (debugConstruct) System.out.println(" root group ");
} else {
String name = groupElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Group name is required (%s)%n", groupElem);
return;
}
String nameInFile = groupElem.getAttributeValue("orgName");
if (nameInFile == null) nameInFile = name;
// see if it exists in referenced dataset
if (refParent != null)
refg = refParent.findGroup(nameInFile);
if (refg == null) { // new
g = new Group(newds, parent, name);
parent.addGroup(g);
if (debugConstruct) System.out.println(" add new group = " + name);
} else {
if (parent != refParent) { // explicit
g = new Group(newds, parent, name);
parent.addGroup(g);
if (debugConstruct) System.out.println(" transfer existing group = " + name);
} else { // modify
g = refg;
if (!nameInFile.equals(name))
g.setName(name);
if (debugConstruct) System.out.println(" modify existing group = " + name);
}
}
}
// look for attributes
java.util.List<Element> attList = groupElem.getChildren("attribute", ncNS);
for (Element attElem : attList) {
readAtt(g, refg, attElem);
}
// look for dimensions
java.util.List<Element> dimList = groupElem.getChildren("dimension", ncNS);
for (Element dimElem : dimList) {
readDim(g, refg, dimElem);
}
// look for variables
java.util.List<Element> varList = groupElem.getChildren("variable", ncNS);
for (Element varElem : varList) {
readVariable(newds, g, refg, varElem);
}
// process remove command
java.util.List<Element> removeList = groupElem.getChildren("remove", ncNS);
for (Element e : removeList) {
cmdRemove(g, e.getAttributeValue("type"), e.getAttributeValue("name"));
}
// look for nested groups
java.util.List<Element> groupList = groupElem.getChildren("group", ncNS);
for (Element gElem : groupList) {
readGroup(newds, refds, g, refg, gElem);
if (debugConstruct) System.out.println(" add group = " + g.getName());
}
}
/* private boolean debugView = false, debugConvert = false;
protected VariableDS readVariable2( NetcdfDataset ds, Element varElem) {
VariableDS v = readVariable( ds, varElem);
// look for logical views
java.util.List viewList = varElem.getChildren("logicalView", ncNS);
for (int j=0; j< viewList.size(); j++) {
Element viewElem = (Element) viewList.get(j);
String value = viewElem.getAttributeValue("section");
if (value != null) {
v.setLogicalView("section", value);
if (debugView) System.out.println("set view = "+value);
}
}
// look for unit conversion
Element unitElem = varElem.getChild("units", ncNS);
if (unitElem != null) {
String value = unitElem.getAttributeValue("convertTo");
if (value != null) {
v.setConvertUnit(value);
if (debugConvert) System.out.println("setConvertUnit on "+v.getName()+" to <" + value+">");
}
}
return v;
} */
/**
* Read the NcML variable element, and nested elements.
*
* @param ds target dataset
* @param g parent Group
* @param refg referenced dataset parent Group - may be same (modify) or different (explicit)
* @param varElem ncml variable element
* @throws java.io.IOException on read error
*/
private void readVariable(NetcdfDataset ds, Group g, Group refg, Element varElem) throws IOException {
String name = varElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Variable name is required (%s)%n", varElem);
return;
}
String nameInFile = varElem.getAttributeValue("orgName");
if (nameInFile == null) nameInFile = name;
// see if it already exists
Variable refv = (refg == null) ? null : refg.findVariable(nameInFile);
if (refv == null) { // new
if (debugConstruct) System.out.println(" add new var = " + name);
g.addVariable(readVariableNew(ds, g, null, varElem));
return;
}
// exists already
DataType dtype = null;
String typeS = varElem.getAttributeValue("type");
if (typeS != null)
dtype = DataType.getType(typeS);
else
dtype = refv.getDataType();
String shape = varElem.getAttributeValue("shape");
Variable v;
if (refg == g) { // modify
v = refv;
v.setName(name);
/* if (dtype != v.getDataType() && v.hasCachedData()) {
Array data = v.read();
Array newData = Array.factory(dtype, v.getShape());
MAMath.copy(newData, data);
v.setCachedData(newData, false);
} */
v.setDataType(dtype);
if (shape != null)
v.setDimensions(shape); // LOOK check conformable
if (debugConstruct) System.out.println(" modify existing var = " + nameInFile);
} else { //explicit - create new
if (refv instanceof Structure) {
v = new StructureDS(ds, g, null, name, (Structure) refv);
v.setDimensions(shape);
} else if (refv instanceof Sequence) {
v = new StructureDS(ds, g, null, name, (Structure) refv);
v.setDimensions(shape);
} else {
v = new VariableDS(g, null, name, refv);
v.setDataType(dtype);
v.setDimensions(shape);
}
if (debugConstruct) System.out.println(" modify explicit var = " + nameInFile);
g.addVariable(v);
}
java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
for (Element attElem : attList) {
readAtt(v, refv, attElem);
}
// process remove command
java.util.List<Element> removeList = varElem.getChildren("remove", ncNS);
for (Element remElem : removeList) {
cmdRemove(v, remElem.getAttributeValue("type"), remElem.getAttributeValue("name"));
}
if (v.getDataType() == DataType.STRUCTURE) {
// deal with nested variables
StructureDS s = (StructureDS) v;
StructureDS refS = (StructureDS) refv;
java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
for (Element vElem : varList) {
readVariableNested(ds, s, refS, vElem);
}
} else {
// deal with values
Element valueElem = varElem.getChild("values", ncNS);
if (valueElem != null) {
readValues(ds, v, varElem, valueElem);
} else {
// see if we need to munge existing data. use case : aggregation
if (v.hasCachedData()) {
Array data;
try {
data = v.read();
} catch (IOException e) {
throw new IllegalStateException(e.getMessage());
}
if (data.getClass() != v.getDataType().getPrimitiveClassType()) {
Array newData = Array.factory(v.getDataType(), v.getShape());
MAMath.copy(newData, data);
v.setCachedData(newData, false);
}
}
}
}
// look for logical views
Element viewElem = varElem.getChild("logicalSection", ncNS);
if (null != viewElem) {
String sectionSpec = viewElem.getAttributeValue("section");
if (sectionSpec != null) {
try {
Section s = new Section(sectionSpec); // parse spec
Section viewSection = Section.fill(s, v.getShape());
// check that its a subset
if (!v.getShapeAsSection().contains(viewSection)) {
errlog.format("Invalid logicalSection on variable=%s section =(%s) original=(%s) %n", v.getFullName(), sectionSpec, v.getShapeAsSection());
return;
}
Variable view = v.section(viewSection);
g.removeVariable(v.getShortName());
g.addVariable(view);
} catch (InvalidRangeException e) {
errlog.format("Invalid logicalSection on variable=%s section=(%s) error=%s %n", v.getFullName(), sectionSpec, e.getMessage());
return;
}
}
}
viewElem = varElem.getChild("logicalSlice", ncNS);
if (null != viewElem) {
String dimName = viewElem.getAttributeValue("dimName");
if (null == dimName) {
errlog.format("NcML logicalSlice: dimName is required, variable=%s %n", v.getFullName());
return;
}
int dim = v.findDimensionIndex(dimName);
if (dim < 0) {
errlog.format("NcML logicalSlice: cant find dimension %s in variable=%s %n", dimName, v.getFullName());
return;
}
String indexS = viewElem.getAttributeValue("index");
int index = -1;
if (null == indexS) {
errlog.format("NcML logicalSlice: index is required, variable=%s %n", v.getFullName());
return;
}
try {
index = Integer.parseInt(indexS);
} catch (NumberFormatException e) {
errlog.format("NcML logicalSlice: index=%s must be integer, variable=%s %n", indexS, v.getFullName());
return;
}
try {
Variable view = v.slice(dim, index);
g.removeVariable(v.getShortName());
g.addVariable(view);
} catch (InvalidRangeException e) {
errlog.format("Invalid logicalSlice (%d,%d) on variable=%s error=%s %n", dim, index, v.getFullName(), e.getMessage());
return;
}
}
}
/**
* Read a NcML variable element, and nested elements, when it creates a new Variable.
*
* @param ds target dataset
* @param g parent Group
* @param parentS parent Structure
* @param varElem ncml variable element
* @return return new Variable
*/
private Variable readVariableNew(NetcdfDataset ds, Group g, Structure parentS, Element varElem) {
String name = varElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Variable name is required (%s)%n", varElem);
return null;
}
String type = varElem.getAttributeValue("type");
if (type == null)
throw new IllegalArgumentException("New variable (" + name + ") must have datatype attribute");
DataType dtype = DataType.getType(type);
String shape = varElem.getAttributeValue("shape");
if (shape == null)
shape = ""; // deprecated, prefer explicit ""
Variable v;
if (dtype == DataType.STRUCTURE) {
StructureDS s = new StructureDS(ds, g, parentS, name, shape, null, null);
v = s;
// look for nested variables
java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
for (Element vElem : varList) {
readVariableNested(ds, s, s, vElem);
}
} else if (dtype == DataType.SEQUENCE) {
Sequence org = new Sequence(ds, g, parentS, name);
SequenceDS s = new SequenceDS(g, org); // barf
v = s;
// look for nested variables
java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
for (Element vElem : varList) {
readVariableNested(ds, s, s, vElem);
}
} else {
v = new VariableDS(ds, g, parentS, name, dtype, shape, null, null);
// deal with values
Element valueElem = varElem.getChild("values", ncNS);
if (valueElem != null)
readValues(ds, v, varElem, valueElem);
// otherwise has fill values.
}
// look for attributes
java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
for (Element attElem : attList)
readAtt(v, null, attElem);
/* now that we have attributes finalized, redo the enhance
if (enhance && (v instanceof VariableDS))
((VariableDS) v).enhance(); */
return v;
}
/**
* Read the NcML variable element, and nested elements.
*
* @param ds target dataset
* @param parentS parent Structure
* @param refStruct reference dataset structure
* @param varElem ncml variable element
*/
private void readVariableNested(NetcdfDataset ds, Structure parentS, Structure refStruct, Element varElem) {
String name = varElem.getAttributeValue("name");
if (name == null) {
errlog.format("NcML Variable name is required (%s)%n", varElem);
return;
}
String nameInFile = varElem.getAttributeValue("orgName");
if (nameInFile == null) nameInFile = name;
// see if it already exists
Variable refv = refStruct.findVariable(nameInFile);
if (refv == null) { // new
if (debugConstruct) System.out.println(" add new var = " + name);
Variable nested = readVariableNew(ds, parentS.getParentGroup(), parentS, varElem);
parentS.addMemberVariable(nested);
return;
}
Variable v;
if (parentS == refStruct) { // modify
v = refv;
v.setName(name);
} else { //explicit
if (refv instanceof Structure) {
v = new StructureDS(parentS.getParentGroup(), (Structure) refv); // true
v.setName(name);
v.setParentStructure(parentS);
} else {
v = new VariableDS(parentS.getParentGroup(), refv, false);
v.setName(name);
v.setParentStructure(parentS);
}
/* if (refv instanceof Structure) {
v = new StructureDS(ds, parentS.getParentGroup(), parentS, name, refv.getDimensionsString(), null, null);
} else {
v = new VariableDS(ds, parentS.getParentGroup(), parentS, name, refv.getDataType(), refv.getDimensionsString(), null, null);
}
v.setIOVar(refv); */
parentS.addMemberVariable(v);
}
if (debugConstruct) System.out.println(" modify existing var = " + nameInFile);
String typeS = varElem.getAttributeValue("type");
if (typeS != null) {
DataType dtype = DataType.getType(typeS);
v.setDataType(dtype);
}
String shape = varElem.getAttributeValue("shape");
if (shape != null) {
v.setDimensions(shape);
}
java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
for (Element attElem : attList) {
readAtt(v, refv, attElem);
}
// process remove command
java.util.List<Element> removeList = varElem.getChildren("remove", ncNS);
for (Element remElem : removeList) {
cmdRemove(v, remElem.getAttributeValue("type"), remElem.getAttributeValue("name"));
}
if ((v.getDataType() == DataType.STRUCTURE) || (v.getDataType() == DataType.SEQUENCE)) {
// deal with nested variables
StructureDS s = (StructureDS) v;
StructureDS refS = (StructureDS) refv;
java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
for (Element vElem : varList) {
readVariableNested(ds, s, refS, vElem);
}
} else {
// deal with values
Element valueElem = varElem.getChild("values", ncNS);
if (valueElem != null)
readValues(ds, v, varElem, valueElem);
}
/* now that we have attributes finalized, redo the enhance
if (enhance && (v instanceof VariableDS))
((VariableDS) v).enhance(); */
}
private void readValues(NetcdfDataset ds, Variable v, Element varElem, Element valuesElem) {
// check if values are specified by attribute
String fromAttribute = valuesElem.getAttributeValue("fromAttribute");
if (fromAttribute != null) {
Attribute att = null;
int pos = fromAttribute.indexOf('@'); // varName@attName
if (pos > 0) {
String varName = fromAttribute.substring(0, pos);
String attName = fromAttribute.substring(pos + 1);
Variable vFrom = ds.getRootGroup().findVariable(varName); // LOOK groups
if (vFrom == null) {
errlog.format("Cant find variable %s %n", fromAttribute);
return;
}
att = vFrom.findAttribute(attName);
} else { // attName or @attName
String attName = (pos == 0) ? fromAttribute.substring(1) : fromAttribute;
att = ds.getRootGroup().findAttribute(attName);
}
if (att == null) {
errlog.format("Cant find attribute %s %n", fromAttribute);
return;
}
Array data = att.getValues();
v.setCachedData(data, true);
return;
}
// check if values are specified by start / increment
String startS = valuesElem.getAttributeValue("start");
String incrS = valuesElem.getAttributeValue("increment");
String nptsS = valuesElem.getAttributeValue("npts");
int npts = (nptsS == null) ? (int) v.getSize() : Integer.parseInt(nptsS);
// either start, increment are specified
if ((startS != null) && (incrS != null)) {
double start = Double.parseDouble(startS);
double incr = Double.parseDouble(incrS);
ds.setValues(v, npts, start, incr);
return;
}
// otherwise values are listed in text
String values = varElem.getChildText("values", ncNS);
String sep = valuesElem.getAttributeValue("separator");
if (sep == null) sep = " ";
if (v.getDataType() == DataType.CHAR) {
int nhave = values.length();
int nwant = (int) v.getSize();
char[] data = new char[nwant];
int min = Math.min(nhave, nwant);
for (int i = 0; i < min; i++) {
data[i] = values.charAt(i);
}
Array dataArray = Array.factory(DataType.CHAR.getPrimitiveClassType(), v.getShape(), data);
v.setCachedData(dataArray, true);
} else {
// or a list of values
List<String> valList = new ArrayList<String>();
StringTokenizer tokn = new StringTokenizer(values, sep);
while (tokn.hasMoreTokens())
valList.add(tokn.nextToken());
ds.setValues(v, valList);
}
}
/////////////////////////////////////////////////////////////////////////////////////////
private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset newds, CancelTask cancelTask) throws IOException {
String dimName = aggElem.getAttributeValue("dimName");
String type = aggElem.getAttributeValue("type");
String recheck = aggElem.getAttributeValue("recheckEvery");
Aggregation agg;
if (type.equals("joinExisting")) {
agg = new AggregationExisting(newds, dimName, recheck);
} else if (type.equals("joinNew")) {
agg = new AggregationNew(newds, dimName, recheck);
} else if (type.equals("tiled")) {
agg = new AggregationTiled(newds, dimName, recheck);
} else if (type.equals("union")) {
agg = new AggregationUnion(newds, dimName, recheck);
} else if (type.equals("forecastModelRunCollection") || type.equals("forecastModelRunSingleCollection")) {
AggregationFmrc aggc = new AggregationFmrc(newds, dimName, recheck);
agg = aggc;
// nested scanFmrc elements
java.util.List<Element> scan2List = aggElem.getChildren("scanFmrc", ncNS);
for (Element scanElem : scan2List) {
String dirLocation = scanElem.getAttributeValue("location");
String regexpPatternString = scanElem.getAttributeValue("regExp");
String suffix = scanElem.getAttributeValue("suffix");
String subdirs = scanElem.getAttributeValue("subdirs");
String olderS = scanElem.getAttributeValue("olderThan");
String runMatcher = scanElem.getAttributeValue("runDateMatcher");
String forecastMatcher = scanElem.getAttributeValue("forecastDateMatcher");
String offsetMatcher = scanElem.getAttributeValue("forecastOffsetMatcher");
// possible relative location
dirLocation = URLnaming.resolve(ncmlLocation, dirLocation);
aggc.addDirectoryScanFmrc(dirLocation, suffix, regexpPatternString, subdirs, olderS, runMatcher, forecastMatcher, offsetMatcher);
if ((cancelTask != null) && cancelTask.isCancel())
return null;
if (debugAggDetail) System.out.println(" debugAgg: nested dirLocation = " + dirLocation);
}
} else {
throw new IllegalArgumentException("Unknown aggregation type=" + type);
}
if (agg instanceof AggregationOuterDimension) {
AggregationOuterDimension aggo = (AggregationOuterDimension) agg;
String timeUnitsChange = aggElem.getAttributeValue("timeUnitsChange");
if (timeUnitsChange != null)
aggo.setTimeUnitsChange(timeUnitsChange.equalsIgnoreCase("true"));
// look for variables that need to be aggregated (aggNew)
java.util.List<Element> list = aggElem.getChildren("variableAgg", ncNS);
for (Element vaggElem : list) {
String varName = vaggElem.getAttributeValue("name");
aggo.addVariable(varName);
}
// look for attributes to promote to variables
list = aggElem.getChildren("promoteGlobalAttribute", ncNS);
for (Element gattElem : list) {
String varName = gattElem.getAttributeValue("name");
String orgName = gattElem.getAttributeValue("orgName");
aggo.addVariableFromGlobalAttribute(varName, orgName);
}
// look for attributes to promote to variables
list = aggElem.getChildren("promoteGlobalAttributeCompose", ncNS);
for (Element gattElem : list) {
String varName = gattElem.getAttributeValue("name");
String format = gattElem.getAttributeValue("format");
String orgName = gattElem.getAttributeValue("orgName");
aggo.addVariableFromGlobalAttributeCompose(varName, format, orgName);
}
// look for variable to cache
list = aggElem.getChildren("cacheVariable", ncNS);
for (Element gattElem : list) {
String varName = gattElem.getAttributeValue("name");
aggo.addCacheVariable(varName, null);
}
}
// nested netcdf elements
java.util.List<Element> ncList = aggElem.getChildren("netcdf", ncNS);
for (Element netcdfElemNested : ncList) {
String location = netcdfElemNested.getAttributeValue("location");
if (location == null)
location = netcdfElemNested.getAttributeValue("uri");
String id = netcdfElemNested.getAttributeValue("id");
String ncoords = netcdfElemNested.getAttributeValue("ncoords");
String coordValueS = netcdfElemNested.getAttributeValue("coordValue");
String sectionSpec = netcdfElemNested.getAttributeValue("section");
// must always open through a NcML reader, in case the netcdf element modifies the dataset
NcmlElementReader reader = new NcmlElementReader(ncmlLocation, location, netcdfElemNested);
String cacheName = ncmlLocation + "#" + Integer.toString(netcdfElemNested.hashCode());
agg.addExplicitDataset(cacheName, location, id, ncoords, coordValueS, sectionSpec, reader);
if ((cancelTask != null) && cancelTask.isCancel())
return null;
if (debugAggDetail) System.out.println(" debugAgg: nested dataset = " + location);
}
// nested scan elements
java.util.List<Element> dirList = aggElem.getChildren("scan", ncNS);
for (Element scanElem : dirList) {
String dirLocation = scanElem.getAttributeValue("location");
String regexpPatternString = scanElem.getAttributeValue("regExp");
String suffix = scanElem.getAttributeValue("suffix");
String subdirs = scanElem.getAttributeValue("subdirs");
String olderS = scanElem.getAttributeValue("olderThan");
String dateFormatMark = scanElem.getAttributeValue("dateFormatMark");
Set<NetcdfDataset.Enhance> enhanceMode = NetcdfDataset.parseEnhanceMode(scanElem.getAttributeValue("enhance"));
// possible relative location
dirLocation = URLnaming.resolve(ncmlLocation, dirLocation);
// can embed a full-blown crawlableDatasetImpl element
Element cdElement = scanElem.getChild("crawlableDatasetImpl", ncNS); // ok if null
agg.addDatasetScan(cdElement, dirLocation, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs, olderS);
if ((cancelTask != null) && cancelTask.isCancel())
return null;
if (debugAggDetail) System.out.println(" debugAgg: nested dirLocation = " + dirLocation);
}
// experimental
Element collElem = aggElem.getChild("collection", ncNS);
if (collElem != null)
agg.addCollection(collElem.getAttributeValue("spec"), collElem.getAttributeValue("olderThan"));
/* <!-- experimental - modify each dataset in aggregation -->
<xsd:choice minOccurs="0" maxOccurs="unbounded">
<xsd:element ref="group"/>
<xsd:element ref="dimension"/>
<xsd:element ref="variable"/>
<xsd:element ref="attribute"/>
<xsd:element ref="remove"/>
</xsd:choice> */
boolean needMerge = aggElem.getChildren("attribute", ncNS).size() > 0;
if (!needMerge) needMerge = aggElem.getChildren("variable", ncNS).size() > 0;
if (!needMerge) needMerge = aggElem.getChildren("dimension", ncNS).size() > 0;
if (!needMerge) needMerge = aggElem.getChildren("group", ncNS).size() > 0;
if (!needMerge) needMerge = aggElem.getChildren("remove", ncNS).size() > 0;
if (needMerge)
agg.setModifications(aggElem);
return agg;
}
private class NcmlElementReader implements ucar.nc2.util.cache.FileFactory {
private Element netcdfElem;
private String ncmlLocation, location;
NcmlElementReader(String ncmlLocation, String location, Element netcdfElem) {
this.ncmlLocation = ncmlLocation;
this.location = location;
this.netcdfElem = netcdfElem;
}
public NetcdfFile open(String cacheName, int buffer_size, CancelTask cancelTask, Object spiObject) throws IOException {
if (debugAggDetail) System.out.println(" NcmlElementReader open nested dataset " + cacheName);
NetcdfFile result = readNcML(ncmlLocation, location, netcdfElem, cancelTask);
result.setLocation(ncmlLocation + "#" + location);
return result;
}
}
/////////////////////////////////////////////
// command procesing
private void cmdRemove(Group g, String type, String name) {
boolean err = false;
if (type.equals("dimension")) {
Dimension dim = g.findDimension(name);
if (dim != null) {
g.remove(dim);
if (debugCmd) System.out.println("CMD remove " + type + " " + name);
} else
err = true;
} else if (type.equals("variable")) {
Variable v = g.findVariable(name);
if (v != null) {
g.remove(v);
if (debugCmd) System.out.println("CMD remove " + type + " " + name);
} else
err = true;
} else if (type.equals("attribute")) {
ucar.nc2.Attribute a = g.findAttribute(name);
if (a != null) {
g.remove(a);
if (debugCmd) System.out.println("CMD remove " + type + " " + name);
} else
err = true;
}
if (err) {
Formatter f = new Formatter();
f.format("CMD remove %s CANT find %s location %s%n", type, name, location);
log.info(f.toString());
}
}
private void cmdRemove(Variable v, String type, String name) {
boolean err = false;
if (type.equals("attribute")) {
ucar.nc2.Attribute a = v.findAttribute(name);
if (a != null) {
v.remove(a);
if (debugCmd) System.out.println("CMD remove " + type + " " + name);
} else
err = true;
} else if (type.equals("variable") && v instanceof Structure) {
Structure s = (Structure) v;
Variable nested = s.findVariable(name);
if (nested != null) {
s.removeMemberVariable(nested);
if (debugCmd) System.out.println("CMD remove " + type + " " + name);
} else
err = true;
}
if (err) {
Formatter f = new Formatter();
f.format("CMD remove %s CANT find %s location %s%n", type, name, location);
log.info(f.toString());
}
}
///////////////////////////////////////////////////////////////////////////////////
/**
* Read an NcML file and write an equivilent NetcdfFile to a physical file, using Netcdf-3 file format.
*
* @param ncmlLocation read this NcML file
* @param fileOutName write to this local file
* @throws IOException on write error
* @see ucar.nc2.FileWriter#writeToFile
*/
public static void writeNcMLToFile(String ncmlLocation, String fileOutName) throws IOException {
NetcdfFile ncd = NetcdfDataset.acquireFile(ncmlLocation, null);
//int dataMode = (ncd.getReferencedFile() != null) ? 1 : 2;
NetcdfFile ncdnew = ucar.nc2.FileWriter.writeToFile(ncd, fileOutName);
ncd.close();
ncdnew.close();
}
/**
* Read an NcML and write an equivilent NetcdfFile to a physical file, using Netcdf-3 file format.
* The NcML may have a referenced dataset in the location URL, in which case the underlying data
* (modified by the NcML is written to the new file. If the NcML does not have a referenced dataset,
* then the new file is filled with fill values, like ncgen.
*
* @param ncml read NcML from this input stream
* @param fileOutName write to this local file
* @throws IOException on error
* @see ucar.nc2.FileWriter#writeToFile
*/
public static void writeNcMLToFile(InputStream ncml, String fileOutName) throws IOException {
NetcdfDataset ncd = NcMLReader.readNcML(ncml, null);
NetcdfFile ncdnew = ucar.nc2.FileWriter.writeToFile(ncd, fileOutName, true);
ncd.close();
ncdnew.close();
}
public static void main(String arg[]) {
String ncmlFile = "C:/data/AStest/oots/test.ncml";
String ncmlFileOut = "C:/TEMP/testNcmlOut.nc";
try {
//NetcdfDataset ncd = NcMLReader.readNcML (ncmlFile, null);
//ncd.writeNcMLG(System.out, true, null);
//System.out.println("NcML = "+ncmlFile);
InputStream in = new FileInputStream(ncmlFile);
writeNcMLToFile(in, ncmlFileOut);
} catch (Exception ioe) {
System.out.println("error = " + ncmlFile);
ioe.printStackTrace();
}
}
}