/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.oodt.opendapps.util;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.ENUM_ELEMENT_TYPE;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.PROF_ATTR_SPEC_TYPE;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.PROF_ELEM_SPEC_TYPE;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.RANGED_ELEMENT_TYPE;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.RES_ATTR_SPEC_TYPE;
//JDK imports
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
//OPENDAP imports
import opendap.dap.BaseType;
import opendap.dap.DArray;
import opendap.dap.DConnect;
import opendap.dap.DDS;
import opendap.dap.DGrid;
//OODT imports
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.opendapps.OpendapProfileElementExtractor;
import org.apache.oodt.opendapps.config.ConstantSpec;
import org.apache.oodt.opendapps.config.DatasetMetElem;
import org.apache.oodt.opendapps.config.OpendapConfig;
import org.apache.oodt.opendapps.config.RewriteSpec;
import org.apache.oodt.profile.EnumeratedProfileElement;
import org.apache.oodt.profile.Profile;
import org.apache.oodt.profile.ProfileAttributes;
import org.apache.oodt.profile.ProfileElement;
import org.apache.oodt.profile.ResourceAttributes;
import org.springframework.util.StringUtils;
/**
*
* Static methods for unraveling and generating {@link ProfileElement}s,
* {@link ProfileAttributes} and {@link ResourceAttributes} for a
* {@link Profile}, derived from a set of OPeNDAP dataset information and an
* {@link OpendapConfig}.
*
*
*/
public class ProfileUtils {
static {
// Note: must override the CAS PathUtils delimiter otherwise every sentence is split at the ',' as different metadata fields.
// The delimiter must be a character that is not commonly used in the metadata values,
// and that it does not a special regular expression character.
// Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
// Cannot user '?', '&' as they are used in URL query strings.
// Cannot use '|' as it is used as multi-part separators in encoding of metadata fields.
PathUtils.DELIMITER = "~";
}
// character separating multiple parts of the same metadata field,
// when more than one piece of information needs to be stored in one field
public final static String CHAR = "|";
// HTTP mime types
public final static String MIME_TYPE_THREDDS = "application/xml+thredds";
public final static String MIME_TYPE_NETCDF = "application/netcdf";
public final static String MIME_TYPE_GRIDFTP = "application/gridftp";
public final static String MIME_TYPE_FTP = "application/ftp";
public final static String MIME_TYPE_LAS = "application/las";
public final static String MIME_TYPE_HTML = "text/html";
public final static String MIME_TYPE_GOOGLE_EARTH = "application/vnd.google-earth.kmz";
public final static String MIME_TYPE_HDF = "application/x-hdf";
public final static String MIME_TYPE_OPENDAP = "application/opendap";
public final static String MIME_TYPE_OPENDAP_DODS = "application/opendap-dods";
public final static String MIME_TYPE_OPENDAP_DAS = "application/opendap-das";
public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
public final static String MIME_TYPE_RSS = "application/rss+xml";
public final static String MIME_TYPE_GIS = "application/gis";
private static final Logger LOG = Logger.getLogger(ProfileUtils.class
.getName());
public static ResourceAttributes getResourceAttributes(OpendapConfig conf,
String opendapUrl, DConnect dConn, Metadata datasetMet) {
ResourceAttributes resAttr = new ResourceAttributes();
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
try {
// first process expanded '[@...]' instructions
List<String> values = multipleEnvVariablesReplacement(spec.getValue(), datasetMet);
// then process standard '[...]' instructions
for (String value : values) {
String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
if (StringUtils.hasText(_value)) {
setResourceAttributes(resAttr, spec.getName(), _value);
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
+ "] in resource attributes: Message: " + e.getMessage());
}
}
}
return resAttr;
}
/**
* Utility method to process environment replacement instructions of the form '[@key]'
* resulting in as many output values as there are values for the environment variable 'key'.
* Note that currently only one such pattern '[@key']' can be processed.
*
* @param value
* @param metadata
* @return
*/
private static List<String> multipleEnvVariablesReplacement(String value, Metadata metadata) {
List<String> newValues = new ArrayList<String>();
// regexp matching found > replace values
int start = value.indexOf("[@");
if (start>=0) {
int end = value.indexOf("]",start+2);
// remove '[@',']' to obtain environment variable key
String envKey = value.substring(start+2,end);
List<String> envValues = metadata.getAllMetadata(envKey);
if (envValues!=null) {
for (String envValue : envValues) {
// create new metadata value for this environment replacement
String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
newValues.add(newValue);
}
}
// regexp matching not found > return original value
} else {
newValues.add(value);
}
return newValues;
}
public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet) {
ProfileAttributes profAttr = new ProfileAttributes();
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(PROF_ATTR_SPEC_TYPE)) {
setProfileAttributesProperty(profAttr, spec.getName(), PathUtils
.replaceEnvVariables(spec.getValue(), datasetMet, true));
}
}
return profAttr;
}
public static Map<String, ProfileElement> getProfileElements(
OpendapConfig conf, DConnect dConn, Metadata datasetMet, Profile profile) throws Exception {
OpendapProfileElementExtractor pe = new OpendapProfileElementExtractor(conf);
Map<String, ProfileElement> profElements = new HashMap<String, ProfileElement>();
// extracts all variables defined in DDS
try {
DDS dds = dConn.getDDS();
// loop over all variables found
Enumeration variables = dds.getVariables();
while (variables.hasMoreElements()) {
BaseType variable = (BaseType)variables.nextElement();
String varName = variable.getName();
if (variable instanceof DArray) {
LOG.log(Level.FINE, "Extracting Darray variable: "+varName);
} else if (variable instanceof DGrid) {
LOG.log(Level.FINE, "Extracting Dgrid variable: "+varName);
}
RewriteSpec spec = getProfileElementSpec(varName, conf);
if (spec!=null) {
// use configuration to set variable re-name and type
String peName = spec.getRename() != null && !spec.getRename().equals("") ? spec.getRename() : spec.getOrigName();
if (spec.getElementType().equals(RANGED_ELEMENT_TYPE)) {
profElements.put(peName, pe.extractRangedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
} else if (spec.getElementType().equals(ENUM_ELEMENT_TYPE)) {
profElements.put(peName, pe.extractEnumeratedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
}
} else {
// if not explicitly configured, assume variable if of RANGED_ELEMENT_TYPE
profElements.put(varName, pe.extractRangedProfileElement(varName, varName, profile, dConn.getDAS()));
}
}
} catch(Exception e) {
e.printStackTrace();
LOG.log(Level.WARNING, "Error extracting metadata from DDS ("+dConn.URL()+") :" +e.getMessage());
// rethrow the exception so that this dataset is not harvested
throw e;
}
// add profile elements from <datasetMetadata> specification
if (datasetMet != null) {
for (DatasetMetElem datasetSpec : conf.getDatasetMetSpecs()) {
// retrieve values from metadata container
List<String> values = datasetMet.getAllMetadata(datasetSpec.getValue());
addValuesToEnumeratedProfileElement(datasetSpec.getProfileElementName(), values, profile, profElements);
}
}
// add profile elements from <constants> specification
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(PROF_ELEM_SPEC_TYPE)) {
// retrieve value from XML configuration file, replace with value from metadata container if required,
// split according to delimiter
String replaceVal = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet);
List<String> values = Arrays.asList(replaceVal.split(PathUtils.DELIMITER));
addValuesToEnumeratedProfileElement(spec.getName(), values, profile, profElements);
}
}
return profElements;
}
/**
* Method to add one or more values to an EnumeratedProfileElement, creating the element if not existing already.
* The supplied values are added only if valid.
*/
private static void addValuesToEnumeratedProfileElement(final String name, final List<String> values, Profile profile, Map<String, ProfileElement> profElements) {
// try retrieve existing profile element
ProfileElement epe = profElements.get(name);
// or create a new one if not found
if (epe==null) {
epe = new EnumeratedProfileElement(profile);
epe.setName(name);
}
if (values!=null) {
for (String value : values) {
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
epe.getValues().add(value);
}
}
}
// only save profile elements with one or more values
if (epe.getValues().size()>0) profElements.put(name, epe);
}
private static void setProfileAttributesProperty(ProfileAttributes attr,
String propName, String value) {
if (propName.equals("profId")) {
attr.setID(value);
} else if (propName.equals("profVersion")) {
attr.setVersion(value);
} else if (propName.equals("profType")) {
attr.setType(value);
} else if (propName.equals("profStatusId")) {
attr.setStatusID(value);
} else if (propName.equals("profSecurityType")) {
attr.setSecurityType(value);
} else if (propName.equals("profParentId")) {
attr.setParent(value);
} else if (propName.equals("profRegAuthority")) {
attr.setRegAuthority(value);
} else if (propName.equals("profChildId")) {
attr.getChildren().addAll(Arrays.asList(value.split(PathUtils.DELIMITER)));
} else if (propName.equals("profRevisionNote")) {
attr.getRevisionNotes().addAll(Arrays.asList(value.split(PathUtils.DELIMITER)));
}
}
private static void setResourceAttributes(ResourceAttributes resAttr,
String propName, String value) {
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
if (propName.equals("Identifier")) {
resAttr.setIdentifier(value);
} else if (propName.equals("Title")) {
resAttr.setTitle(value);
} else if (propName.equals("Format")) {
resAttr.getFormats().addAll( parseValues(value) );
} else if (propName.equals("Description")) {
resAttr.setDescription(value);
} else if (propName.equals("Creator")) {
resAttr.getCreators().addAll( parseValues(value) );
} else if (propName.equals("Subject")) {
resAttr.getSubjects().addAll( parseValues(value) );
} else if (propName.equals("Publisher")) {
resAttr.getPublishers().addAll( parseValues(value) );
} else if (propName.equals("Contributor")) {
resAttr.getContributors().addAll( parseValues(value) );
} else if (propName.equals("Date")) {
resAttr.getDates().addAll( parseValues(value) );
} else if (propName.equals("Type")) {
resAttr.getTypes().addAll( parseValues(value) );
} else if (propName.equals("Source")) {
resAttr.getSources().addAll( parseValues(value) );
} else if (propName.equals("Language")) {
resAttr.getLanguages().addAll( parseValues(value) );
} else if (propName.equals("Relation")) {
resAttr.getRelations().addAll( parseValues(value) );
} else if (propName.equals("Coverage")) {
resAttr.getCoverages().addAll( parseValues(value) );
} else if (propName.equals("Rights")) {
resAttr.getRights().addAll( parseValues(value) );
} else if (propName.equals("resContext")) {
resAttr.getResContexts().addAll( parseValues(value) );
} else if (propName.equals("resAggregation")) {
resAttr.setResAggregation(value);
} else if (propName.equals("resClass")) {
resAttr.setResClass(value);
} else if (propName.equals("resLocation")) {
resAttr.getResLocations().addAll( parseValues(value) );
}
}
}
/**
* Utility method to discover the rewrite specification for a named variable, if available.
* @param name
* @param conf
*/
private static RewriteSpec getProfileElementSpec(String origName, OpendapConfig conf) {
for (RewriteSpec spec : conf.getRewriteSpecs()) {
if (spec.getOrigName().equals(origName)) {
return spec;
}
}
return null;
}
/**
* Utility method to split a metadata field value according to the known delimiter.
* @param value
* @return
*/
public static List<String> parseValues(String value) {
List<String> values = new ArrayList<String>();
for (String val : value.split(PathUtils.DELIMITER)) {
if (StringUtils.hasText(val) && !val.equalsIgnoreCase("null")) {
values.add(val);
}
}
return values;
}
/**
* Method to add a (name,value) pair to the metadata container if the value is not null or empty,
* and doesn't exist already.
* @param met
* @param field
* @param value
*/
public static void addIfNotNull(Metadata met, String key, String value) {
// do not add a null value
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
// do not add the same value twice for the same metadata key
if (!met.containsKey(key) || !met.getAllMetadata(key).contains(value)) {
met.addMetadata(key, value);
}
}
}
/**
* Method to add multiple (key, value) pairs to the metadata container if not existing already.
* @param met
* @param field
* @param value
*/
public static void addIfNotExisting(Metadata metadata, String key, Enumeration<String> values) {
if (StringUtils.hasText(key) && !metadata.containsKey(key)) {
while (values.hasMoreElements()) {
String value = values.nextElement();
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
metadata.addMetadata(key,value);
}
}
}
}
// inspired from ASLv2 code at:
// http://www.java2s.com/Code/Java/Data-Type/ISO8601dateparsingutility.htm
public static String toISO8601(Date date) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
TimeZone tz = TimeZone.getTimeZone("UTC");
df.setTimeZone(tz);
String output = df.format(date);
return output;
}
}