/**
* Copyright (C) 2009 - present by OpenGamma Inc. and the OpenGamma group of companies
*
* Please see distribution for license.
*/
package com.opengamma.masterdb.batch;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newConcurrentMap;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static com.opengamma.lambdava.streams.Lambdava.newArray;
import static com.opengamma.util.db.HibernateDbUtils.eqOrIsNull;
import static org.apache.commons.lang.StringUtils.defaultString;
import java.io.Serializable;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
import org.hibernate.Criteria;
import org.hibernate.HibernateException;
import org.hibernate.Query;
import org.hibernate.Session;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Restrictions;
import org.joda.beans.Bean;
import org.joda.beans.Property;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import org.springframework.orm.hibernate3.HibernateCallback;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.threeten.bp.Instant;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.opengamma.DataNotFoundException;
import com.opengamma.batch.RunCreationMode;
import com.opengamma.batch.SnapshotMode;
import com.opengamma.batch.domain.CalculationConfiguration;
import com.opengamma.batch.domain.ComputeFailure;
import com.opengamma.batch.domain.ComputeFailureKey;
import com.opengamma.batch.domain.ComputeHost;
import com.opengamma.batch.domain.ComputeNode;
import com.opengamma.batch.domain.FunctionUniqueId;
import com.opengamma.batch.domain.HbComputationTargetSpecification;
import com.opengamma.batch.domain.LiveDataField;
import com.opengamma.batch.domain.MarketData;
import com.opengamma.batch.domain.MarketDataValue;
import com.opengamma.batch.domain.RiskRun;
import com.opengamma.batch.domain.RiskValueRequirement;
import com.opengamma.batch.domain.RiskValueSpecification;
import com.opengamma.batch.domain.StatusEntry;
import com.opengamma.core.marketdatasnapshot.SnapshotDataBundle;
import com.opengamma.elsql.ElSqlBundle;
import com.opengamma.engine.ComputationTarget;
import com.opengamma.engine.ComputationTargetResolver;
import com.opengamma.engine.ComputationTargetSpecification;
import com.opengamma.engine.calcnode.InvocationResult;
import com.opengamma.engine.calcnode.MissingValue;
import com.opengamma.engine.value.ComputedValue;
import com.opengamma.engine.value.ComputedValueResult;
import com.opengamma.engine.value.ValueProperties;
import com.opengamma.engine.value.ValueRequirement;
import com.opengamma.engine.value.ValueSpecification;
import com.opengamma.engine.view.ExecutionLogWithContext;
import com.opengamma.engine.view.ViewCalculationResultModel;
import com.opengamma.engine.view.ViewComputationResultModel;
import com.opengamma.engine.view.ViewResultEntry;
import com.opengamma.engine.view.cycle.ViewCycleMetadata;
import com.opengamma.financial.conversion.ResultConverter;
import com.opengamma.financial.conversion.ResultConverterCache;
import com.opengamma.id.ObjectId;
import com.opengamma.id.UniqueId;
import com.opengamma.id.VersionCorrection;
import com.opengamma.masterdb.AbstractDbMaster;
import com.opengamma.util.ArgumentChecker;
import com.opengamma.util.db.DbConnector;
import com.opengamma.util.db.DbDateUtils;
import com.opengamma.util.db.DbMapSqlParameterSource;
import com.opengamma.util.tuple.Pair;
/**
* A batch master implementation using a database for persistence.
* <p>
* This is a full implementation of the batch master using an SQL database.
* This implementation uses Hibernate to write all static data, including LiveData snapshots.
* Risk itself is written using direct JDBC.
* <p>
* Full details of the API are in {@link com.opengamma.masterdb.batch.document.BatchMaster}.
* <p>
* This class is mutable but must be treated as immutable after configuration.
*/
public class DbBatchWriter extends AbstractDbMaster {
/**
* The default scheme for unique identifiers.
*/
public static final String IDENTIFIER_SCHEME_DEFAULT = "DbBat";
/**
* The batch risk sequence name.
*/
public static final String RSK_SEQUENCE_NAME = "rsk_batch_seq";
/**
* The set of types to skip when encountered in market data because they already belong to a snapshot.
*/
private static final Set<Class<?>> SKIP_MARKET_DATA_WRITE_TYPES = ImmutableSet.<Class<?>>of(SnapshotDataBundle.class);
private final Map<String, Long> _calculationConfigurations = newConcurrentMap();
private final Map<ValueRequirement, Long> _riskValueRequirements = newConcurrentMap();
private final Map<ValueSpecification, Long> _riskValueSpecifications = newConcurrentMap();
private final Map<ComputationTargetSpecification, Long> _computationTargets = newConcurrentMap();
private final Map<Long, RiskRun> _riskRunsByIds = newConcurrentMap();
private final Map<Long, Map<Pair<Long, Long>, StatusEntry>> _statusCacheByRunId = newConcurrentMap();
private final Map<Long, Map<ComputeFailureKey, ComputeFailure>> _computeFailureCacheByRunId = newConcurrentMap();
/** Logger. */
private static final Logger s_logger = LoggerFactory.getLogger(DbBatchWriter.class);
private final ComputationTargetResolver _computationTargetResolver;
/**
* The Result converter cache.
*/
private ResultConverterCache _resultConverterCache;
/**
* Creates an instance.
*
* @param dbConnector the database connector, not null
*/
public DbBatchWriter(final DbConnector dbConnector, final ComputationTargetResolver computationTargetResolver) {
super(dbConnector, IDENTIFIER_SCHEME_DEFAULT);
_resultConverterCache = new ResultConverterCache();
_computationTargetResolver = computationTargetResolver;
setElSqlBundle(ElSqlBundle.of(dbConnector.getDialect().getElSqlConfig(), DbBatchWriter.class));
}
public RiskRun getRiskRunById(final Long id) {
return getHibernateTemplate().execute(new HibernateCallback<RiskRun>() {
@Override
public RiskRun doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("RiskRun.one.byId");
query.setLong("id", id);
return (RiskRun) query.uniqueResult();
}
});
}
protected ComputeHost getOrCreateComputeHost(final String hostName) {
ComputeHost computeHost = getHibernateTemplate().execute(new HibernateCallback<ComputeHost>() {
@Override
public ComputeHost doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("ComputeHost.one.byHostName");
query.setString("hostName", hostName);
return (ComputeHost) query.uniqueResult();
}
});
if (computeHost == null) {
computeHost = new ComputeHost();
computeHost.setHostName(hostName);
getHibernateTemplate().save(computeHost);
getHibernateTemplate().flush();
}
return computeHost;
}
protected ComputeNode getOrCreateComputeNode(final String nodeId) {
ArgumentChecker.notNull(nodeId, "nodeId");
String hostName = nodeId;
int slashIndex = nodeId.indexOf('/'); // e.g., mymachine-t5500/0/1, see LocalCalculationNode.java. Should refactor nodeId to a class with two strings, host and node id
if (slashIndex != -1) {
hostName = nodeId.substring(0, slashIndex);
}
final ComputeHost host = getOrCreateComputeHost(hostName);
ComputeNode node = getHibernateTemplate().execute(new HibernateCallback<ComputeNode>() {
@Override
public ComputeNode doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("ComputeNode.one.byNodeName");
query.setString("nodeName", nodeId);
return (ComputeNode) query.uniqueResult();
}
});
if (node == null) {
node = new ComputeNode();
node.setComputeHost(host);
node.setNodeName(nodeId);
getHibernateTemplate().save(node);
getHibernateTemplate().flush();
}
return node;
}
protected MarketData getMarketDataInTransaction(final ObjectId snapshotId) {
MarketData liveDataValues = getHibernateTemplate().execute(new HibernateCallback<MarketData>() {
@Override
public MarketData doInHibernate(Session session) throws HibernateException, SQLException {
Long id = extractOid(snapshotId);
return (MarketData) session.get(MarketData.class, id);
}
});
if (liveDataValues == null) {
throw new IllegalArgumentException("Snapshot for " + snapshotId + " cannot be found");
}
return liveDataValues;
}
protected LiveDataField getLiveDataField(final String fieldName) {
LiveDataField field = getHibernateTemplate().execute(new HibernateCallback<LiveDataField>() {
@Override
public LiveDataField doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("LiveDataField.one.byName");
query.setString("name", fieldName);
return (LiveDataField) query.uniqueResult();
}
});
if (field == null) {
field = new LiveDataField();
field.setName(fieldName);
getHibernateTemplate().save(field);
getHibernateTemplate().flush();
}
return field;
}
public HbComputationTargetSpecification getComputationTarget(final ComputationTargetSpecification spec) {
return getTransactionTemplateRetrying(getMaxRetries()).execute(new TransactionCallback<HbComputationTargetSpecification>() {
@Override
public HbComputationTargetSpecification doInTransaction(final TransactionStatus status) {
return getComputationTargetIntransaction(spec);
}
});
}
protected HbComputationTargetSpecification getComputationTargetIntransaction(final ComputationTargetSpecification spec) {
return getHibernateTemplate().execute(new HibernateCallback<HbComputationTargetSpecification>() {
@Override
public HbComputationTargetSpecification doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("ComputationTargetSpecification.one.byTypeAndUid");
query.setString("uidScheme", spec.getUniqueId().getScheme());
query.setString("uidValue", spec.getUniqueId().getValue());
query.setString("uidVersion", spec.getUniqueId().getVersion());
query.setParameter("type", spec.getType());
return (HbComputationTargetSpecification) query.uniqueResult();
}
});
}
protected HbComputationTargetSpecification getOrCreateComputationTargetInTransaction(final ComputationTargetSpecification spec) {
HbComputationTargetSpecification hbComputationTargetSpecification = getComputationTarget(spec);
if (hbComputationTargetSpecification == null) {
hbComputationTargetSpecification = new HbComputationTargetSpecification();
hbComputationTargetSpecification.setType(spec.getType());
hbComputationTargetSpecification.setUniqueId(spec.getUniqueId());
getHibernateTemplate().save(hbComputationTargetSpecification);
getHibernateTemplate().flush();
}
return hbComputationTargetSpecification;
}
protected CalculationConfiguration getCalculationConfiguration(final String name) {
CalculationConfiguration calcConfig = getHibernateTemplate().execute(new HibernateCallback<CalculationConfiguration>() {
@Override
public CalculationConfiguration doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("CalculationConfiguration.one.byName");
query.setString("name", name);
return (CalculationConfiguration) query.uniqueResult();
}
});
if (calcConfig == null) {
calcConfig = new CalculationConfiguration();
calcConfig.setName(name);
getHibernateTemplate().save(calcConfig);
getHibernateTemplate().flush();
}
return calcConfig;
}
protected RiskValueRequirement getRiskValueRequirement(final ValueProperties requirement) {
final String synthesizedForm = RiskValueRequirement.synthesize(requirement);
RiskValueRequirement riskValueRequirement = getHibernateTemplate().execute(new HibernateCallback<RiskValueRequirement>() {
@Override
public RiskValueRequirement doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("RiskValueRequirement.one.bySynthesizedForm");
query.setString("requirement", synthesizedForm);
return (RiskValueRequirement) query.uniqueResult();
}
});
if (riskValueRequirement == null) {
riskValueRequirement = new RiskValueRequirement(requirement);
getHibernateTemplate().save(riskValueRequirement);
getHibernateTemplate().flush();
}
return riskValueRequirement;
}
protected RiskValueSpecification getRiskValueSpecification(final ValueProperties specification) {
final String synthesizedForm = RiskValueSpecification.synthesize(specification);
RiskValueSpecification riskValueSpecification = getHibernateTemplate().execute(new HibernateCallback<RiskValueSpecification>() {
@Override
public RiskValueSpecification doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("RiskValueSpecification.one.bySynthesizedForm");
query.setString("specification", synthesizedForm);
return (RiskValueSpecification) query.uniqueResult();
}
});
if (riskValueSpecification == null) {
riskValueSpecification = new RiskValueSpecification(specification);
getHibernateTemplate().save(riskValueSpecification);
getHibernateTemplate().flush();
}
return riskValueSpecification;
}
protected FunctionUniqueId getFunctionUniqueIdInTransaction(final String uniqueId) {
FunctionUniqueId functionUniqueId = getHibernateTemplate().execute(new HibernateCallback<FunctionUniqueId>() {
@Override
public FunctionUniqueId doInHibernate(Session session) throws HibernateException, SQLException {
Query query = session.getNamedQuery("FunctionUniqueId.one.byUniqueId");
query.setString("uniqueId", uniqueId);
return (FunctionUniqueId) query.uniqueResult();
}
});
if (functionUniqueId == null) {
functionUniqueId = new FunctionUniqueId();
functionUniqueId.setUniqueId(uniqueId);
getHibernateTemplate().save(functionUniqueId);
getHibernateTemplate().flush();
}
return functionUniqueId;
}
//-------------------------------------------------------------------------
protected Instant restartRunInTransaction(RiskRun riskRun) {
Instant now = now();
riskRun.setStartInstant(now);
riskRun.setNumRestarts(riskRun.getNumRestarts() + 1);
riskRun.setComplete(false);
getHibernateTemplate().update(riskRun);
getHibernateTemplate().flush();
deleteRiskFailuresInTransaction(riskRun);
return riskRun.getCreateInstant();
}
/**
* Creates a, empty {@code ConcurrentMap} instance. With default ArrayList value
*
* @return a new, empty {@code ConcurrentMap}
*/
@SuppressWarnings("deprecation")
private static <K, V> Map<K, Collection<V>> newHashMapWithDefaultCollection() {
return (new MapMaker()).makeComputingMap(new Function<K, Collection<V>>() {
@Override
public Collection<V> apply(K input) {
return newArrayList();
}
});
}
protected void populateRiskValueRequirements(ViewCycleMetadata cycleMetadata) {
populateRiskValueSpecifications(cycleMetadata);
Map<Map<String, Object>, Collection<ValueRequirement>> data = newHashMapWithDefaultCollection();
for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
Map<ValueSpecification, Set<ValueRequirement>> outputs = cycleMetadata.getTerminalOutputs(configName);
for (ValueSpecification specification : outputs.keySet()) {
Long specificationId = _riskValueSpecifications.get(specification);
for (ValueRequirement requirement : outputs.get(specification)) {
Map<String, Object> attribs = newHashMap();
attribs.put("synthetic_form", RiskValueSpecification.synthesize(requirement.getConstraints()));
attribs.put("specification_id", specificationId);
data.get(attribs).add(requirement);
}
}
}
_riskValueRequirements.putAll(populate(data, getElSqlBundle().getSql("SelectRiskValueRequirement"), getElSqlBundle().getSql("InsertRiskValueRequirement"), RSK_SEQUENCE_NAME));
}
protected void populateRiskValueSpecifications(ViewCycleMetadata cycleMetadata) {
Map<Map<String, Object>, Collection<ValueSpecification>> data = newHashMapWithDefaultCollection();
for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
for (ValueSpecification specification : cycleMetadata.getTerminalOutputs(configName).keySet()) {
Map<String, Object> attribs = newHashMap();
attribs.put("synthetic_form", RiskValueSpecification.synthesize(specification.getProperties()));
data.get(attribs).add(specification);
}
}
_riskValueSpecifications.putAll(populate(data, getElSqlBundle().getSql("SelectRiskValueSpecification"), getElSqlBundle().getSql("InsertRiskValueSpecification"), RSK_SEQUENCE_NAME));
}
protected void populateComputationTargets(Collection<ComputationTargetSpecification> computationTargetSpecifications) {
Map<Map<String, Object>, Collection<ComputationTargetSpecification>> computationTargetsData = newHashMapWithDefaultCollection();
for (ComputationTargetSpecification targetSpecification : computationTargetSpecifications) {
Map<String, Object> attribs = newHashMap();
String idScheme;
String idValue;
String idVersion;
if (ComputationTargetSpecification.NULL.equals(targetSpecification)) {
idScheme = null;
idValue = null;
idVersion = null;
} else {
UniqueId uniqueId = targetSpecification.getUniqueId();
idScheme = uniqueId.getScheme();
idValue = uniqueId.getValue();
idVersion = uniqueId.getVersion();
}
attribs.put("id_scheme", idScheme);
attribs.put("id_value", idValue);
attribs.put("id_version", idVersion);
attribs.put("type", targetSpecification.getType().toString());
computationTargetsData.get(attribs).add(targetSpecification);
}
//------------------------------
String selectComputationTargetSpecificationSql = getElSqlBundle().getSql("SelectComputationTargetSpecification");
String selectComputationTargetSpecificationWithNullVersionSql = getElSqlBundle().getSql("SelectComputationTargetSpecificationWithNullVersion");
String insertComputationTargetSpecificationSql = getElSqlBundle().getSql("InsertComputationTargetSpecification");
final List<DbMapSqlParameterSource> insertArgsList = new ArrayList<DbMapSqlParameterSource>();
Map<ComputationTargetSpecification, Long> cache = newHashMap();
for (Map.Entry<Map<String, Object>, Collection<ComputationTargetSpecification>> attribsToObjects : computationTargetsData.entrySet()) {
Map<String, Object> attribs = attribsToObjects.getKey();
String selectSql;
if (attribs.get("id_version") == null) {
selectSql = selectComputationTargetSpecificationWithNullVersionSql;
} else {
selectSql = selectComputationTargetSpecificationSql;
}
final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
for (String attribName : attribs.keySet()) {
selectArgs.addValue(attribName, attribs.get(attribName));
}
List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
if (results.isEmpty()) {
// select avoids creating unecessary id, but id may still not be used
final long id = nextId(RSK_SEQUENCE_NAME);
final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource().addValue("id", id);
for (String attribName : attribs.keySet()) {
insertArgs.addValue(attribName, attribs.get(attribName));
}
insertArgsList.add(insertArgs);
//
for (ComputationTargetSpecification obj : attribsToObjects.getValue()) {
cache.put(obj, id);
}
} else {
Map<String, Object> result = results.get(0);
for (ComputationTargetSpecification obj : attribsToObjects.getValue()) {
cache.put(obj, (Long) result.get("ID"));
}
}
}
getJdbcTemplate().batchUpdate(insertComputationTargetSpecificationSql, insertArgsList.toArray(new DbMapSqlParameterSource[insertArgsList.size()]));
//------------------------------
_computationTargets.putAll(cache);
}
protected void populateCalculationConfigurations(Long riskRunId, ViewCycleMetadata cycleMetadata) {
Map<Map<String, Object>, Collection<String>> data = newHashMapWithDefaultCollection();
for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
Map<String, Object> map = newHashMap();
map.put("name", configName);
map.put("run_id", riskRunId);
data.get(map).add(configName);
}
_calculationConfigurations.putAll(populate(data, getElSqlBundle().getSql("SelectConfigName"), getElSqlBundle().getSql("InsertConfigName"), RSK_SEQUENCE_NAME));
}
protected <T> Map<T, Long> populate(Map<Map<String, Object>, Collection<T>> data, String selectSql, String insertSql, String pkSequenceName) {
final List<DbMapSqlParameterSource> insertArgsList = new ArrayList<DbMapSqlParameterSource>();
Map<T, Long> cache = newHashMap();
for (Map.Entry<Map<String, Object>, Collection<T>> attribsToObjects : data.entrySet()) {
Map<String, Object> attribs = attribsToObjects.getKey();
final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
for (String attribName : attribs.keySet()) {
selectArgs.addValue(attribName, attribs.get(attribName));
}
List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
if (results.isEmpty()) {
// select avoids creating unecessary id, but id may still not be used
final long id = nextId(pkSequenceName);
final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource().addValue("id", id);
for (String attribName : attribs.keySet()) {
insertArgs.addValue(attribName, attribs.get(attribName));
}
insertArgsList.add(insertArgs);
//
for (T obj : attribsToObjects.getValue()) {
cache.put(obj, id);
}
} else {
Map<String, Object> result = results.get(0);
for (T obj : attribsToObjects.getValue()) {
cache.put(obj, (Long) result.get("ID"));
}
}
}
getJdbcTemplate().batchUpdate(insertSql, insertArgsList.toArray(new DbMapSqlParameterSource[insertArgsList.size()]));
return cache;
}
protected Map<String, Object> getAttributes(Map<String, Object> attribs, String selectSql) {
final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
for (String paramName : attribs.keySet()) {
selectArgs.addValue(paramName, attribs.get(paramName));
}
List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
if (results.isEmpty()) {
return null;
} else if (results.size() == 1) {
return results.get(0);
} else {
throw new IllegalArgumentException("The query: \n" + selectSql + " \nshould be constructed so it returns at most one record.");
}
}
protected Long getId(Map<String, Object> attribs, String selectSql) {
Map<String, Object> attributes = getAttributes(attribs, selectSql);
if (attribs != null) {
return (Long) attributes.get("ID");
} else {
return null;
}
}
public static Class<?>[] getHibernateMappingClasses() {
return new HibernateBatchDbFiles().getHibernateMappingFiles();
}
public void deleteSnapshotInTransaction(ObjectId batchSnapshotId) {
final Long id = extractOid(batchSnapshotId);
MapSqlParameterSource parameters = new MapSqlParameterSource()
.addValue("snapshot_id", id);
getJdbcTemplate().update(getElSqlBundle().getSql("DeleteDataSnapshotEntries"), parameters);
getJdbcTemplate().update(getElSqlBundle().getSql("DeleteDataSnapshot"), parameters);
}
public void endBatchInTransaction(ObjectId batchUniqueId) {
ArgumentChecker.notNull(batchUniqueId, "uniqueId");
s_logger.info("Ending batch {}", batchUniqueId);
RiskRun run = getRiskRun(batchUniqueId);
//
_statusCacheByRunId.remove(run.getId());
_computeFailureCacheByRunId.remove(run.getId());
_riskRunsByIds.remove(run.getId());
//
Instant now = now();
run.setEndInstant(now);
run.setComplete(true);
getHibernateTemplate().update(run);
}
protected void deleteRiskValuesInTransaction(RiskRun riskRun) {
MapSqlParameterSource parameters = new MapSqlParameterSource()
.addValue("run_id", riskRun.getId());
getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskValues"), parameters);
}
protected void deleteRiskFailuresInTransaction(RiskRun riskRun) {
MapSqlParameterSource parameters = new MapSqlParameterSource()
.addValue("run_id", riskRun.getId());
getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskFailureReason"), parameters);
getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskFailure"), parameters);
}
protected void deleteRunInTransaction(RiskRun run) {
s_logger.info("Deleting run {}", run);
deleteRiskValuesInTransaction(run);
deleteRiskFailuresInTransaction(run);
getHibernateTemplate().deleteAll(run.getProperties());
getHibernateTemplate().deleteAll(run.getCalculationConfigurations());
getHibernateTemplate().delete(run);
getHibernateTemplate().flush();
}
public void deleteBatchInTransaction(ObjectId batchUniqueId) {
s_logger.info("Deleting batch {}", batchUniqueId);
RiskRun run = getRiskRun(batchUniqueId);
deleteRunInTransaction(run);
}
protected RiskRun findRiskRunInDbInTransaction(
final Instant valuationTime,
final VersionCorrection versionCorrection,
final UniqueId viewDefinitionUid,
final UniqueId marketDataBaseUid) {
return getHibernateTemplate().execute(new HibernateCallback<RiskRun>() {
@Override
public RiskRun doInHibernate(Session session) throws HibernateException, SQLException {
Criteria criteria = session.createCriteria(RiskRun.class);
criteria.add(Restrictions.eq("valuationTime", valuationTime));
criteria.add(Restrictions.eq("versionCorrection", versionCorrection));
criteria.add(Restrictions.eq("viewDefinitionUidScheme", viewDefinitionUid.getScheme()));
criteria.add(Restrictions.eq("viewDefinitionUidValue", viewDefinitionUid.getValue()));
criteria.add(eqOrIsNull("viewDefinitionUidVersion", viewDefinitionUid.getVersion()));
criteria.createCriteria("marketData")
.add(Restrictions.eq("baseUidScheme", marketDataBaseUid.getScheme()))
.add(Restrictions.eq("baseUidValue", marketDataBaseUid.getValue()))
.add(eqOrIsNull("baseUidVersion", marketDataBaseUid.getVersion()));
return (RiskRun) criteria.uniqueResult();
}
});
}
protected RiskRun findRiskRunInDb(
final Instant valuationTime,
final VersionCorrection versionCorrection,
final UniqueId viewDefinitionUid,
final UniqueId marketDataBaseUid) {
return getTransactionTemplateRetrying(getMaxRetries()).execute(new TransactionCallback<RiskRun>() {
@Override
public RiskRun doInTransaction(final TransactionStatus status) {
return findRiskRunInDbInTransaction(valuationTime, versionCorrection, viewDefinitionUid, marketDataBaseUid);
}
});
}
private RiskRun findRiskRunInDb(ObjectId uniqueId) {
ArgumentChecker.notNull(uniqueId, "uniqueId");
final Long id = extractOid(uniqueId);
return getRiskRunById(id);
}
protected RiskRun getRiskRun(ObjectId batchId) {
RiskRun run = findRiskRunInDb(batchId);
if (run == null) {
throw new DataNotFoundException("Cannot find run in database for " + batchId);
} else {
return run;
}
}
protected RiskRun createRiskRunInTransaction(
final UniqueId viewDefinitionUid,
final UniqueId baseMarketDataSnapshotUid,
final VersionCorrection versionCorrection,
final Instant valuationTime,
final Map<String, String> batchParameters,
final SnapshotMode snapshotMode) {
Instant now = Instant.now();
MarketData values = createOrGetMarketDataInTransaction(baseMarketDataSnapshotUid);
RiskRun riskRun = new RiskRun();
riskRun.setMarketData(values);
riskRun.setVersionCorrection(versionCorrection);
riskRun.setViewDefinitionUid(viewDefinitionUid);
riskRun.setValuationTime(valuationTime);
riskRun.setCreateInstant(now);
riskRun.setStartInstant(now);
riskRun.setNumRestarts(0);
riskRun.setComplete(false);
riskRun.setSnapshotMode(snapshotMode);
for (Map.Entry<String, String> parameter : batchParameters.entrySet()) {
riskRun.addProperty(parameter.getKey(), parameter.getValue());
}
getHibernateTemplate().save(riskRun);
getHibernateTemplate().saveOrUpdateAll(riskRun.getProperties());
getHibernateTemplate().flush();
return riskRun;
}
public synchronized RiskRun startBatchInTransaction(ViewCycleMetadata cycleMetadata, Map<String, String> batchParameters, RunCreationMode runCreationMode, SnapshotMode snapshotMode) {
s_logger.info("Starting batch ... {}", cycleMetadata);
RiskRun run;
switch (runCreationMode) {
case AUTO:
run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(),
cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());
if (run != null) {
// also check parameter equality
Map<String, String> existingProperties = run.getPropertiesMap();
if (!existingProperties.equals(batchParameters)) {
Set<Map.Entry<String, String>> symmetricDiff = Sets.symmetricDifference(existingProperties.entrySet(), batchParameters.entrySet());
throw new IllegalStateException("Run parameters stored in DB differ from new parameters with respect to: " + symmetricDiff);
}
}
if (run == null) {
run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
} else {
restartRunInTransaction(run);
}
break;
case CREATE_NEW_OVERWRITE:
run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(), cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());
if (run != null) {
deleteRunInTransaction(run);
}
run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
break;
case CREATE_NEW:
run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
break;
case REUSE_EXISTING:
run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(), cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());
if (run == null) {
throw new IllegalStateException("Cannot find run in database for " + cycleMetadata);
}
restartRunInTransaction(run);
break;
default:
throw new RuntimeException("Unexpected run creation mode " + runCreationMode);
}
populateCalculationConfigurations(run.getId(), cycleMetadata);
populateRiskValueRequirements(cycleMetadata);
Collection<ComputationTargetSpecification> computationTargets = newArrayList();
for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
for (com.opengamma.engine.ComputationTargetSpecification computationTarget : cycleMetadata.getComputationTargets(configName)) {
computationTargets.add(computationTarget);
}
}
populateComputationTargets(computationTargets);
_statusCacheByRunId.put(run.getId(), new ConcurrentHashMap<Pair<Long, Long>, StatusEntry>());
_computeFailureCacheByRunId.put(run.getId(), new ConcurrentHashMap<ComputeFailureKey, ComputeFailure>());
_riskRunsByIds.put(run.getId(), run);
return run;
}
public MarketData createOrGetMarketDataInTransaction(final UniqueId baseUid) {
s_logger.info("Creating Market Data {} ", baseUid);
MarketData marketData = getHibernateTemplate().execute(new HibernateCallback<MarketData>() {
@Override
public MarketData doInHibernate(Session session) throws HibernateException, SQLException {
final DetachedCriteria criteria = DetachedCriteria.forClass(MarketData.class);
criteria.add(Restrictions.eq("baseUidScheme", baseUid.getScheme()))
.add(Restrictions.eq("baseUidValue", baseUid.getValue()))
.add(eqOrIsNull("baseUidVersion", baseUid.getVersion()));
@SuppressWarnings("unchecked")
List<MarketData> datas = getHibernateTemplate().findByCriteria(criteria, 0, 1);
if (datas.size() > 0) {
return datas.get(0);
} else {
return null;
}
}
});
if (marketData != null) {
s_logger.info("Snapshot " + baseUid + " already exists. No need to create.");
} else {
marketData = new MarketData();
marketData.setBaseUid(baseUid);
getHibernateTemplate().save(marketData);
getHibernateTemplate().flush();
}
return marketData;
}
@SuppressWarnings("unchecked")
public void addComputedValuesToMarketDataInTransaction(ObjectId marketDataId, Set<ComputedValue> values) {
if (values == null || values.isEmpty()) {
return;
}
Set<MarketDataValue> marketDataValues = newHashSet();
for (ComputedValue value : values) {
if (value.getValue() != null && SKIP_MARKET_DATA_WRITE_TYPES.contains(value.getValue().getClass())) {
s_logger.debug("Skipping market data value " + value + " because market data already persisted in snapshot");
continue;
}
final ResultConverter<Object> resultConverter;
try {
resultConverter = (ResultConverter<Object>) _resultConverterCache.getConverter(value.getValue());
} catch (IllegalArgumentException e) {
s_logger.error("No converter for market data value of type " + value.getValue().getClass() + " for " + value.getSpecification());
continue;
}
Map<String, Double> valueAsDoublesMap = resultConverter.convert(value.getSpecification().getValueName(), value.getValue());
for (Map.Entry<String, Double> valueEntry : valueAsDoublesMap.entrySet()) {
final String doubleValueName = valueEntry.getKey();
final Double doubleValue = ensureDatabasePrecision(valueEntry.getValue());
marketDataValues.add(new MarketDataValue(value.getSpecification().getTargetSpecification(), doubleValue, doubleValueName));
}
}
addValuesToMarketDataInTransaction(marketDataId, marketDataValues);
}
public void addValuesToMarketDataInTransaction(ObjectId marketDataId, Set<MarketDataValue> values) {
if (values == null || values.isEmpty()) {
return;
}
s_logger.info("Adding {} market data values to {}", values.size(), marketDataId);
MarketData marketData = getMarketDataInTransaction(marketDataId);
if (marketData == null) {
throw new IllegalArgumentException("Market data " + marketDataId + " cannot be found");
}
List<DbMapSqlParameterSource> marketDataValuesInserts = newArrayList();
Collection<ComputationTargetSpecification> computationTargetSpecifications = newHashSet();
for (MarketDataValue value : values) {
ComputationTargetSpecification targetSpecification = value.getComputationTargetSpecification();
computationTargetSpecifications.add(targetSpecification);
}
populateComputationTargets(computationTargetSpecifications);
Collection<Long> ids = newArrayList();
for (MarketDataValue value : values) {
final ComputationTargetSpecification targetSpec = value.getComputationTargetSpecification();
final long id = nextId(RSK_SEQUENCE_NAME);
ids.add(id);
final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource()
.addValue("id", id)
.addValue("snapshot_id", marketData.getId())
.addValue("computation_target_id", _computationTargets.get(targetSpec))
.addValue("name", value.getName())
.addValue("value", value.getValue());
marketDataValuesInserts.add(insertArgs);
}
getJdbcTemplate().batchUpdate(
getElSqlBundle().getSql("InsertMarketDataValue"),
marketDataValuesInserts.toArray(new DbMapSqlParameterSource[marketDataValuesInserts.size()])
);
String sqlUpdate = getElSqlBundle().getSql("CopyMarketDataValue").replace("INSERTION_IDS", StringUtils.join(ids, ", "));
getJdbcTemplate().getJdbcOperations().update(sqlUpdate);
String sqlDelete = "DELETE FROM rsk_live_data_snapshot_entry_insertion WHERE id in (INSERTION_IDS)".replace("INSERTION_IDS", StringUtils.join(ids, ", "));
getJdbcTemplate().getJdbcOperations().update(sqlDelete);
}
//-------------------------------------------------------------------------
@SuppressWarnings("unchecked")
public synchronized void addJobResultsInTransaction(TransactionStatus transactionStatus, ObjectId runId, ViewComputationResultModel resultModel) {
ArgumentChecker.notNull(runId, "runId");
ArgumentChecker.notNull(resultModel, "resultModel");
final long riskRunId = extractOid(runId);
ArgumentChecker.notNull(riskRunId, "riskRunId");
Map<ComputeFailureKey, ComputeFailure> computeFailureCache = _computeFailureCacheByRunId.get(riskRunId);
Map<Pair<Long, Long>, StatusEntry> statusCache = _statusCacheByRunId.get(riskRunId);
Map<ValueSpecification, BatchResultWriterFailure> errorCache = populateErrorCache(computeFailureCache, resultModel.getAllResults());
RiskRun run = _riskRunsByIds.get(riskRunId);
if (run.getSnapshotMode().equals(SnapshotMode.WRITE_THROUGH)) {
addComputedValuesToMarketDataInTransaction(run.getMarketData().getObjectId(), resultModel.getAllMarketData());
}
for (String calcConfigName : resultModel.getCalculationConfigurationNames()) {
ViewCalculationResultModel viewCalculationResultModel = resultModel.getCalculationResult(calcConfigName);
final Set<ComputationTargetSpecification> successfulTargets = newHashSet();
final Set<ComputationTargetSpecification> failedTargets = newHashSet();
List<SqlParameterSource> targetProperties = newArrayList();
List<SqlParameterSource> successes = newArrayList();
List<SqlParameterSource> failures = newArrayList();
List<SqlParameterSource> failureReasons = newArrayList();
Instant evalInstant = Instant.now();
long calcConfId = _calculationConfigurations.get(calcConfigName);
for (final ComputationTargetSpecification targetSpec : viewCalculationResultModel.getAllTargets()) {
final long computationTargetId = _computationTargets.get(targetSpec);
boolean specFailures = false;
for (final ComputedValueResult computedValue : viewCalculationResultModel.getAllValues(targetSpec)) {
ResultConverter<Object> resultConverter = null;
if (!(computedValue.getValue() instanceof MissingValue)) {
try {
resultConverter = (ResultConverter<Object>) _resultConverterCache.getConverter(computedValue.getValue());
} catch (IllegalArgumentException e) {
s_logger.info("No converter for value of type " + computedValue.getValue().getClass() + " for " + computedValue.getSpecification());
}
}
final ValueSpecification specification = computedValue.getSpecification();
if (!_riskValueSpecifications.containsKey(specification)) {
s_logger.error("Unexpected result specification " + specification + ". Result cannot be written. Result value was " + computedValue.getValue());
continue;
}
final long valueSpecificationId = _riskValueSpecifications.get(specification);
final long functionUniqueId = getFunctionUniqueIdInTransaction(specification.getFunctionUniqueId()).getId();
final long computeNodeId = getOrCreateComputeNode(computedValue.getComputeNodeId()).getId();
if (resultConverter != null && computedValue.getInvocationResult() == InvocationResult.SUCCESS) {
s_logger.debug("Writing value {} for value spec {}", computedValue.getValue(), specification);
Map<String, Double> valueAsDoublesMap = resultConverter.convert(computedValue.getSpecification().getValueName(), computedValue.getValue());
for (Map.Entry<String, Double> valueEntry : valueAsDoublesMap.entrySet()) {
final String valueName = valueEntry.getKey();
final Double doubleValue = ensureDatabasePrecision(valueEntry.getValue());
final long successId = nextId(RSK_SEQUENCE_NAME);
successes.add(getSuccessArgs(successId, riskRunId, evalInstant, calcConfId, computationTargetId, valueSpecificationId, functionUniqueId, computeNodeId, valueName, doubleValue));
}
} else {
s_logger.info("Writing failure for {} with invocation result {}, {} ",
newArray(computedValue.getSpecification(), computedValue.getInvocationResult(), computedValue.getAggregatedExecutionLog()));
specFailures = true;
final long failureId = nextId(RSK_SEQUENCE_NAME);
failures.add(getFailureArgs(failureId, riskRunId, evalInstant, calcConfId, computationTargetId, valueSpecificationId, functionUniqueId, computeNodeId, specification.getValueName()));
BatchResultWriterFailure cachedFailure = errorCache.get(specification);
if (cachedFailure != null) {
for (long computeFailureId : cachedFailure.getComputeFailureIds()) {
ArgumentChecker.notNull(computeFailureId, "computeFailureId");
final long failureReasonId = nextId(RSK_SEQUENCE_NAME);
failureReasons.add(getFailureReasonArgs(failureReasonId, failureId, computeFailureId));
}
}
}
}
StatusEntry.Status status = getStatus(statusCache, calcConfigName, targetSpec);
if (specFailures || status == StatusEntry.Status.FAILURE) {
successfulTargets.remove(targetSpec);
failedTargets.add(targetSpec);
} else {
successfulTargets.add(targetSpec);
}
// storing target data
ComputationTarget computationTarget = _computationTargetResolver.resolve(targetSpec, VersionCorrection.LATEST);
Object targetValue = computationTarget.getValue();
if (targetValue instanceof Bean) {
Bean bean = (Bean) targetValue;
for (String propertyName : bean.propertyNames()) {
Property<Object> property = bean.property(propertyName);
final long targetPropertyId = nextId(RSK_SEQUENCE_NAME);
targetProperties.add(getTargetPropertyArgs(targetPropertyId, computationTargetId, propertyName, property.get() == null ? "NULL" : property.get().toString()));
}
}
}
if (successes.isEmpty()
&& failures.isEmpty()
&& failureReasons.isEmpty()
&& successfulTargets.isEmpty()
&& failedTargets.isEmpty()) {
s_logger.debug("Nothing to write to DB for {}", resultModel);
return;
}
Object preSuccessSavepoint = transactionStatus.createSavepoint();
try {
getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskSuccess"), successes.toArray(new DbMapSqlParameterSource[successes.size()]));
} catch (Exception e) {
s_logger.error("Failed to write successful calculations to batch database. Converting to failures.", e);
transactionStatus.rollbackToSavepoint(preSuccessSavepoint);
if (!successes.isEmpty()) {
String exceptionClass = e.getClass().getName();
String exceptionMsg = e.getMessage();
final StringBuilder buffer = new StringBuilder();
for (StackTraceElement element : e.getStackTrace()) {
buffer.append(element.toString()).append("\n");
}
final String stackTrace = buffer.toString();
for (SqlParameterSource success : successes) {
failures.add(convertSuccessToFailure(success));
final long failureId = getId(success);
final long functionId = getFunctionId(success);
ComputeFailureKey computeFailureKey = new ComputeFailureKey(String.valueOf(functionId), exceptionClass, exceptionMsg, stackTrace);
ComputeFailure computeFailure = getComputeFailureFromDb(computeFailureCache, computeFailureKey);
final long failureReasonId = nextId(RSK_SEQUENCE_NAME);
failureReasons.add(getFailureReasonArgs(failureReasonId, failureId, computeFailure.getId()));
}
failedTargets.addAll(successfulTargets);
successes.clear();
successfulTargets.clear();
targetProperties.clear();
}
}
Object preTargetPropertiesFailureSavepoint = transactionStatus.createSavepoint();
try {
getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertTargetProperties"), targetProperties.toArray(new DbMapSqlParameterSource[targetProperties.size()]));
} catch (Exception e) {
s_logger.error("Failed to write target properties to batch database", e);
transactionStatus.rollbackToSavepoint(preTargetPropertiesFailureSavepoint);
}
Object preFailureSavepoint = transactionStatus.createSavepoint();
try {
getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskFailure"), failures.toArray(new DbMapSqlParameterSource[failures.size()]));
getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskFailureReason"), failureReasons.toArray(new DbMapSqlParameterSource[failureReasons.size()]));
} catch (Exception e) {
s_logger.error("Failed to write failures to batch database", e);
transactionStatus.rollbackToSavepoint(preFailureSavepoint);
}
updateStatusEntries(riskRunId, statusCache, calcConfigName, StatusEntry.Status.SUCCESS, successfulTargets);
updateStatusEntries(riskRunId, statusCache, calcConfigName, StatusEntry.Status.FAILURE, failedTargets);
}
}
private long getFunctionId(SqlParameterSource args) {
return (Long) args.getValue("function_unique_id");
}
private long getId(SqlParameterSource args) {
return (Long) args.getValue("id");
}
private DbMapSqlParameterSource getFailureReasonArgs(long failureReasonId, long failureId, long computeFailureId) {
final DbMapSqlParameterSource args = new DbMapSqlParameterSource();
args.addValue("id", failureReasonId);
args.addValue("rsk_failure_id", failureId);
args.addValue("compute_failure_id", computeFailureId);
return args;
}
private SqlParameterSource getSuccessArgs(long successId, long riskRunId, Instant evalInstant, long calcConfId,
long computationTargetId, long valueSpecificationId, long functionUniqueId, long computeNodeId, String valueName, Double doubleValue) {
DbMapSqlParameterSource args = new DbMapSqlParameterSource();
args.addValue("id", successId);
args.addValue("calculation_configuration_id", calcConfId);
args.addValue("name", valueName);
args.addValue("value_specification_id", valueSpecificationId);
args.addValue("function_unique_id", functionUniqueId);
args.addValue("computation_target_id", computationTargetId);
args.addValue("run_id", riskRunId);
args.addValue("value", doubleValue);
args.addTimestamp("eval_instant", evalInstant);
args.addValue("compute_node_id", computeNodeId);
return args;
}
private SqlParameterSource getTargetPropertyArgs(long targetPropertyId, long computationTargetId, String propertyKey, String propertyValue) {
DbMapSqlParameterSource args = new DbMapSqlParameterSource();
args.addValue("id", targetPropertyId);
args.addValue("target_id", computationTargetId);
args.addValue("property_key", propertyKey);
args.addValue("property_value", propertyValue);
return args;
}
private SqlParameterSource getFailureArgs(long failureId, long riskRunId, Instant evalInstant, long calcConfId,
long computationTargetId, long valueSpecificationId, long functionUniqueId, long computeNodeId, String valueName) {
DbMapSqlParameterSource args = new DbMapSqlParameterSource();
args.addValue("id", failureId);
args.addValue("calculation_configuration_id", calcConfId);
args.addValue("name", valueName);
args.addValue("value_specification_id", valueSpecificationId);
args.addValue("function_unique_id", functionUniqueId);
args.addValue("computation_target_id", computationTargetId);
args.addValue("run_id", riskRunId);
args.addTimestamp("eval_instant", evalInstant);
args.addValue("compute_node_id", computeNodeId);
return args;
}
private SqlParameterSource convertSuccessToFailure(SqlParameterSource successArgs) {
return getFailureArgs(
(Long) successArgs.getValue("id"),
(Long) successArgs.getValue("run_id"),
DbDateUtils.fromSqlTimestamp((Timestamp) successArgs.getValue("eval_instant")),
(Long) successArgs.getValue("calculation_configuration_id"),
(Long) successArgs.getValue("computation_target_id"),
(Long) successArgs.getValue("value_specification_id"),
(Long) getFunctionId(successArgs),
(Long) successArgs.getValue("compute_node_id"),
(String) successArgs.getValue("name"));
}
/**
* STAGE 1. Populate error information in the cache.
* This is done for all items and will populate table rsk_compute_failure.
*
* @param computeFailureCache the cache
* @param results the results
* @return the error cache, not null
*/
protected Map<ValueSpecification, BatchResultWriterFailure> populateErrorCache(
Map<ComputeFailureKey, ComputeFailure> computeFailureCache, Collection<ViewResultEntry> results) {
Map<ValueSpecification, BatchResultWriterFailure> errorCache = Maps.newHashMap();
for (ViewResultEntry result : results) {
populateErrorCache(computeFailureCache, errorCache, result);
}
return errorCache;
}
@SuppressWarnings("unchecked")
protected void updateStatusEntries(long runId,
Map<Pair<Long, Long>, StatusEntry> statusCache,
String calcConfName,
StatusEntry.Status status,
Collection<ComputationTargetSpecification> targets) {
Long calcConfId = _calculationConfigurations.get(calcConfName);
List<DbMapSqlParameterSource> inserts = newArrayList();
List<DbMapSqlParameterSource> updates = newArrayList();
for (ComputationTargetSpecification target : targets) {
Long computationTargetId = _computationTargets.get(target);
DbMapSqlParameterSource params = new DbMapSqlParameterSource();
// this assumes that _searchKey2StatusEntry has already been populated
// in getStatus()
Pair<Long, Long> key = Pair.of(calcConfId, computationTargetId);
StatusEntry statusEntry = statusCache.get(key);
if (statusEntry != null) {
statusEntry.setStatus(status);
params.addValue("id", statusEntry.getId());
params.addValue("run_id", runId);
params.addValue("status", statusEntry.getStatus().ordinal());
updates.add(params);
} else {
final long statusId = nextId(RSK_SEQUENCE_NAME);
final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource();
insertArgs.addValue("ID", statusId);
statusEntry = new StatusEntry();
statusEntry.setId(statusId);
statusEntry.setRunId(runId);
statusEntry.setStatus(status);
statusEntry.setCalculationConfigurationId(calcConfId);
statusEntry.setComputationTargetId(computationTargetId);
statusCache.put(key, statusEntry);
params.addValue("id", statusId);
params.addValue("run_id", runId);
params.addValue("calculation_configuration_id", calcConfId);
params.addValue("computation_target_id", computationTargetId);
params.addValue("status", statusEntry.getStatus().ordinal());
inserts.add(params);
}
}
s_logger.info("Inserting {} and updating {} {} status entries", (Object[]) newArray(inserts.size(), updates.size(), status));
SqlParameterSource[] batchArgsArray = inserts.toArray(new DbMapSqlParameterSource[inserts.size()]);
int[] counts = getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertFromRunStatus"), batchArgsArray);
checkCount(status + " insert", batchArgsArray, counts);
batchArgsArray = updates.toArray(new DbMapSqlParameterSource[updates.size()]);
counts = getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("UpdateFromRunStatus"), batchArgsArray);
checkCount(status + " update", batchArgsArray, counts);
s_logger.info("Inserted {} and updated {} {} status entries", (Object[]) newArray(inserts.size(), updates.size(), status));
}
private int checkCount(String rowType, SqlParameterSource[] batchArgsArray, int[] counts) {
int totalCount = 0;
for (int count : counts) {
totalCount += count;
}
if (totalCount != batchArgsArray.length) {
throw new RuntimeException(rowType + " insert count is wrong: expected = " +
batchArgsArray.length + " actual = " + totalCount);
}
return totalCount;
}
private static Double ensureDatabasePrecision(Double value) {
if (value == null) {
return null;
}
// Java's smallest double is 4.9e-324, but most databases would underflow.
// Postgres is 1e-307, Oracle is 2.2e-307, SQL Server is 2.2e-308.
if (Math.abs(value) < 1e-300) {
return 0d;
}
return value;
}
protected StatusEntry.Status getStatus(Map<Pair<Long, Long>, StatusEntry> statusCache, String calcConfName, ComputationTargetSpecification ct) {
Long calcConfId = _calculationConfigurations.get(calcConfName);
Long computationTargetId = _computationTargets.get(ct);
// first check to see if this status has already been queried for
// and if the answer could therefore be found in the cache
Pair<Long, Long> key = Pair.of(calcConfId, computationTargetId);
if (statusCache.containsKey(key)) {
StatusEntry existingStatusEntryInDb = statusCache.get(key);
if (existingStatusEntryInDb != null) {
// status entry in db.
return existingStatusEntryInDb.getStatus();
} else {
// no status entry in db.
return StatusEntry.Status.NOT_RUNNING;
}
}
MapSqlParameterSource args = new MapSqlParameterSource();
args.addValue("calculation_configuration_id", calcConfId);
args.addValue("computation_target_id", computationTargetId);
try {
StatusEntry statusEntry = getJdbcTemplate().queryForObject(
getElSqlBundle().getSql("SelectStatusEntry"),
args,
DbBatchUtils.ROW_MAPPER);
// status entry in db found.
statusCache.put(key, statusEntry);
return statusEntry.getStatus();
} catch (IncorrectResultSizeDataAccessException e) {
// no status entry in the db.
statusCache.remove(key);
return StatusEntry.Status.NOT_RUNNING;
}
}
protected void populateErrorCache(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, Map<ValueSpecification, BatchResultWriterFailure> errorCache, ViewResultEntry item) {
BatchResultWriterFailure cachedFailure = new BatchResultWriterFailure();
switch (item.getComputedValue().getInvocationResult()) {
case FUNCTION_THREW_EXCEPTION:
// an "original" failure
//
// There will only be 1 failure reason.
ComputeFailure computeFailure = getComputeFailureFromDb(computeFailureCache, item);
cachedFailure.addComputeFailureId(computeFailure.getId());
break;
case MISSING_INPUTS:
// There may be 1-N failure reasons - one for each failed
// function in the subtree below this node. (This
// only includes "original", i.e., lowest-level, failures.)
for (ValueSpecification missingInput : item.getComputedValue().getMissingInputs()) {
BatchResultWriterFailure inputFailure = errorCache.get(missingInput);
if (inputFailure == null) {
ComputeFailureKey computeFailureKey = new ComputeFailureKey(
missingInput.getFunctionUniqueId(),
"N/A",
"Missing input " + missingInput,
"N/A");
computeFailure = getComputeFailureFromDb(computeFailureCache, computeFailureKey);
cachedFailure.addComputeFailureId(computeFailure.getId());
} else {
cachedFailure.addComputeFailureIds(inputFailure.getComputeFailureIds());
}
}
break;
}
if (!cachedFailure.getComputeFailureIds().isEmpty()) {
errorCache.put(item.getComputedValue().getSpecification(), cachedFailure);
}
}
/*package*/ ComputeFailure getComputeFailureFromDb(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ViewResultEntry item) {
if (item.getComputedValue().getInvocationResult() != InvocationResult.FUNCTION_THREW_EXCEPTION) {
throw new IllegalArgumentException("Please give a failed item");
}
ExecutionLogWithContext rootLog = item.getComputedValue().getAggregatedExecutionLog().getRootLog();
String exceptionClass = rootLog != null ? rootLog.getExecutionLog().getExceptionClass() : null;
String exceptionMessage = rootLog != null ? rootLog.getExecutionLog().getExceptionMessage() : null;
String exceptionStackTrace = rootLog != null ? rootLog.getExecutionLog().getExceptionStackTrace() : null;
//ensure we don't end up with null going into the ComputeFailureKey for these strings.
//this will probably be due to the fact that the rootLog was null.
exceptionClass = defaultString(exceptionClass, "No logging information available");
exceptionMessage = defaultString(exceptionMessage, "No logging information available");
exceptionStackTrace = defaultString(exceptionStackTrace, "No logging information available");
ComputeFailureKey computeFailureKey = new ComputeFailureKey(item.getComputedValue().getSpecification().getFunctionUniqueId(), exceptionClass, exceptionMessage, exceptionStackTrace);
return getComputeFailureFromDb(computeFailureCache, computeFailureKey);
}
public ComputeFailure getComputeFailureFromDb(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ComputeFailureKey computeFailureKey) {
ComputeFailure computeFailure = computeFailureCache.get(computeFailureKey);
if (computeFailure != null) {
return computeFailure;
}
try {
int id = getJdbcTemplate().queryForObject(getElSqlBundle().getSql("SelectComputeFailureId"), DbBatchUtils.toSqlParameterSource(computeFailureKey), Integer.class);
computeFailure = new ComputeFailure();
computeFailure.setId(id);
computeFailure.setFunctionId(computeFailureKey.getFunctionId());
computeFailure.setExceptionClass(computeFailureKey.getExceptionClass());
computeFailure.setExceptionMsg(computeFailureKey.getExceptionMsg());
computeFailure.setStackTrace(computeFailureKey.getStackTrace());
computeFailureCache.put(computeFailureKey, computeFailure);
return computeFailure;
} catch (IncorrectResultSizeDataAccessException e) {
// Not seen a failure like this before - create and write to database
try {
computeFailure = saveComputeFailure(computeFailureCache, computeFailureKey);
return computeFailure;
} catch (DataAccessException e2) {
s_logger.error("Failed to save compute failure", e2);
throw new RuntimeException("Failed to save compute failure", e2);
}
}
}
public ComputeFailure saveComputeFailure(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ComputeFailureKey computeFailureKey) {
ComputeFailure computeFailure;
computeFailure = new ComputeFailure();
final long computeFailureId = nextId(RSK_SEQUENCE_NAME);
computeFailure.setId(computeFailureId);
computeFailure.setFunctionId(computeFailureKey.getFunctionId());
computeFailure.setExceptionClass(computeFailureKey.getExceptionClass());
computeFailure.setExceptionMsg(computeFailureKey.getExceptionMsg());
computeFailure.setStackTrace(computeFailureKey.getStackTrace());
int rowCount = getJdbcTemplate().update(getElSqlBundle().getSql("InsertComputeFailure"), DbBatchUtils.toSqlParameterSource(computeFailure));
if (rowCount == 1) {
computeFailureCache.put(computeFailureKey, computeFailure);
return computeFailure;
}
return computeFailure;
}
/**
* Instances of this class are saved in the computation cache for each
* failure (whether the failure is 'original' or due to missing inputs).
* The set of Longs is a set of compute failure IDs (referencing
* rsk_compute_failure(id)). The set is built bottom up.
* For example, if A has two children, B and C, and B has failed
* due to error 12, and C has failed due to errors 15 and 16, then
* A has failed due to errors 12, 15, and 16.
*/
protected static class BatchResultWriterFailure implements MissingValue, Serializable {
/** Serialization version. */
private static final long serialVersionUID = 1L;
private Set<Long> _computeFailureIds = new HashSet<Long>();
public Set<Long> getComputeFailureIds() {
return Collections.unmodifiableSet(_computeFailureIds);
}
public void setComputeFailureIds(Set<Long> computeFailureIds) {
_computeFailureIds = computeFailureIds;
}
public void addComputeFailureId(Long computeFailureId) {
addComputeFailureIds(Collections.singleton(computeFailureId));
}
public void addComputeFailureIds(Set<? extends Long> computeFailureIds) {
_computeFailureIds.addAll(computeFailureIds);
}
}
}