Package com.opengamma.masterdb.batch

Source Code of com.opengamma.masterdb.batch.DbBatchWriter

/**
* Copyright (C) 2009 - present by OpenGamma Inc. and the OpenGamma group of companies
*
* Please see distribution for license.
*/
package com.opengamma.masterdb.batch;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newConcurrentMap;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static com.opengamma.lambdava.streams.Lambdava.newArray;
import static com.opengamma.util.db.HibernateDbUtils.eqOrIsNull;
import static org.apache.commons.lang.StringUtils.defaultString;

import java.io.Serializable;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.lang.StringUtils;
import org.hibernate.Criteria;
import org.hibernate.HibernateException;
import org.hibernate.Query;
import org.hibernate.Session;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Restrictions;
import org.joda.beans.Bean;
import org.joda.beans.Property;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import org.springframework.orm.hibernate3.HibernateCallback;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.threeten.bp.Instant;

import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.opengamma.DataNotFoundException;
import com.opengamma.batch.RunCreationMode;
import com.opengamma.batch.SnapshotMode;
import com.opengamma.batch.domain.CalculationConfiguration;
import com.opengamma.batch.domain.ComputeFailure;
import com.opengamma.batch.domain.ComputeFailureKey;
import com.opengamma.batch.domain.ComputeHost;
import com.opengamma.batch.domain.ComputeNode;
import com.opengamma.batch.domain.FunctionUniqueId;
import com.opengamma.batch.domain.HbComputationTargetSpecification;
import com.opengamma.batch.domain.LiveDataField;
import com.opengamma.batch.domain.MarketData;
import com.opengamma.batch.domain.MarketDataValue;
import com.opengamma.batch.domain.RiskRun;
import com.opengamma.batch.domain.RiskValueRequirement;
import com.opengamma.batch.domain.RiskValueSpecification;
import com.opengamma.batch.domain.StatusEntry;
import com.opengamma.core.marketdatasnapshot.SnapshotDataBundle;
import com.opengamma.elsql.ElSqlBundle;
import com.opengamma.engine.ComputationTarget;
import com.opengamma.engine.ComputationTargetResolver;
import com.opengamma.engine.ComputationTargetSpecification;
import com.opengamma.engine.calcnode.InvocationResult;
import com.opengamma.engine.calcnode.MissingValue;
import com.opengamma.engine.value.ComputedValue;
import com.opengamma.engine.value.ComputedValueResult;
import com.opengamma.engine.value.ValueProperties;
import com.opengamma.engine.value.ValueRequirement;
import com.opengamma.engine.value.ValueSpecification;
import com.opengamma.engine.view.ExecutionLogWithContext;
import com.opengamma.engine.view.ViewCalculationResultModel;
import com.opengamma.engine.view.ViewComputationResultModel;
import com.opengamma.engine.view.ViewResultEntry;
import com.opengamma.engine.view.cycle.ViewCycleMetadata;
import com.opengamma.financial.conversion.ResultConverter;
import com.opengamma.financial.conversion.ResultConverterCache;
import com.opengamma.id.ObjectId;
import com.opengamma.id.UniqueId;
import com.opengamma.id.VersionCorrection;
import com.opengamma.masterdb.AbstractDbMaster;
import com.opengamma.util.ArgumentChecker;
import com.opengamma.util.db.DbConnector;
import com.opengamma.util.db.DbDateUtils;
import com.opengamma.util.db.DbMapSqlParameterSource;
import com.opengamma.util.tuple.Pair;

/**
* A batch master implementation using a database for persistence.
* <p>
* This is a full implementation of the batch master using an SQL database.
* This implementation uses Hibernate to write all static data, including LiveData snapshots.
* Risk itself is written using direct JDBC.
* <p>
* Full details of the API are in {@link com.opengamma.masterdb.batch.document.BatchMaster}.
* <p>
* This class is mutable but must be treated as immutable after configuration.
*/
public class DbBatchWriter extends AbstractDbMaster {

  /**
   * The default scheme for unique identifiers.
   */
  public static final String IDENTIFIER_SCHEME_DEFAULT = "DbBat";
  /**
   * The batch risk sequence name.
   */
  public static final String RSK_SEQUENCE_NAME = "rsk_batch_seq";
  /**
   * The set of types to skip when encountered in market data because they already belong to a snapshot.
   */
  private static final Set<Class<?>> SKIP_MARKET_DATA_WRITE_TYPES = ImmutableSet.<Class<?>>of(SnapshotDataBundle.class);

  private final Map<String, Long> _calculationConfigurations = newConcurrentMap();
  private final Map<ValueRequirement, Long> _riskValueRequirements = newConcurrentMap();
  private final Map<ValueSpecification, Long> _riskValueSpecifications = newConcurrentMap();
  private final Map<ComputationTargetSpecification, Long> _computationTargets = newConcurrentMap();

  private final Map<Long, RiskRun> _riskRunsByIds = newConcurrentMap();
  private final Map<Long, Map<Pair<Long, Long>, StatusEntry>> _statusCacheByRunId = newConcurrentMap();
  private final Map<Long, Map<ComputeFailureKey, ComputeFailure>> _computeFailureCacheByRunId = newConcurrentMap();

  /** Logger. */
  private static final Logger s_logger = LoggerFactory.getLogger(DbBatchWriter.class);
  private final ComputationTargetResolver _computationTargetResolver;

  /**
   * The Result converter cache.
   */
  private ResultConverterCache _resultConverterCache;

  /**
   * Creates an instance.
   *
   * @param dbConnector  the database connector, not null
   */
  public DbBatchWriter(final DbConnector dbConnector, final ComputationTargetResolver computationTargetResolver) {
    super(dbConnector, IDENTIFIER_SCHEME_DEFAULT);
    _resultConverterCache = new ResultConverterCache();
    _computationTargetResolver = computationTargetResolver;
    setElSqlBundle(ElSqlBundle.of(dbConnector.getDialect().getElSqlConfig(), DbBatchWriter.class));
  }

  public RiskRun getRiskRunById(final Long id) {
    return getHibernateTemplate().execute(new HibernateCallback<RiskRun>() {
      @Override
      public RiskRun doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("RiskRun.one.byId");
        query.setLong("id", id);
        return (RiskRun) query.uniqueResult();
      }
    });
  }

  protected ComputeHost getOrCreateComputeHost(final String hostName) {
    ComputeHost computeHost = getHibernateTemplate().execute(new HibernateCallback<ComputeHost>() {
      @Override
      public ComputeHost doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("ComputeHost.one.byHostName");
        query.setString("hostName", hostName);
        return (ComputeHost) query.uniqueResult();
      }
    });
    if (computeHost == null) {
      computeHost = new ComputeHost();
      computeHost.setHostName(hostName);
      getHibernateTemplate().save(computeHost);
      getHibernateTemplate().flush();
    }
    return computeHost;
  }

  protected ComputeNode getOrCreateComputeNode(final String nodeId) {
    ArgumentChecker.notNull(nodeId, "nodeId");
    String hostName = nodeId;
    int slashIndex = nodeId.indexOf('/'); // e.g., mymachine-t5500/0/1, see LocalCalculationNode.java. Should refactor nodeId to a class with two strings, host and node id
    if (slashIndex != -1) {
      hostName = nodeId.substring(0, slashIndex);
    }
    final ComputeHost host = getOrCreateComputeHost(hostName);

    ComputeNode node = getHibernateTemplate().execute(new HibernateCallback<ComputeNode>() {
      @Override
      public ComputeNode doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("ComputeNode.one.byNodeName");
        query.setString("nodeName", nodeId);
        return (ComputeNode) query.uniqueResult();
      }
    });
    if (node == null) {
      node = new ComputeNode();
      node.setComputeHost(host);
      node.setNodeName(nodeId);
      getHibernateTemplate().save(node);
      getHibernateTemplate().flush();
    }
    return node;
  }

  protected MarketData getMarketDataInTransaction(final ObjectId snapshotId) {

    MarketData liveDataValues = getHibernateTemplate().execute(new HibernateCallback<MarketData>() {
      @Override
      public MarketData doInHibernate(Session session) throws HibernateException, SQLException {
        Long id = extractOid(snapshotId);
        return (MarketData) session.get(MarketData.class, id);
      }
    });

    if (liveDataValues == null) {
      throw new IllegalArgumentException("Snapshot for " + snapshotId + " cannot be found");
    }
    return liveDataValues;
  }


  protected LiveDataField getLiveDataField(final String fieldName) {
    LiveDataField field = getHibernateTemplate().execute(new HibernateCallback<LiveDataField>() {
      @Override
      public LiveDataField doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("LiveDataField.one.byName");
        query.setString("name", fieldName);
        return (LiveDataField) query.uniqueResult();
      }
    });
    if (field == null) {
      field = new LiveDataField();
      field.setName(fieldName);
      getHibernateTemplate().save(field);
      getHibernateTemplate().flush();
    }
    return field;
  }

  public HbComputationTargetSpecification getComputationTarget(final ComputationTargetSpecification spec) {
    return getTransactionTemplateRetrying(getMaxRetries()).execute(new TransactionCallback<HbComputationTargetSpecification>() {
      @Override
      public HbComputationTargetSpecification doInTransaction(final TransactionStatus status) {
        return getComputationTargetIntransaction(spec);
      }
    });
  }

  protected HbComputationTargetSpecification getComputationTargetIntransaction(final ComputationTargetSpecification spec) {
    return getHibernateTemplate().execute(new HibernateCallback<HbComputationTargetSpecification>() {
      @Override
      public HbComputationTargetSpecification doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("ComputationTargetSpecification.one.byTypeAndUid");

        query.setString("uidScheme", spec.getUniqueId().getScheme());
        query.setString("uidValue", spec.getUniqueId().getValue());
        query.setString("uidVersion", spec.getUniqueId().getVersion());

        query.setParameter("type", spec.getType());

        return (HbComputationTargetSpecification) query.uniqueResult();
      }
    });
  }

  protected HbComputationTargetSpecification getOrCreateComputationTargetInTransaction(final ComputationTargetSpecification spec) {
    HbComputationTargetSpecification hbComputationTargetSpecification = getComputationTarget(spec);
    if (hbComputationTargetSpecification == null) {
      hbComputationTargetSpecification = new HbComputationTargetSpecification();
      hbComputationTargetSpecification.setType(spec.getType());
      hbComputationTargetSpecification.setUniqueId(spec.getUniqueId());
      getHibernateTemplate().save(hbComputationTargetSpecification);
      getHibernateTemplate().flush();
    }
    return hbComputationTargetSpecification;
  }

  protected CalculationConfiguration getCalculationConfiguration(final String name) {
    CalculationConfiguration calcConfig = getHibernateTemplate().execute(new HibernateCallback<CalculationConfiguration>() {
      @Override
      public CalculationConfiguration doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("CalculationConfiguration.one.byName");
        query.setString("name", name);
        return (CalculationConfiguration) query.uniqueResult();
      }
    });
    if (calcConfig == null) {
      calcConfig = new CalculationConfiguration();
      calcConfig.setName(name);
      getHibernateTemplate().save(calcConfig);
      getHibernateTemplate().flush();
    }
    return calcConfig;
  }

  protected RiskValueRequirement getRiskValueRequirement(final ValueProperties requirement) {
    final String synthesizedForm = RiskValueRequirement.synthesize(requirement);
    RiskValueRequirement riskValueRequirement = getHibernateTemplate().execute(new HibernateCallback<RiskValueRequirement>() {
      @Override
      public RiskValueRequirement doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("RiskValueRequirement.one.bySynthesizedForm");
        query.setString("requirement", synthesizedForm);
        return (RiskValueRequirement) query.uniqueResult();
      }
    });
    if (riskValueRequirement == null) {
      riskValueRequirement = new RiskValueRequirement(requirement);
      getHibernateTemplate().save(riskValueRequirement);
      getHibernateTemplate().flush();
    }
    return riskValueRequirement;
  }

  protected RiskValueSpecification getRiskValueSpecification(final ValueProperties specification) {
    final String synthesizedForm = RiskValueSpecification.synthesize(specification);
    RiskValueSpecification riskValueSpecification = getHibernateTemplate().execute(new HibernateCallback<RiskValueSpecification>() {
      @Override
      public RiskValueSpecification doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("RiskValueSpecification.one.bySynthesizedForm");
        query.setString("specification", synthesizedForm);
        return (RiskValueSpecification) query.uniqueResult();
      }
    });
    if (riskValueSpecification == null) {
      riskValueSpecification = new RiskValueSpecification(specification);
      getHibernateTemplate().save(riskValueSpecification);
      getHibernateTemplate().flush();
    }
    return riskValueSpecification;
  }

  protected FunctionUniqueId getFunctionUniqueIdInTransaction(final String uniqueId) {
    FunctionUniqueId functionUniqueId = getHibernateTemplate().execute(new HibernateCallback<FunctionUniqueId>() {
      @Override
      public FunctionUniqueId doInHibernate(Session session) throws HibernateException, SQLException {
        Query query = session.getNamedQuery("FunctionUniqueId.one.byUniqueId");
        query.setString("uniqueId", uniqueId);
        return (FunctionUniqueId) query.uniqueResult();
      }
    });
    if (functionUniqueId == null) {
      functionUniqueId = new FunctionUniqueId();
      functionUniqueId.setUniqueId(uniqueId);
      getHibernateTemplate().save(functionUniqueId);
      getHibernateTemplate().flush();
    }
    return functionUniqueId;
  }

  //-------------------------------------------------------------------------


  protected Instant restartRunInTransaction(RiskRun riskRun) {
    Instant now = now();
    riskRun.setStartInstant(now);
    riskRun.setNumRestarts(riskRun.getNumRestarts() + 1);
    riskRun.setComplete(false);

    getHibernateTemplate().update(riskRun);
    getHibernateTemplate().flush();
    deleteRiskFailuresInTransaction(riskRun);

    return riskRun.getCreateInstant();
  }

  /**
   * Creates a, empty {@code ConcurrentMap} instance. With default ArrayList value
   *
   * @return a new, empty {@code ConcurrentMap}
   */
  @SuppressWarnings("deprecation")
  private static <K, V> Map<K, Collection<V>> newHashMapWithDefaultCollection() {
    return (new MapMaker()).makeComputingMap(new Function<K, Collection<V>>() {
      @Override
      public Collection<V> apply(K input) {
        return newArrayList();
      }
    });
  }

  protected void populateRiskValueRequirements(ViewCycleMetadata cycleMetadata) {
    populateRiskValueSpecifications(cycleMetadata);
   
    Map<Map<String, Object>, Collection<ValueRequirement>> data = newHashMapWithDefaultCollection();
    for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
      Map<ValueSpecification, Set<ValueRequirement>> outputs = cycleMetadata.getTerminalOutputs(configName);
      for (ValueSpecification specification : outputs.keySet()) {
        Long specificationId = _riskValueSpecifications.get(specification);
        for (ValueRequirement requirement : outputs.get(specification)) {
          Map<String, Object> attribs = newHashMap();
          attribs.put("synthetic_form", RiskValueSpecification.synthesize(requirement.getConstraints()));
          attribs.put("specification_id", specificationId);
          data.get(attribs).add(requirement);
        }
      }
    }
    _riskValueRequirements.putAll(populate(data, getElSqlBundle().getSql("SelectRiskValueRequirement"), getElSqlBundle().getSql("InsertRiskValueRequirement"), RSK_SEQUENCE_NAME));
  }

  protected void populateRiskValueSpecifications(ViewCycleMetadata cycleMetadata) {
    Map<Map<String, Object>, Collection<ValueSpecification>> data = newHashMapWithDefaultCollection();
    for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
      for (ValueSpecification specification : cycleMetadata.getTerminalOutputs(configName).keySet()) {
        Map<String, Object> attribs = newHashMap();
        attribs.put("synthetic_form", RiskValueSpecification.synthesize(specification.getProperties()));
        data.get(attribs).add(specification);
      }
    }
    _riskValueSpecifications.putAll(populate(data, getElSqlBundle().getSql("SelectRiskValueSpecification"), getElSqlBundle().getSql("InsertRiskValueSpecification"), RSK_SEQUENCE_NAME));
  }

  protected void populateComputationTargets(Collection<ComputationTargetSpecification> computationTargetSpecifications) {
    Map<Map<String, Object>, Collection<ComputationTargetSpecification>> computationTargetsData = newHashMapWithDefaultCollection();
    for (ComputationTargetSpecification targetSpecification : computationTargetSpecifications) {
      Map<String, Object> attribs = newHashMap();
     
      String idScheme;
      String idValue;
      String idVersion;
      if (ComputationTargetSpecification.NULL.equals(targetSpecification)) {
        idScheme = null;
        idValue = null;
        idVersion = null;
      } else {
        UniqueId uniqueId = targetSpecification.getUniqueId();
        idScheme = uniqueId.getScheme();
        idValue = uniqueId.getValue();
        idVersion = uniqueId.getVersion();
      }
      attribs.put("id_scheme", idScheme);
      attribs.put("id_value", idValue);
      attribs.put("id_version", idVersion);
      attribs.put("type", targetSpecification.getType().toString());
      computationTargetsData.get(attribs).add(targetSpecification);
    }

    //------------------------------

    String selectComputationTargetSpecificationSql = getElSqlBundle().getSql("SelectComputationTargetSpecification");
    String selectComputationTargetSpecificationWithNullVersionSql = getElSqlBundle().getSql("SelectComputationTargetSpecificationWithNullVersion");
    String insertComputationTargetSpecificationSql = getElSqlBundle().getSql("InsertComputationTargetSpecification");


    final List<DbMapSqlParameterSource> insertArgsList = new ArrayList<DbMapSqlParameterSource>();

    Map<ComputationTargetSpecification, Long> cache = newHashMap();

    for (Map.Entry<Map<String, Object>, Collection<ComputationTargetSpecification>> attribsToObjects : computationTargetsData.entrySet()) {
      Map<String, Object> attribs = attribsToObjects.getKey();

      String selectSql;
      if (attribs.get("id_version") == null) {
        selectSql = selectComputationTargetSpecificationWithNullVersionSql;
      } else {
        selectSql = selectComputationTargetSpecificationSql;
      }

      final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
      for (String attribName : attribs.keySet()) {
        selectArgs.addValue(attribName, attribs.get(attribName));
      }
      List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
      if (results.isEmpty()) {
        // select avoids creating unecessary id, but id may still not be used
        final long id = nextId(RSK_SEQUENCE_NAME);
        final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource().addValue("id", id);
        for (String attribName : attribs.keySet()) {
          insertArgs.addValue(attribName, attribs.get(attribName));
        }
        insertArgsList.add(insertArgs);
        //
        for (ComputationTargetSpecification obj : attribsToObjects.getValue()) {
          cache.put(obj, id);
        }
      } else {
        Map<String, Object> result = results.get(0);
        for (ComputationTargetSpecification obj : attribsToObjects.getValue()) {
          cache.put(obj, (Long) result.get("ID"));
        }
      }
    }
    getJdbcTemplate().batchUpdate(insertComputationTargetSpecificationSql, insertArgsList.toArray(new DbMapSqlParameterSource[insertArgsList.size()]));

    //------------------------------

    _computationTargets.putAll(cache);
  }

  protected void populateCalculationConfigurations(Long riskRunId, ViewCycleMetadata cycleMetadata) {
    Map<Map<String, Object>, Collection<String>> data = newHashMapWithDefaultCollection();
    for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
      Map<String, Object> map = newHashMap();
      map.put("name", configName);
      map.put("run_id", riskRunId);
      data.get(map).add(configName);
    }
    _calculationConfigurations.putAll(populate(data, getElSqlBundle().getSql("SelectConfigName"), getElSqlBundle().getSql("InsertConfigName"), RSK_SEQUENCE_NAME));
  }

  protected <T> Map<T, Long> populate(Map<Map<String, Object>, Collection<T>> data, String selectSql, String insertSql, String pkSequenceName) {
    final List<DbMapSqlParameterSource> insertArgsList = new ArrayList<DbMapSqlParameterSource>();

    Map<T, Long> cache = newHashMap();
    for (Map.Entry<Map<String, Object>, Collection<T>> attribsToObjects : data.entrySet()) {
      Map<String, Object> attribs = attribsToObjects.getKey();
      final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
      for (String attribName : attribs.keySet()) {
        selectArgs.addValue(attribName, attribs.get(attribName));
      }
      List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
      if (results.isEmpty()) {
        // select avoids creating unecessary id, but id may still not be used
        final long id = nextId(pkSequenceName);
        final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource().addValue("id", id);
        for (String attribName : attribs.keySet()) {
          insertArgs.addValue(attribName, attribs.get(attribName));
        }
        insertArgsList.add(insertArgs);
        //
        for (T obj : attribsToObjects.getValue()) {
          cache.put(obj, id);
        }
      } else {
        Map<String, Object> result = results.get(0);
        for (T obj : attribsToObjects.getValue()) {
          cache.put(obj, (Long) result.get("ID"));
        }
      }
    }
    getJdbcTemplate().batchUpdate(insertSql, insertArgsList.toArray(new DbMapSqlParameterSource[insertArgsList.size()]));

    return cache;
  }

  protected Map<String, Object> getAttributes(Map<String, Object> attribs, String selectSql) {
    final DbMapSqlParameterSource selectArgs = new DbMapSqlParameterSource();
    for (String paramName : attribs.keySet()) {
      selectArgs.addValue(paramName, attribs.get(paramName));
    }
    List<Map<String, Object>> results = getJdbcTemplate().queryForList(selectSql, selectArgs);
    if (results.isEmpty()) {
      return null;
    } else if (results.size() == 1) {
      return results.get(0);
    } else {
      throw new IllegalArgumentException("The query: \n" + selectSql + " \nshould be constructed so it returns at most one record.");
    }
  }

  protected Long getId(Map<String, Object> attribs, String selectSql) {
    Map<String, Object> attributes = getAttributes(attribs, selectSql);
    if (attribs != null) {
      return (Long) attributes.get("ID");
    } else {
      return null;
    }
  }

  public static Class<?>[] getHibernateMappingClasses() {
    return new HibernateBatchDbFiles().getHibernateMappingFiles();
  }

  public void deleteSnapshotInTransaction(ObjectId batchSnapshotId) {
    final Long id = extractOid(batchSnapshotId);
    MapSqlParameterSource parameters = new MapSqlParameterSource()
      .addValue("snapshot_id", id);
    getJdbcTemplate().update(getElSqlBundle().getSql("DeleteDataSnapshotEntries"), parameters);
    getJdbcTemplate().update(getElSqlBundle().getSql("DeleteDataSnapshot"), parameters);
  }

  public void endBatchInTransaction(ObjectId batchUniqueId) {
    ArgumentChecker.notNull(batchUniqueId, "uniqueId");
    s_logger.info("Ending batch {}", batchUniqueId);
    RiskRun run = getRiskRun(batchUniqueId);
    //
    _statusCacheByRunId.remove(run.getId());
    _computeFailureCacheByRunId.remove(run.getId());
    _riskRunsByIds.remove(run.getId());
    //
    Instant now = now();
    run.setEndInstant(now);
    run.setComplete(true);
    getHibernateTemplate().update(run);
  }

  protected void deleteRiskValuesInTransaction(RiskRun riskRun) {
    MapSqlParameterSource parameters = new MapSqlParameterSource()
      .addValue("run_id", riskRun.getId());
    getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskValues"), parameters);
  }

  protected void deleteRiskFailuresInTransaction(RiskRun riskRun) {
    MapSqlParameterSource parameters = new MapSqlParameterSource()
      .addValue("run_id", riskRun.getId());
    getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskFailureReason"), parameters);
    getJdbcTemplate().update(getElSqlBundle().getSql("DeleteRiskFailure"), parameters);
  }

  protected void deleteRunInTransaction(RiskRun run) {
    s_logger.info("Deleting run {}", run);
    deleteRiskValuesInTransaction(run);
    deleteRiskFailuresInTransaction(run);
    getHibernateTemplate().deleteAll(run.getProperties());
    getHibernateTemplate().deleteAll(run.getCalculationConfigurations());
    getHibernateTemplate().delete(run);
    getHibernateTemplate().flush();
  }

  public void deleteBatchInTransaction(ObjectId batchUniqueId) {
    s_logger.info("Deleting batch {}", batchUniqueId);
    RiskRun run = getRiskRun(batchUniqueId);
    deleteRunInTransaction(run);
  }

  protected RiskRun findRiskRunInDbInTransaction(
    final Instant valuationTime,
    final VersionCorrection versionCorrection,
    final UniqueId viewDefinitionUid,
    final UniqueId marketDataBaseUid) {
    return getHibernateTemplate().execute(new HibernateCallback<RiskRun>() {
      @Override
      public RiskRun doInHibernate(Session session) throws HibernateException, SQLException {
        Criteria criteria = session.createCriteria(RiskRun.class);
        criteria.add(Restrictions.eq("valuationTime", valuationTime));
        criteria.add(Restrictions.eq("versionCorrection", versionCorrection));
        criteria.add(Restrictions.eq("viewDefinitionUidScheme", viewDefinitionUid.getScheme()));
        criteria.add(Restrictions.eq("viewDefinitionUidValue", viewDefinitionUid.getValue()));
        criteria.add(eqOrIsNull("viewDefinitionUidVersion", viewDefinitionUid.getVersion()));

        criteria.createCriteria("marketData")
          .add(Restrictions.eq("baseUidScheme", marketDataBaseUid.getScheme()))
          .add(Restrictions.eq("baseUidValue", marketDataBaseUid.getValue()))
          .add(eqOrIsNull("baseUidVersion", marketDataBaseUid.getVersion()));

        return (RiskRun) criteria.uniqueResult();
      }
    });
  }

  protected RiskRun findRiskRunInDb(
    final Instant valuationTime,
    final VersionCorrection versionCorrection,
    final UniqueId viewDefinitionUid,
    final UniqueId marketDataBaseUid) {
    return getTransactionTemplateRetrying(getMaxRetries()).execute(new TransactionCallback<RiskRun>() {
      @Override
      public RiskRun doInTransaction(final TransactionStatus status) {
        return findRiskRunInDbInTransaction(valuationTime, versionCorrection, viewDefinitionUid, marketDataBaseUid);
      }
    });
  }

  private RiskRun findRiskRunInDb(ObjectId uniqueId) {
    ArgumentChecker.notNull(uniqueId, "uniqueId");
    final Long id = extractOid(uniqueId);
    return getRiskRunById(id);
  }

  protected RiskRun getRiskRun(ObjectId batchId) {
    RiskRun run = findRiskRunInDb(batchId);
    if (run == null) {
      throw new DataNotFoundException("Cannot find run in database for " + batchId);
    } else {
      return run;
    }
  }

  protected RiskRun createRiskRunInTransaction(
    final UniqueId viewDefinitionUid,
    final UniqueId baseMarketDataSnapshotUid,
    final VersionCorrection versionCorrection,
    final Instant valuationTime,
    final Map<String, String> batchParameters,
    final SnapshotMode snapshotMode) {

    Instant now = Instant.now();

    MarketData values = createOrGetMarketDataInTransaction(baseMarketDataSnapshotUid);

    RiskRun riskRun = new RiskRun();
    riskRun.setMarketData(values);
    riskRun.setVersionCorrection(versionCorrection);
    riskRun.setViewDefinitionUid(viewDefinitionUid);
    riskRun.setValuationTime(valuationTime);
    riskRun.setCreateInstant(now);
    riskRun.setStartInstant(now);
    riskRun.setNumRestarts(0);
    riskRun.setComplete(false);
    riskRun.setSnapshotMode(snapshotMode);

    for (Map.Entry<String, String> parameter : batchParameters.entrySet()) {
      riskRun.addProperty(parameter.getKey(), parameter.getValue());
    }

    getHibernateTemplate().save(riskRun);
    getHibernateTemplate().saveOrUpdateAll(riskRun.getProperties());
    getHibernateTemplate().flush();

    return riskRun;
  }

  public synchronized RiskRun startBatchInTransaction(ViewCycleMetadata cycleMetadata, Map<String, String> batchParameters, RunCreationMode runCreationMode, SnapshotMode snapshotMode) {
    s_logger.info("Starting batch ... {}", cycleMetadata);

    RiskRun run;
    switch (runCreationMode) {
      case AUTO:
        run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(),
            cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());

        if (run != null) {
          // also check parameter equality
          Map<String, String> existingProperties = run.getPropertiesMap();

          if (!existingProperties.equals(batchParameters)) {
            Set<Map.Entry<String, String>> symmetricDiff = Sets.symmetricDifference(existingProperties.entrySet(), batchParameters.entrySet());
            throw new IllegalStateException("Run parameters stored in DB differ from new parameters with respect to: " + symmetricDiff);
          }
        }

        if (run == null) {
          run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
              cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
        } else {
          restartRunInTransaction(run);
        }
        break;

      case CREATE_NEW_OVERWRITE:
        run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(), cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());
        if (run != null) {
          deleteRunInTransaction(run);
        }

        run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
            cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
        break;

      case CREATE_NEW:
        run = createRiskRunInTransaction(cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId(),
            cycleMetadata.getVersionCorrection(), cycleMetadata.getValuationTime(), batchParameters, snapshotMode);
        break;

      case REUSE_EXISTING:
        run = findRiskRunInDb(cycleMetadata.getValuationTime(), cycleMetadata.getVersionCorrection(), cycleMetadata.getViewDefinitionId(), cycleMetadata.getMarketDataSnapshotId());
        if (run == null) {
          throw new IllegalStateException("Cannot find run in database for " + cycleMetadata);
        }
        restartRunInTransaction(run);
        break;

      default:
        throw new RuntimeException("Unexpected run creation mode " + runCreationMode);
    }

    populateCalculationConfigurations(run.getId(), cycleMetadata);
    populateRiskValueRequirements(cycleMetadata);

    Collection<ComputationTargetSpecification> computationTargets = newArrayList();
    for (final String configName : cycleMetadata.getAllCalculationConfigurationNames()) {
      for (com.opengamma.engine.ComputationTargetSpecification computationTarget : cycleMetadata.getComputationTargets(configName)) {
        computationTargets.add(computationTarget);
      }
    }
    populateComputationTargets(computationTargets);

    _statusCacheByRunId.put(run.getId(), new ConcurrentHashMap<Pair<Long, Long>, StatusEntry>());
    _computeFailureCacheByRunId.put(run.getId(), new ConcurrentHashMap<ComputeFailureKey, ComputeFailure>());

    _riskRunsByIds.put(run.getId(), run);

    return run;
  }

  public MarketData createOrGetMarketDataInTransaction(final UniqueId baseUid) {
    s_logger.info("Creating Market Data {} ", baseUid);
    MarketData marketData = getHibernateTemplate().execute(new HibernateCallback<MarketData>() {
      @Override
      public MarketData doInHibernate(Session session) throws HibernateException, SQLException {

        final DetachedCriteria criteria = DetachedCriteria.forClass(MarketData.class);
        criteria.add(Restrictions.eq("baseUidScheme", baseUid.getScheme()))
          .add(Restrictions.eq("baseUidValue", baseUid.getValue()))
          .add(eqOrIsNull("baseUidVersion", baseUid.getVersion()));

        @SuppressWarnings("unchecked")
        List<MarketData> datas = getHibernateTemplate().findByCriteria(criteria, 0, 1);
        if (datas.size() > 0) {
          return datas.get(0);
        } else {
          return null;
        }
      }
    });
    if (marketData != null) {
      s_logger.info("Snapshot " + baseUid + " already exists. No need to create.");
    } else {
      marketData = new MarketData();
      marketData.setBaseUid(baseUid);
      getHibernateTemplate().save(marketData);
      getHibernateTemplate().flush();
    }
    return marketData;
  }
 
  @SuppressWarnings("unchecked")
  public void addComputedValuesToMarketDataInTransaction(ObjectId marketDataId, Set<ComputedValue> values) {
    if (values == null || values.isEmpty()) {
      return;
    }
    Set<MarketDataValue> marketDataValues = newHashSet();
    for (ComputedValue value : values) {
      if (value.getValue() != null && SKIP_MARKET_DATA_WRITE_TYPES.contains(value.getValue().getClass())) {
        s_logger.debug("Skipping market data value " + value + " because market data already persisted in snapshot");
        continue;
      }
      final ResultConverter<Object> resultConverter;
      try {
        resultConverter = (ResultConverter<Object>) _resultConverterCache.getConverter(value.getValue());
      } catch (IllegalArgumentException e) {
        s_logger.error("No converter for market data value of type " + value.getValue().getClass() + " for " + value.getSpecification());
        continue;
      }
      Map<String, Double> valueAsDoublesMap = resultConverter.convert(value.getSpecification().getValueName(), value.getValue());
      for (Map.Entry<String, Double> valueEntry : valueAsDoublesMap.entrySet()) {
        final String doubleValueName = valueEntry.getKey();
        final Double doubleValue = ensureDatabasePrecision(valueEntry.getValue());
        marketDataValues.add(new MarketDataValue(value.getSpecification().getTargetSpecification(), doubleValue, doubleValueName));
      }
    }
    addValuesToMarketDataInTransaction(marketDataId, marketDataValues);
  }

  public void addValuesToMarketDataInTransaction(ObjectId marketDataId, Set<MarketDataValue> values) {
    if (values == null || values.isEmpty()) {
      return;
    }
    s_logger.info("Adding {} market data values to {}", values.size(), marketDataId);
   
    MarketData marketData = getMarketDataInTransaction(marketDataId);
    if (marketData == null) {
      throw new IllegalArgumentException("Market data " + marketDataId + " cannot be found");
    }
   
    List<DbMapSqlParameterSource> marketDataValuesInserts = newArrayList();

    Collection<ComputationTargetSpecification> computationTargetSpecifications = newHashSet();
    for (MarketDataValue value : values) {
      ComputationTargetSpecification targetSpecification = value.getComputationTargetSpecification();
      computationTargetSpecifications.add(targetSpecification);
    }
    populateComputationTargets(computationTargetSpecifications);

    Collection<Long> ids = newArrayList();

    for (MarketDataValue value : values) {
      final ComputationTargetSpecification targetSpec = value.getComputationTargetSpecification();
      final long id = nextId(RSK_SEQUENCE_NAME);
      ids.add(id);
      final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource()
        .addValue("id", id)
        .addValue("snapshot_id", marketData.getId())
        .addValue("computation_target_id", _computationTargets.get(targetSpec))
        .addValue("name", value.getName())
        .addValue("value", value.getValue());
      marketDataValuesInserts.add(insertArgs);         
    }

    getJdbcTemplate().batchUpdate(
      getElSqlBundle().getSql("InsertMarketDataValue"),
      marketDataValuesInserts.toArray(new DbMapSqlParameterSource[marketDataValuesInserts.size()])
    );

    String sqlUpdate = getElSqlBundle().getSql("CopyMarketDataValue").replace("INSERTION_IDS", StringUtils.join(ids, ", "));
    getJdbcTemplate().getJdbcOperations().update(sqlUpdate);
   
    String sqlDelete = "DELETE FROM rsk_live_data_snapshot_entry_insertion WHERE id in (INSERTION_IDS)".replace("INSERTION_IDS", StringUtils.join(ids, ", "));
    getJdbcTemplate().getJdbcOperations().update(sqlDelete);
  }

  //-------------------------------------------------------------------------
  @SuppressWarnings("unchecked")
  public synchronized void addJobResultsInTransaction(TransactionStatus transactionStatus, ObjectId runId, ViewComputationResultModel resultModel) {
    ArgumentChecker.notNull(runId, "runId");
    ArgumentChecker.notNull(resultModel, "resultModel");
   
    final long riskRunId = extractOid(runId);
    ArgumentChecker.notNull(riskRunId, "riskRunId");

    Map<ComputeFailureKey, ComputeFailure> computeFailureCache = _computeFailureCacheByRunId.get(riskRunId);
    Map<Pair<Long, Long>, StatusEntry> statusCache = _statusCacheByRunId.get(riskRunId);

    Map<ValueSpecification, BatchResultWriterFailure> errorCache = populateErrorCache(computeFailureCache, resultModel.getAllResults());
   
    RiskRun run = _riskRunsByIds.get(riskRunId);
    if (run.getSnapshotMode().equals(SnapshotMode.WRITE_THROUGH)) {
      addComputedValuesToMarketDataInTransaction(run.getMarketData().getObjectId(), resultModel.getAllMarketData());
    }

    for (String calcConfigName : resultModel.getCalculationConfigurationNames()) {
      ViewCalculationResultModel viewCalculationResultModel = resultModel.getCalculationResult(calcConfigName);
     
      final Set<ComputationTargetSpecification> successfulTargets = newHashSet();
      final Set<ComputationTargetSpecification> failedTargets = newHashSet();

      List<SqlParameterSource> targetProperties = newArrayList();
      List<SqlParameterSource> successes = newArrayList();
      List<SqlParameterSource> failures = newArrayList();
      List<SqlParameterSource> failureReasons = newArrayList();

      Instant evalInstant = Instant.now();

      long calcConfId = _calculationConfigurations.get(calcConfigName);

      for (final ComputationTargetSpecification targetSpec : viewCalculationResultModel.getAllTargets()) {
        final long computationTargetId = _computationTargets.get(targetSpec);
        boolean specFailures = false;
        for (final ComputedValueResult computedValue : viewCalculationResultModel.getAllValues(targetSpec)) {
          ResultConverter<Object> resultConverter  = null;
          if (!(computedValue.getValue() instanceof MissingValue)) {
            try {
              resultConverter = (ResultConverter<Object>) _resultConverterCache.getConverter(computedValue.getValue());
            } catch (IllegalArgumentException e) {
              s_logger.info("No converter for value of type " + computedValue.getValue().getClass() + " for " + computedValue.getSpecification());
            }           
          }

          final ValueSpecification specification = computedValue.getSpecification();
          if (!_riskValueSpecifications.containsKey(specification)) {
            s_logger.error("Unexpected result specification " + specification + ". Result cannot be written. Result value was " + computedValue.getValue());
            continue;
          }
          final long valueSpecificationId = _riskValueSpecifications.get(specification);
          final long functionUniqueId = getFunctionUniqueIdInTransaction(specification.getFunctionUniqueId()).getId();
          final long computeNodeId = getOrCreateComputeNode(computedValue.getComputeNodeId()).getId();
         
          if (resultConverter != null && computedValue.getInvocationResult() == InvocationResult.SUCCESS) {
            s_logger.debug("Writing value {} for value spec {}", computedValue.getValue(), specification);
            Map<String, Double> valueAsDoublesMap = resultConverter.convert(computedValue.getSpecification().getValueName(), computedValue.getValue());
            for (Map.Entry<String, Double> valueEntry : valueAsDoublesMap.entrySet()) {
              final String valueName = valueEntry.getKey();
              final Double doubleValue = ensureDatabasePrecision(valueEntry.getValue());
              final long successId = nextId(RSK_SEQUENCE_NAME);
              successes.add(getSuccessArgs(successId, riskRunId, evalInstant, calcConfId, computationTargetId, valueSpecificationId, functionUniqueId, computeNodeId, valueName, doubleValue));
            }
          } else {
            s_logger.info("Writing failure for {} with invocation result {}, {} ",
                newArray(computedValue.getSpecification(), computedValue.getInvocationResult(), computedValue.getAggregatedExecutionLog()));
            specFailures = true;
           
            final long failureId = nextId(RSK_SEQUENCE_NAME);
            failures.add(getFailureArgs(failureId, riskRunId, evalInstant, calcConfId, computationTargetId, valueSpecificationId, functionUniqueId, computeNodeId, specification.getValueName()));

            BatchResultWriterFailure cachedFailure = errorCache.get(specification);
            if (cachedFailure != null) {
              for (long computeFailureId : cachedFailure.getComputeFailureIds()) {
                ArgumentChecker.notNull(computeFailureId, "computeFailureId");
                final long failureReasonId = nextId(RSK_SEQUENCE_NAME);
                failureReasons.add(getFailureReasonArgs(failureReasonId, failureId, computeFailureId));
              }
            }      
          }
        }
        StatusEntry.Status status = getStatus(statusCache, calcConfigName, targetSpec);
        if (specFailures || status == StatusEntry.Status.FAILURE) {
          successfulTargets.remove(targetSpec);
          failedTargets.add(targetSpec);
        } else {
          successfulTargets.add(targetSpec);
        }

        // storing target data
        ComputationTarget computationTarget = _computationTargetResolver.resolve(targetSpec, VersionCorrection.LATEST);
        Object targetValue = computationTarget.getValue();
        if (targetValue instanceof Bean) {
          Bean bean = (Bean) targetValue;
          for (String propertyName : bean.propertyNames()) {
            Property<Object> property = bean.property(propertyName);
            final long targetPropertyId = nextId(RSK_SEQUENCE_NAME);
            targetProperties.add(getTargetPropertyArgs(targetPropertyId, computationTargetId, propertyName, property.get() == null ? "NULL" : property.get().toString()));
          }
        }
      }

      if (successes.isEmpty()
        && failures.isEmpty()
        && failureReasons.isEmpty()
        && successfulTargets.isEmpty()
        && failedTargets.isEmpty()) {
        s_logger.debug("Nothing to write to DB for {}", resultModel);
        return;
      }

      Object preSuccessSavepoint = transactionStatus.createSavepoint();
      try {
        getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskSuccess"), successes.toArray(new DbMapSqlParameterSource[successes.size()]));
      } catch (Exception e) {
        s_logger.error("Failed to write successful calculations to batch database. Converting to failures.", e);
        transactionStatus.rollbackToSavepoint(preSuccessSavepoint);
        if (!successes.isEmpty()) {
          String exceptionClass = e.getClass().getName();
          String exceptionMsg = e.getMessage();
          final StringBuilder buffer = new StringBuilder();
          for (StackTraceElement element : e.getStackTrace()) {
            buffer.append(element.toString()).append("\n");
          }
          final String stackTrace = buffer.toString();
          for (SqlParameterSource success : successes) {
            failures.add(convertSuccessToFailure(success));
            final long failureId = getId(success);
            final long functionId = getFunctionId(success);
            ComputeFailureKey computeFailureKey = new ComputeFailureKey(String.valueOf(functionId), exceptionClass, exceptionMsg, stackTrace);
            ComputeFailure computeFailure = getComputeFailureFromDb(computeFailureCache, computeFailureKey);
            final long failureReasonId = nextId(RSK_SEQUENCE_NAME);
            failureReasons.add(getFailureReasonArgs(failureReasonId, failureId, computeFailure.getId()));
          }
          failedTargets.addAll(successfulTargets);
          successes.clear();
          successfulTargets.clear();
          targetProperties.clear();
        }
      }
      Object preTargetPropertiesFailureSavepoint = transactionStatus.createSavepoint();
      try {
        getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertTargetProperties"), targetProperties.toArray(new DbMapSqlParameterSource[targetProperties.size()]));
      } catch (Exception e) {
        s_logger.error("Failed to write target properties to batch database", e);
        transactionStatus.rollbackToSavepoint(preTargetPropertiesFailureSavepoint);
      }
      Object preFailureSavepoint = transactionStatus.createSavepoint();
      try {
        getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskFailure"), failures.toArray(new DbMapSqlParameterSource[failures.size()]));
        getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertRiskFailureReason"), failureReasons.toArray(new DbMapSqlParameterSource[failureReasons.size()]));
      } catch (Exception e) {
        s_logger.error("Failed to write failures to batch database", e);
        transactionStatus.rollbackToSavepoint(preFailureSavepoint);
      }

      updateStatusEntries(riskRunId, statusCache, calcConfigName, StatusEntry.Status.SUCCESS, successfulTargets);
      updateStatusEntries(riskRunId, statusCache, calcConfigName, StatusEntry.Status.FAILURE, failedTargets);
    }
  }

  private long getFunctionId(SqlParameterSource args) {
    return (Long) args.getValue("function_unique_id");
  }
 
  private long getId(SqlParameterSource args) {
    return (Long) args.getValue("id");
  }

  private DbMapSqlParameterSource getFailureReasonArgs(long failureReasonId, long failureId, long computeFailureId) {
    final DbMapSqlParameterSource args = new DbMapSqlParameterSource();
    args.addValue("id", failureReasonId);
    args.addValue("rsk_failure_id", failureId);
    args.addValue("compute_failure_id", computeFailureId);
    return args;
  }

  private SqlParameterSource getSuccessArgs(long successId, long riskRunId, Instant evalInstant, long calcConfId,
      long computationTargetId, long valueSpecificationId, long functionUniqueId, long computeNodeId, String valueName, Double doubleValue) {
    DbMapSqlParameterSource args = new DbMapSqlParameterSource();
    args.addValue("id", successId);
    args.addValue("calculation_configuration_id", calcConfId);
    args.addValue("name", valueName);
    args.addValue("value_specification_id", valueSpecificationId);
    args.addValue("function_unique_id", functionUniqueId);
    args.addValue("computation_target_id", computationTargetId);
    args.addValue("run_id", riskRunId);
    args.addValue("value", doubleValue);
    args.addTimestamp("eval_instant", evalInstant);
    args.addValue("compute_node_id", computeNodeId);
    return args;
  }

  private SqlParameterSource getTargetPropertyArgs(long targetPropertyId, long computationTargetId, String propertyKey, String propertyValue) {
    DbMapSqlParameterSource args = new DbMapSqlParameterSource();
    args.addValue("id", targetPropertyId);
    args.addValue("target_id", computationTargetId);
    args.addValue("property_key", propertyKey);
    args.addValue("property_value", propertyValue);
    return args;
  }

  private SqlParameterSource getFailureArgs(long failureId, long riskRunId, Instant evalInstant, long calcConfId,
      long computationTargetId, long valueSpecificationId, long functionUniqueId, long computeNodeId, String valueName) {
    DbMapSqlParameterSource args = new DbMapSqlParameterSource();
    args.addValue("id", failureId);
    args.addValue("calculation_configuration_id", calcConfId);
    args.addValue("name", valueName);
    args.addValue("value_specification_id", valueSpecificationId);
    args.addValue("function_unique_id", functionUniqueId);
    args.addValue("computation_target_id", computationTargetId);
    args.addValue("run_id", riskRunId);
    args.addTimestamp("eval_instant", evalInstant);
    args.addValue("compute_node_id", computeNodeId);
    return args;
  }
 
  private SqlParameterSource convertSuccessToFailure(SqlParameterSource successArgs) {
    return getFailureArgs(
        (Long) successArgs.getValue("id"),
        (Long) successArgs.getValue("run_id"),
        DbDateUtils.fromSqlTimestamp((Timestamp) successArgs.getValue("eval_instant")),
        (Long) successArgs.getValue("calculation_configuration_id"),
        (Long) successArgs.getValue("computation_target_id"),
        (Long) successArgs.getValue("value_specification_id"),
        (Long) getFunctionId(successArgs),
        (Long) successArgs.getValue("compute_node_id"),
        (String) successArgs.getValue("name"));
  }

  /**
   * STAGE 1. Populate error information in the cache.
   * This is done for all items and will populate table rsk_compute_failure.
   *
   * @param computeFailureCache  the cache
   * @param results  the results
   * @return the error cache, not null
   */
  protected Map<ValueSpecification, BatchResultWriterFailure> populateErrorCache(
      Map<ComputeFailureKey, ComputeFailure> computeFailureCache, Collection<ViewResultEntry> results) {
    Map<ValueSpecification, BatchResultWriterFailure> errorCache = Maps.newHashMap();
    for (ViewResultEntry result : results) {
      populateErrorCache(computeFailureCache, errorCache, result);
    }
    return errorCache;
  }

  @SuppressWarnings("unchecked")
  protected void updateStatusEntries(long runId,
                                     Map<Pair<Long, Long>, StatusEntry> statusCache,
                                     String calcConfName,
                                     StatusEntry.Status status,
                                     Collection<ComputationTargetSpecification> targets) {

    Long calcConfId = _calculationConfigurations.get(calcConfName);

    List<DbMapSqlParameterSource> inserts = newArrayList();
    List<DbMapSqlParameterSource> updates = newArrayList();

    for (ComputationTargetSpecification target : targets) {
      Long computationTargetId = _computationTargets.get(target);

      DbMapSqlParameterSource params = new DbMapSqlParameterSource();

      // this assumes that _searchKey2StatusEntry has already been populated
      // in getStatus()
      Pair<Long, Long> key = Pair.of(calcConfId, computationTargetId);
      StatusEntry statusEntry = statusCache.get(key);
      if (statusEntry != null) {
        statusEntry.setStatus(status);
        params.addValue("id", statusEntry.getId());
        params.addValue("run_id", runId);
        params.addValue("status", statusEntry.getStatus().ordinal());
        updates.add(params);
      } else {
        final long statusId = nextId(RSK_SEQUENCE_NAME);

        final DbMapSqlParameterSource insertArgs = new DbMapSqlParameterSource();
        insertArgs.addValue("ID", statusId);
        statusEntry = new StatusEntry();
        statusEntry.setId(statusId);
        statusEntry.setRunId(runId);
        statusEntry.setStatus(status);
        statusEntry.setCalculationConfigurationId(calcConfId);
        statusEntry.setComputationTargetId(computationTargetId);
        statusCache.put(key, statusEntry);

        params.addValue("id", statusId);
        params.addValue("run_id", runId);
        params.addValue("calculation_configuration_id", calcConfId);
        params.addValue("computation_target_id", computationTargetId);
        params.addValue("status", statusEntry.getStatus().ordinal());
        inserts.add(params);
      }

    }

    s_logger.info("Inserting {} and updating {} {} status entries", (Object[]) newArray(inserts.size(), updates.size(), status));

    SqlParameterSource[] batchArgsArray = inserts.toArray(new DbMapSqlParameterSource[inserts.size()]);
    int[] counts = getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("InsertFromRunStatus"), batchArgsArray);
    checkCount(status + " insert", batchArgsArray, counts);

    batchArgsArray = updates.toArray(new DbMapSqlParameterSource[updates.size()]);
    counts = getJdbcTemplate().batchUpdate(getElSqlBundle().getSql("UpdateFromRunStatus"), batchArgsArray);
    checkCount(status + " update", batchArgsArray, counts);

    s_logger.info("Inserted {} and updated {} {} status entries", (Object[]) newArray(inserts.size(), updates.size(), status));
  }

  private int checkCount(String rowType, SqlParameterSource[] batchArgsArray, int[] counts) {
    int totalCount = 0;
    for (int count : counts) {
      totalCount += count;
    }
    if (totalCount != batchArgsArray.length) {
      throw new RuntimeException(rowType + " insert count is wrong: expected = " +
        batchArgsArray.length + " actual = " + totalCount);
    }
    return totalCount;
  }
 
  private static Double ensureDatabasePrecision(Double value) {
    if (value == null) {
      return null;
    }
    // Java's smallest double is 4.9e-324, but most databases would underflow.
    // Postgres is 1e-307, Oracle is 2.2e-307, SQL Server is 2.2e-308.
    if (Math.abs(value) < 1e-300) {
      return 0d;
    }
    return value;
  }

  protected StatusEntry.Status getStatus(Map<Pair<Long, Long>, StatusEntry> statusCache, String calcConfName, ComputationTargetSpecification ct) {
    Long calcConfId = _calculationConfigurations.get(calcConfName);
    Long computationTargetId = _computationTargets.get(ct);

    // first check to see if this status has already been queried for
    // and if the answer could therefore be found in the cache

    Pair<Long, Long> key = Pair.of(calcConfId, computationTargetId);
    if (statusCache.containsKey(key)) {
      StatusEntry existingStatusEntryInDb = statusCache.get(key);
      if (existingStatusEntryInDb != null) {
        // status entry in db.
        return existingStatusEntryInDb.getStatus();
      } else {
        // no status entry in db.
        return StatusEntry.Status.NOT_RUNNING;
      }
    }

    MapSqlParameterSource args = new MapSqlParameterSource();
    args.addValue("calculation_configuration_id", calcConfId);
    args.addValue("computation_target_id", computationTargetId);

    try {
      StatusEntry statusEntry = getJdbcTemplate().queryForObject(
        getElSqlBundle().getSql("SelectStatusEntry"),
        args,
        DbBatchUtils.ROW_MAPPER);

      // status entry in db found.
      statusCache.put(key, statusEntry);

      return statusEntry.getStatus();

    } catch (IncorrectResultSizeDataAccessException e) {
      // no status entry in the db.
      statusCache.remove(key);
      return StatusEntry.Status.NOT_RUNNING;
    }
  }


  protected void populateErrorCache(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, Map<ValueSpecification, BatchResultWriterFailure> errorCache, ViewResultEntry item) {
    BatchResultWriterFailure cachedFailure = new BatchResultWriterFailure();

    switch (item.getComputedValue().getInvocationResult()) {

      case FUNCTION_THREW_EXCEPTION:

        // an "original" failure
        //
        // There will only be 1 failure reason.

        ComputeFailure computeFailure = getComputeFailureFromDb(computeFailureCache, item);
        cachedFailure.addComputeFailureId(computeFailure.getId());

        break;

      case MISSING_INPUTS:

        // There may be 1-N failure reasons - one for each failed
        // function in the subtree below this node. (This
        // only includes "original", i.e., lowest-level, failures.)

        for (ValueSpecification missingInput : item.getComputedValue().getMissingInputs()) {

          BatchResultWriterFailure inputFailure = errorCache.get(missingInput);

          if (inputFailure == null) {

            ComputeFailureKey computeFailureKey = new ComputeFailureKey(
              missingInput.getFunctionUniqueId(),
              "N/A",
              "Missing input " + missingInput,
              "N/A");
            computeFailure = getComputeFailureFromDb(computeFailureCache, computeFailureKey);
            cachedFailure.addComputeFailureId(computeFailure.getId());

          } else {

            cachedFailure.addComputeFailureIds(inputFailure.getComputeFailureIds());

          }
        }

        break;
    }

    if (!cachedFailure.getComputeFailureIds().isEmpty()) {
      errorCache.put(item.getComputedValue().getSpecification(), cachedFailure);
    }
  }

  /*package*/ ComputeFailure getComputeFailureFromDb(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ViewResultEntry item) {
    if (item.getComputedValue().getInvocationResult() != InvocationResult.FUNCTION_THREW_EXCEPTION) {
      throw new IllegalArgumentException("Please give a failed item");
    }

    ExecutionLogWithContext rootLog = item.getComputedValue().getAggregatedExecutionLog().getRootLog();
    String exceptionClass = rootLog != null ? rootLog.getExecutionLog().getExceptionClass() : null;
    String exceptionMessage = rootLog != null ? rootLog.getExecutionLog().getExceptionMessage() : null;
    String exceptionStackTrace = rootLog != null ? rootLog.getExecutionLog().getExceptionStackTrace() : null;
    //ensure we don't end up with null going into the ComputeFailureKey for these strings.
    //this will probably be due to the fact that the rootLog was null.
    exceptionClass = defaultString(exceptionClass, "No logging information available");
    exceptionMessage = defaultString(exceptionMessage, "No logging information available");
    exceptionStackTrace = defaultString(exceptionStackTrace, "No logging information available");
    ComputeFailureKey computeFailureKey = new ComputeFailureKey(item.getComputedValue().getSpecification().getFunctionUniqueId(), exceptionClass, exceptionMessage, exceptionStackTrace);
    return getComputeFailureFromDb(computeFailureCache, computeFailureKey);
  }

  public ComputeFailure getComputeFailureFromDb(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ComputeFailureKey computeFailureKey) {
    ComputeFailure computeFailure = computeFailureCache.get(computeFailureKey);
    if (computeFailure != null) {
      return computeFailure;
    }
    try {
      int id = getJdbcTemplate().queryForObject(getElSqlBundle().getSql("SelectComputeFailureId"), DbBatchUtils.toSqlParameterSource(computeFailureKey), Integer.class);
      computeFailure = new ComputeFailure();
      computeFailure.setId(id);
      computeFailure.setFunctionId(computeFailureKey.getFunctionId());
      computeFailure.setExceptionClass(computeFailureKey.getExceptionClass());
      computeFailure.setExceptionMsg(computeFailureKey.getExceptionMsg());
      computeFailure.setStackTrace(computeFailureKey.getStackTrace());
      computeFailureCache.put(computeFailureKey, computeFailure);
      return computeFailure;
    } catch (IncorrectResultSizeDataAccessException e) {
      // Not seen a failure like this before - create and write to database
      try {
        computeFailure = saveComputeFailure(computeFailureCache, computeFailureKey);
        return computeFailure;
      } catch (DataAccessException e2) {
        s_logger.error("Failed to save compute failure", e2);
        throw new RuntimeException("Failed to save compute failure", e2);
      }
    }
  }

  public ComputeFailure saveComputeFailure(Map<ComputeFailureKey, ComputeFailure> computeFailureCache, ComputeFailureKey computeFailureKey) {
    ComputeFailure computeFailure;
    computeFailure = new ComputeFailure();
    final long computeFailureId = nextId(RSK_SEQUENCE_NAME);
    computeFailure.setId(computeFailureId);
    computeFailure.setFunctionId(computeFailureKey.getFunctionId());
    computeFailure.setExceptionClass(computeFailureKey.getExceptionClass());
    computeFailure.setExceptionMsg(computeFailureKey.getExceptionMsg());
    computeFailure.setStackTrace(computeFailureKey.getStackTrace());

    int rowCount = getJdbcTemplate().update(getElSqlBundle().getSql("InsertComputeFailure"), DbBatchUtils.toSqlParameterSource(computeFailure));
    if (rowCount == 1) {
      computeFailureCache.put(computeFailureKey, computeFailure);
      return computeFailure;
    }
    return computeFailure;
  }

  /**
   * Instances of this class are saved in the computation cache for each
   * failure (whether the failure is 'original' or due to missing inputs).
   * The set of Longs is a set of compute failure IDs (referencing
   * rsk_compute_failure(id)). The set is built bottom up.
   * For example, if A has two children, B and C, and B has failed
   * due to error 12, and C has failed due to errors 15 and 16, then
   * A has failed due to errors 12, 15, and 16.
   */
  protected static class BatchResultWriterFailure implements MissingValue, Serializable {
    /** Serialization version. */
    private static final long serialVersionUID = 1L;
    private Set<Long> _computeFailureIds = new HashSet<Long>();

    public Set<Long> getComputeFailureIds() {
      return Collections.unmodifiableSet(_computeFailureIds);
    }

    public void setComputeFailureIds(Set<Long> computeFailureIds) {
      _computeFailureIds = computeFailureIds;
    }

    public void addComputeFailureId(Long computeFailureId) {
      addComputeFailureIds(Collections.singleton(computeFailureId));
    }

    public void addComputeFailureIds(Set<? extends Long> computeFailureIds) {
      _computeFailureIds.addAll(computeFailureIds);
    }
  }
 
}
TOP

Related Classes of com.opengamma.masterdb.batch.DbBatchWriter

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.