/******************************************************************************
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
* the specific language governing rights and limitations under the License.
*
* The Original Code is: Jsoda
* The Initial Developer of the Original Code is: William Wong (williamw520@gmail.com)
* Portions created by William Wong are Copyright (C) 2012 William Wong, All Rights Reserved.
*
******************************************************************************/
package wwutil.jsoda;
import java.io.*;
import java.net.*;
import java.util.*;
import java.lang.reflect.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.services.simpledb.AmazonSimpleDBClient;
import com.amazonaws.services.simpledb.model.CreateDomainRequest;
import com.amazonaws.services.simpledb.model.DeleteDomainRequest;
import com.amazonaws.services.simpledb.model.ListDomainsResult;
import com.amazonaws.services.simpledb.model.PutAttributesRequest;
import com.amazonaws.services.simpledb.model.BatchPutAttributesRequest;
import com.amazonaws.services.simpledb.model.ReplaceableItem;
import com.amazonaws.services.simpledb.model.ReplaceableAttribute;
import com.amazonaws.services.simpledb.model.GetAttributesRequest;
import com.amazonaws.services.simpledb.model.GetAttributesResult;
import com.amazonaws.services.simpledb.model.Attribute;
import com.amazonaws.services.simpledb.model.SelectRequest;
import com.amazonaws.services.simpledb.model.SelectResult;
import com.amazonaws.services.simpledb.model.Item;
import com.amazonaws.services.simpledb.model.DeleteAttributesRequest;
import com.amazonaws.services.simpledb.model.BatchDeleteAttributesRequest;
import com.amazonaws.services.simpledb.model.DeletableItem;
import com.amazonaws.services.simpledb.model.UpdateCondition;
import com.amazonaws.services.simpledb.util.SimpleDBUtils;
import wwutil.sys.TlsMap;
import wwutil.sys.ReflectUtil;
import wwutil.model.MemCacheable;
import wwutil.model.annotation.DbType;
import wwutil.model.annotation.Model;
import wwutil.model.annotation.CachePolicy;
import wwutil.model.annotation.DefaultGUID;
import wwutil.model.annotation.DefaultComposite;
import wwutil.model.annotation.CacheByField;
/**
* SimpleDB specific functions
*/
class SimpleDBService implements DbService
{
private static Log log = LogFactory.getLog(SimpleDBService.class);
static final Set<String> sOperatorMap = new HashSet<String>(){{
add(Filter.NULL);
add(Filter.NOT_NULL);
add(Filter.EQ);
add(Filter.NE);
add(Filter.LE);
add(Filter.LT);
add(Filter.GE);
add(Filter.GT);
add(Filter.LIKE);
add(Filter.NOT_LIKE);
add(Filter.BETWEEN);
add(Filter.IN);
}};
public static final String ITEM_NAME = "itemName()";
public static final int MAX_PUT_ITEMS = 25; // SimpleDB has a limit of 25 items per batch.
private Jsoda jsoda;
private AmazonSimpleDBClient sdbClient;
private String endPoint;
// AWS Access Key ID and Secret Access Key
public SimpleDBService(Jsoda jsoda, AWSCredentials cred)
throws Exception
{
this.jsoda = jsoda;
this.sdbClient = new AmazonSimpleDBClient(cred);
}
public void shutdown() {
sdbClient.shutdown();
}
public DbType getDbType() {
return DbType.SimpleDB;
}
public String getDbTypeId() {
return "SDB";
}
public void setDbEndpoint(String endpoint) {
this.endPoint = endpoint;
sdbClient.setEndpoint(endpoint);
}
public String getDbEndpoint() {
return this.endPoint;
}
// Delegated SimpleDB API
public void createModelTable(String modelName) {
sdbClient.createDomain(new CreateDomainRequest(jsoda.getModelTable(modelName)));
}
public void deleteTable(String tableName) {
sdbClient.deleteDomain(new DeleteDomainRequest(tableName));
}
public List<String> listTables() {
ListDomainsResult list = sdbClient.listDomains();
return list.getDomainNames();
}
private String makeCompositePk(String modelName, Object id, Object rangeKey)
throws Exception
{
String idStr = DataUtil.encodeValueToAttrStr(id, jsoda.getIdField(modelName).getType());
String rangeStr = DataUtil.encodeValueToAttrStr(rangeKey, jsoda.getRangeField(modelName).getType());
String pk = idStr.length() + ":" + idStr + "/" + rangeStr;
return pk;
}
private String[] parseCompositePk(String modelName, String compositePk) {
int index = compositePk.indexOf(":");
String lenStr = compositePk.substring(0, index);
int len = Integer.parseInt(lenStr);
String idStr = compositePk.substring(index + 1, index + 1 + len);
String rangeStr = compositePk.substring(index + 1 + len + 1);
return new String[] {idStr, rangeStr};
}
private String makeIdValue(String modelName, Object id, Object rangeKey)
throws Exception
{
String idStr = DataUtil.encodeValueToAttrStr(id, jsoda.getIdField(modelName).getType());
Field rangeField = jsoda.getRangeField(modelName);
String pk = rangeField == null ? idStr : makeCompositePk(modelName, id, rangeKey);
return pk;
}
private String makeIdValue(String modelName, Object dataObj)
throws Exception
{
Field idField = jsoda.getIdField(modelName);
Field rangeField = jsoda.getRangeField(modelName);
Object id = idField.get(dataObj);
Object rangeKey = rangeField == null ? null : rangeField.get(dataObj);
return makeIdValue(modelName, id, rangeKey);
}
public <T> void putObj(Class<T> modelClass, T dataObj, String expectedField, Object expectedValue, boolean expectedExists)
throws Exception
{
String modelName = jsoda.getModelName(modelClass);
String table = jsoda.getModelTable(modelName);
String idValue = makeIdValue(modelName, dataObj);
PutAttributesRequest req =
expectedField == null ?
new PutAttributesRequest(table, idValue, buildAttrs(dataObj, modelName)) :
new PutAttributesRequest(table, idValue, buildAttrs(dataObj, modelName),
buildExpectedValue(modelName, expectedField, expectedValue, expectedExists));
sdbClient.putAttributes(req);
}
public <T> void putObjs(Class<T> modelClass, List<T> dataObjs)
throws Exception
{
String modelName = jsoda.getModelName(modelClass);
int offset = 0;
String table = jsoda.getModelTable(modelName);
while (offset < dataObjs.size()) {
List<ReplaceableItem> items = buildPutItems(dataObjs, modelName, offset);
offset += items.size();
sdbClient.batchPutAttributes(new BatchPutAttributesRequest(table, items));
}
}
public <T> T getObj(Class<T> modelClass, Object id, Object rangeKey)
throws Exception
{
if (id == null)
throw new IllegalArgumentException("Id cannot be null.");
String modelName = jsoda.getModelName(modelClass);
String table = jsoda.getModelTable(modelName);
String idValue = makeIdValue(modelName, id, rangeKey);
GetAttributesResult result = sdbClient.getAttributes(new GetAttributesRequest(table, idValue));
if (result.getAttributes().size() == 0)
return null; // not existed.
return buildLoadObj(modelClass, modelName, idValue, result.getAttributes(), null);
}
public void delete(String modelName, Object id, Object rangeKey)
throws Exception
{
if (id == null)
throw new IllegalArgumentException("Id cannot be null.");
String table = jsoda.getModelTable(modelName);
String idValue = makeIdValue(modelName, id, rangeKey);
sdbClient.deleteAttributes(new DeleteAttributesRequest(table, idValue));
}
public void batchDelete(String modelName, List idList, List rangeKeyList)
throws Exception
{
String table = jsoda.getModelTable(modelName);
List<DeletableItem> items = new ArrayList<DeletableItem>();
for (int i = 0; i < idList.size(); i++) {
String idValue = makeIdValue(modelName, idList.get(i), rangeKeyList == null ? null : rangeKeyList.get(i));
items.add(new DeletableItem().withName(idValue));
}
sdbClient.batchDeleteAttributes(new BatchDeleteAttributesRequest(table, items));
}
public void validateFilterOperator(String operator) {
if (!sOperatorMap.contains(operator))
throw new UnsupportedOperationException("Unsupported operator: " + operator);
}
@SuppressWarnings("unchecked")
public <T> long queryCount(Class<T> modelClass, Query<T> query)
throws JsodaException
{
String modelName = jsoda.getModelName(modelClass);
String queryStr = toQueryStr(query, true);
SelectRequest request = new SelectRequest(queryStr, query.consistentRead);
try {
for (Item item : sdbClient.select(request).getItems()) {
for (Attribute attr : item.getAttributes()) {
String attrName = attr.getName();
String fieldValue = attr.getValue();
long count = Long.parseLong(fieldValue);
return count;
}
}
} catch(Exception e) {
throw new JsodaException("Query failed. Query: " + request.getSelectExpression() + " Error: " + e.getMessage(), e);
}
throw new JsodaException("Query failed. Not result for count query.");
}
@SuppressWarnings("unchecked")
public <T> List<T> queryRun(Class<T> modelClass, Query<T> query, boolean continueFromLastRun)
throws JsodaException
{
List<T> resultObjs = new ArrayList<T>();
if (continueFromLastRun && !queryHasNext(query))
return resultObjs;
String queryStr = toQueryStr(query, false);
log.info("Query: " + queryStr);
SelectRequest request = new SelectRequest(queryStr, query.consistentRead);
if (continueFromLastRun)
request.setNextToken((String)query.nextKey);
try {
SelectResult result = sdbClient.select(request);
query.nextKey = request.getNextToken();
for (Item item : result.getItems()) {
String idValue = item.getName(); // get the id from the item's name()
T obj = buildLoadObj(modelClass, query.modelName, idValue, item.getAttributes(), query);
resultObjs.add(obj);
}
return resultObjs;
} catch(Exception e) {
throw new JsodaException("Query failed. Query: " + request.getSelectExpression() + " Error: " + e.getMessage(), e);
}
}
public <T> boolean queryHasNext(Query<T> query) {
return query.nextKey != null;
}
public String getFieldAttrName(String modelName, String fieldName) {
// SimpleDB's attribute name for single Id always maps to "itemName()"
if (jsoda.getRangeField(modelName) == null && jsoda.isIdField(modelName, fieldName))
return ITEM_NAME;
String attrName = jsoda.getFieldAttrMap(modelName).get(fieldName);
return attrName != null ? SimpleDBUtils.quoteName(attrName) : null;
}
private List<ReplaceableAttribute> buildAttrs(Object dataObj, String modelName)
throws Exception
{
List<ReplaceableAttribute> attrs = new ArrayList<ReplaceableAttribute>();
for (Map.Entry<String, String> fieldAttr : jsoda.getFieldAttrMap(modelName).entrySet()) {
String fieldName = fieldAttr.getKey();
String attrName = fieldAttr.getValue();
Field field = jsoda.getField(modelName, fieldName);
Object value = field.get(dataObj);
String fieldValueStr = DataUtil.encodeValueToAttrStr(value, field.getType());
// Skip null value field. No attribute stored at db.
if (fieldValueStr == null)
continue;
// Add attr:fieldValueStr to list. Skip the single Id field. Treats single Id field as the itemName key in SimpleDB.
if (!(jsoda.getRangeField(modelName) == null && jsoda.isIdField(modelName, fieldName)))
attrs.add(new ReplaceableAttribute(attrName, fieldValueStr, true));
}
return attrs;
}
private UpdateCondition buildExpectedValue(String modelName, String expectedField, Object expectedValue, boolean expectedExists)
throws Exception
{
if (expectedValue == null)
throw new IllegalArgumentException("ExpectedValue cannot be null.");
String attrName = jsoda.getFieldAttrMap(modelName).get(expectedField);
String fieldValue = DataUtil.encodeValueToAttrStr(expectedValue, jsoda.getField(modelName, expectedField).getType());
UpdateCondition cond = new UpdateCondition();
cond.setExists(expectedExists);
cond.setName(attrName);
if (expectedExists) {
cond.setValue(fieldValue);
}
return cond;
}
private List<ReplaceableItem> buildPutItems(List dataObjs, String modelName, int offset)
throws Exception
{
List<ReplaceableItem> items = new ArrayList<ReplaceableItem>();
for (int i = offset; i < dataObjs.size() && items.size() < MAX_PUT_ITEMS; i++) {
Object dataObj = dataObjs.get(i);
String idValue = makeIdValue(modelName, dataObj);
items.add(new ReplaceableItem(idValue, buildAttrs(dataObj, modelName)));
}
return items;
}
private <T> T buildLoadObj(Class<T> modelClass, String modelName, String idValue, List<Attribute> attrs, Query query)
throws Exception
{
T obj = modelClass.newInstance();
Map<String, Field> attrFieldMap = jsoda.getAttrFieldMap(modelName);
// Set the attr field
for (Attribute attr : attrs) {
String attrName = attr.getName();
String attrStr = attr.getValue();
Field field = attrFieldMap.get(attrName);
//log.debug("attrName " + attrName + " attrStr: " + attrStr);
if (field == null) {
log.warn("Attribute " + attrName + " from db has no corresponding field in model class " + modelClass);
continue;
}
DataUtil.setFieldValueStr(obj, field, attrStr);
}
if (query == null) {
backfillIdAndRange(modelClass, modelName, obj, idValue);
} else {
// Any select type involving the id or the range key need them to be backfilled.
switch (query.selectType) {
case Query.SELECT_ALL:
case Query.SELECT_ID:
case Query.SELECT_ID_RANGE:
case Query.SELECT_ID_OTHERS:
case Query.SELECT_ID_RANGE_OTHERS:
case Query.SELECT_RANGE:
case Query.SELECT_RANGE_OTHERS:
backfillIdAndRange(modelClass, modelName, obj, idValue);
break;
case Query.SELECT_OTHERS:
break;
}
}
return obj;
}
private <T> void backfillIdAndRange(Class<T> modelClass, String modelName, T obj, String idValue)
throws Exception
{
Field idField = jsoda.getIdField(modelName);
Field rangeField = jsoda.getRangeField(modelName);
if (jsoda.getRangeField(modelName) == null) {
// Backfill idField with the the item's name as the idValue.
DataUtil.setFieldValueStr(obj, idField, idValue);
} else {
// Decode the idField and rangeField from the idValue
String[] pair = parseCompositePk(modelName, idValue);
idField.set(obj, DataUtil.decodeAttrStrToValue(pair[0], idField.getType()));
rangeField.set(obj, DataUtil.decodeAttrStrToValue(pair[1], rangeField.getType()));
}
}
private <T> String toQueryStr(Query<T> query, boolean selectCount) {
StringBuilder sb = new StringBuilder();
addSelectStr(query, selectCount, sb);
addFromStr(query, sb);
addFilterStr(query, sb);
addOrderbyStr(query, sb);
addLimitStr(query, sb);
return sb.toString();
}
private <T> void addSelectStr(Query<T> query, boolean selectCount, StringBuilder sb) {
if (selectCount) {
sb.append("select count(*) ");
return;
}
switch (query.selectType) {
case Query.SELECT_ALL:
{
sb.append("select * ");
return;
}
case Query.SELECT_ID:
{
// Select just the id field (the ITEM_NAME() term).
sb.append("select ").append(ITEM_NAME);
return;
}
case Query.SELECT_ID_RANGE:
case Query.SELECT_ID_OTHERS:
case Query.SELECT_ID_RANGE_OTHERS:
{
int index = 0;
for (String term : query.selectTerms) {
// Skip the Id term as SimpleDB doesn't allow mixing of Select itemName(), other1, other2.
// Id field is always back-fill during post query processing from the item name so it will be in the result.
if (jsoda.isIdField(query.modelName, term))
continue;
sb.append(index++ == 0 ? "select " : ", ");
sb.append(getFieldAttrName(query.modelName, term));
}
return;
}
case Query.SELECT_RANGE:
case Query.SELECT_RANGE_OTHERS:
case Query.SELECT_OTHERS:
{
// Id field is not needed.
// Id field is always back-fill during post query processing from the item name so it will be in the result.
int index = 0;
for (String term : query.selectTerms) {
sb.append(index++ == 0 ? "select " : ", ");
sb.append(getFieldAttrName(query.modelName, term));
}
return;
}
}
}
private <T> void addFromStr(Query<T> query, StringBuilder sb) {
sb.append(" from ").append(SimpleDBUtils.quoteName(jsoda.getModelTable(query.modelName)));
}
private <T> void addFilterStr(Query<T> query, StringBuilder sb) {
int index = 0;
for (Filter filter : query.filters) {
sb.append(index++ == 0 ? " where " : " and ");
filter.toSimpleDBConditionStr(sb);
}
}
private <T> void addOrderbyStr(Query<T> query, StringBuilder sb) {
int index = 0;
for (String orderby : query.orderbyFields) {
sb.append(index++ == 0 ? " order by " : ", ");
String term = orderby.substring(1);
String ascDesc = orderby.charAt(0) == '+' ? " asc" : " desc";
sb.append(getFieldAttrName(query.modelName, term));
sb.append(ascDesc);
}
}
private <T> void addLimitStr(Query<T> query, StringBuilder sb) {
if (query.limit > 0)
sb.append(" limit ").append(query.limit);
}
}