package org.yaac.server.service;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.Lists.newLinkedList;
import static com.google.common.collect.Lists.transform;
import static com.google.common.collect.Maps.newHashMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.inject.Inject;
import org.antlr.runtime.RecognitionException;
import org.yaac.client.service.CRUDService;
import org.yaac.server.delegate.MemcacheServiceDelegate;
import org.yaac.server.egql.EGQLUtil;
import org.yaac.server.egql.evaluator.EvaluationResult;
import org.yaac.server.egql.evaluator.Evaluator;
import org.yaac.server.egql.processor.ProcessData.ProcessDataRecord;
import org.yaac.server.util.AutoBeanUtil;
import org.yaac.server.util.DatastoreUtil;
import org.yaac.shared.ErrorCode;
import org.yaac.shared.SharedConstants.Datastore;
import org.yaac.shared.YaacException;
import org.yaac.shared.crud.MetaNamespace;
import org.yaac.shared.editor.EntityHierarchy;
import org.yaac.shared.editor.EntityInfo;
import org.yaac.shared.editor.EntityUpdateInfo;
import org.yaac.shared.editor.PropertyUpdateInfo;
import org.yaac.shared.file.FileDownloadPath;
import org.yaac.shared.property.BlobKeyPropertyInfo;
import org.yaac.shared.property.BlobPropertyInfo;
import org.yaac.shared.property.BooleanPropertyInfo;
import org.yaac.shared.property.CategoryPropertyInfo;
import org.yaac.shared.property.DatePropertyInfo;
import org.yaac.shared.property.DoublePropertyInfo;
import org.yaac.shared.property.EmailPropertyInfo;
import org.yaac.shared.property.GeoPtPropertyInfo;
import org.yaac.shared.property.KeyInfo;
import org.yaac.shared.property.LinkPropertyInfo;
import org.yaac.shared.property.ListPropertyInfo;
import org.yaac.shared.property.LongPropertyInfo;
import org.yaac.shared.property.PhoneNumberPropertyInfo;
import org.yaac.shared.property.PostalAddressPropertyInfo;
import org.yaac.shared.property.PropertyInfo;
import org.yaac.shared.property.PropertyType;
import org.yaac.shared.property.ShortBlobPropertyInfo;
import org.yaac.shared.property.StringPropertyInfo;
import org.yaac.shared.property.TextPropertyInfo;
import org.yaac.shared.property.UserPropertyInfo;
import com.google.appengine.api.NamespaceManager;
import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.datastore.AsyncDatastoreService;
import com.google.appengine.api.datastore.Blob;
import com.google.appengine.api.datastore.Category;
import com.google.appengine.api.datastore.DatastoreNeedIndexException;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Email;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.GeoPt;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.Link;
import com.google.appengine.api.datastore.PhoneNumber;
import com.google.appengine.api.datastore.PostalAddress;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.ShortBlob;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.users.User;
import com.google.common.base.Function;
import com.googlecode.gql4j.GqlQuery;
/**
* @author Max Zhu (thebbsky@gmail.com)
*
*/
public class CRUDServiceImpl implements CRUDService {
private final Logger logger;
private final MemcacheServiceDelegate memcache;
@Inject
CRUDServiceImpl(Logger logger, MemcacheServiceDelegate memcache) {
super();
this.logger = logger;
this.memcache = memcache;
}
@Override
public List<MetaNamespace> loadMetaData() {
AsyncDatastoreService datastore = DatastoreServiceFactory.getAsyncDatastoreService();
// step 1 : load all namespaces
Query nsQ = new Query(Query.NAMESPACE_METADATA_KIND).setKeysOnly();
Iterable<Entity> namespaces = datastore.prepare(nsQ).asIterable();
// step 2 : load all kinds and properties per each namespaces
String defaultNs = NamespaceManager.get();
Map<String, Iterable<Entity>> kindsMap = new HashMap<String, Iterable<Entity>>();
Map<String, Iterable<Entity>> propertiesMap = new HashMap<String, Iterable<Entity>>();
for (Entity e : namespaces) {
String namespace = e.getKey().getName();
if (isNullOrEmpty(namespace)) {
NamespaceManager.set(defaultNs);
} else {
NamespaceManager.set(namespace);
}
Query kindQ = new Query(Query.KIND_METADATA_KIND).setKeysOnly();
kindsMap.put(namespace, datastore.prepare(kindQ).asIterable());
// ancestor query is not used here, properties will be grouped in memory
// not keyOnly query because we will need to load property_representation
Query propertyQ = new Query(Query.PROPERTY_METADATA_KIND);
propertiesMap.put(namespace, datastore.prepare(propertyQ).asIterable());
}
// set namespace back
NamespaceManager.set(defaultNs);
// step 3 : prepare results
return DatastoreUtil.buildMetaData(kindsMap, propertiesMap);
}
@Override
public List<EntityInfo> loadEntities(String kind, String filter, int start, int length
) throws YaacException {
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
Query q = isNullOrEmpty(filter) ?
new Query(kind) : new GqlQuery("select * from " + kind + " " + filter).query();
FetchOptions options = FetchOptions.Builder.withLimit(length).offset(start);
try {
Iterable<Entity> entities = datastore.prepare(q).asIterable(options);
List<EntityInfo> result = new LinkedList<EntityInfo>();
for (Entity e : entities) {
result.add(DatastoreUtil.convert(e));
}
return result;
} catch (DatastoreNeedIndexException e) {
logger.log(Level.INFO,
"exception when loading entities, " +
" kind = " + kind +
" filter = " + filter +
" start = " + start +
" length = " + length, e);
throw new YaacException(null, e.getMissingIndexDefinitionXml());
}
}
@Override
public EntityHierarchy loadEntityHierarchy(String keyString) throws YaacException {
// precondition checking
if (isNullOrEmpty(keyString)) {
logger.info("Empty key string");
return null;
}
try {
Key key = null;
try {
EvaluationResult r = EGQLUtil.parser(keyString).bool_exp().e.evaluate((ProcessDataRecord)null);
key = (Key) r.getPayload();
} catch (Exception e) {
logger.info("failed to load key in formatted version, fallback to parse as keyString");
// any exception, fallback
key = KeyFactory.stringToKey(keyString);
}
// load data
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
Map<Key, Entity> resultMap = datastore.get(DatastoreUtil.withAllAncesterKeys(key));
EntityHierarchy hierarchy = new EntityHierarchy(
DatastoreUtil.convert(key), DatastoreUtil.convert(resultMap.get(key)));
while (key.getParent() != null) {
key = key.getParent();
hierarchy.put(DatastoreUtil.convert(key), DatastoreUtil.convert(resultMap.get(key)));
}
return hierarchy;
} catch (IllegalArgumentException e) {
// invalid key string
throw new YaacException(null, "Invalid key string");
}
}
@Override
public void updateEntity(Collection<EntityUpdateInfo> infos) {
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
Map<Key, EntityUpdateInfo> infoMap = newHashMap();
// step 1 : load existing entities from datastore
List<Key> keys = new ArrayList<Key>(infos.size());
for (EntityUpdateInfo info : infos) {
Key key = DatastoreUtil.convert(info.getOrigKey());
keys.add(key);
infoMap.put(key, info);
}
Map<Key, Entity> entityMap = datastore.get(keys);
// step 2 : update
List<Entity> updateList = newLinkedList();
for (Key key : infoMap.keySet()) {
EntityUpdateInfo info = infoMap.get(key);
Entity entity = entityMap.get(key);
updateList.add(applyChange(entity, info.getChanges()));
}
datastore.put(updateList);
}
/**
* @param entity
* @param changes
* @return
*/
Entity applyChange(Entity entity, List<PropertyUpdateInfo> changes) {
for (PropertyUpdateInfo change : changes) {
if (Datastore.KEY_RESERVED_NAME.equals(change.getName())) {
// key is changed
// clone properties to new entity
Key newKey = DatastoreUtil.convert((KeyInfo)change.getNewInfo());
Entity newEntity = new Entity(newKey);
newEntity.setPropertiesFrom(entity);
// remove existing entity
DatastoreServiceFactory.getAsyncDatastoreService().delete(entity.getKey());
entity = newEntity;
} else {
if (change.isDeleteFlag()) {
// simply delete the property
entity.removeProperty(change.getName());
continue;
}
// change name
if (change.getNewName() != null) {
Object value = entity.getProperty(change.getName());
entity.removeProperty(change.getName());
entity.setProperty(change.getNewName(), value);
}
if (change.getNewInfo() != null) { // value changed
entity.setProperty(change.getNewName() == null ? change.getName() : change.getNewName(),
convert(change.getNewInfo(), entity));
}
}
}
return entity;
}
/**
* convert from PropertyInfo to object value
*
* @param info
* @return
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
private Object convert(PropertyInfo info, Entity e) {
PropertyType type = PropertyType.typeOf(info);
switch (type) {
case BLOB:
BlobPropertyInfo blobInfo = (BlobPropertyInfo) info;
if (blobInfo.getRawData() == null) {
FileDownloadPath path = AutoBeanUtil.decode(FileDownloadPath.class, blobInfo.getDownloadPath());
switch (path.getType()) {
case MEMCACHE:
byte [] data = (byte[]) memcache.get(path.getKeyStr());
// throw exception when rawData == null? cache get expired is possible
return new Blob(data);
case DATASTORE_BLOB:
// simply return existing one
Object val = e.getProperty(path.getFieldName());
if (val instanceof Blob) {
return val;
} else if (val instanceof List) {
return ((List)val).get(path.getIndex());
} else {
// should not happen
throw new IllegalArgumentException("Can't locate blob via path " + path.getFieldName() + " " + path.getIndex());
}
default:
// should not happen
throw new IllegalArgumentException("Can't locate blob via path " + path.getFieldName() + " " + path.getIndex());
}
} else {
// edit blob value directly
return new Blob(blobInfo.getRawData());
}
case BLOB_KEY:
return new BlobKey(((BlobKeyPropertyInfo) info).getBlobKey());
case BOOL:
return ((BooleanPropertyInfo) info).getPayload();
case CATEGORY:
return new Category(((CategoryPropertyInfo) info).getPayload());
case DOUBLE:
return ((DoublePropertyInfo) info).getPayload();
case EMAIL:
return new Email(((EmailPropertyInfo) info).getPayload());
case GEOPT:
GeoPtPropertyInfo geoPtInfo = (GeoPtPropertyInfo) info;
return new GeoPt(geoPtInfo.getLatitude(), geoPtInfo.getLongitude());
case IM_HANDLE:
// TODO
return null;
case KEY:
return DatastoreUtil.convert((KeyInfo)info);
case LINK:
return new Link(((LinkPropertyInfo) info).getPayload());
case LIST:
ListPropertyInfo listInfo = (ListPropertyInfo) info;
List list = newLinkedList();
for (PropertyInfo i : listInfo.getPayload()) {
list.add(convert(i, e));
}
return list;
case LONG:
return ((LongPropertyInfo) info).getPayload();
case NULL:
return null;
case PHONE_NO:
return new PhoneNumber(((PhoneNumberPropertyInfo) info).getPayload());
case POSTTAL_ADDRESS:
return new PostalAddress(((PostalAddressPropertyInfo) info).getPayload());
case SHORT_BLOB:
return new ShortBlob(((ShortBlobPropertyInfo) info).getPayload().getBytes());
case STRING:
return ((StringPropertyInfo) info).getPayload();
case TEXT:
TextPropertyInfo textInfo = (TextPropertyInfo) info;
if (textInfo.getRawText() == null) {
FileDownloadPath path = AutoBeanUtil.decode(FileDownloadPath.class, textInfo.getDownloadPath());
switch (path.getType()) {
case MEMCACHE:
byte [] data = (byte[]) memcache.get(path.getKeyStr());
// throw exception when rawData == null? cache get expired is possible
return new Text(new String(data));
case DATASTORE_TEXT:
// simply return existing one
Object val = e.getProperty(path.getFieldName());
if (val instanceof Text) {
return val;
} else if (val instanceof List) {
return ((List)val).get(path.getIndex());
} else {
// should not happen
throw new IllegalArgumentException("Can't locate text via path " + path.getFieldName() + " " + path.getIndex());
}
default:
// should not happen
throw new IllegalArgumentException("Can't locate text via path " + path.getFieldName() + " " + path.getIndex());
}
} else {
// edit text value directly
return new Text(textInfo.getRawText());
}
case TIMESTAMP:
return ((DatePropertyInfo) info).getPayload();
case USER:
UserPropertyInfo upi = (UserPropertyInfo) info;
// nickname is a derived field, no need to set here
return new User(upi.getEmail(), upi.getAuthDomain(), upi.getUserId(), upi.getFederatedIdentity());
default:
throw new IllegalArgumentException("Invalid type " + info.getClass());
}
}
@Override
public PropertyInfo parsePropertyExp(String keyString, String exp, List<String> filePaths) throws YaacException {
try {
// property can also be used during evaluation
Evaluator e = EGQLUtil.parser(exp).bool_exp().e;
if (!e.aggregationChildren().isEmpty()) {
// aggregation function is not allowed here
throw new YaacException(ErrorCode.E301, null);
}
// load targeting entity
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
final Entity entity = datastore.get(KeyFactory.stringToKey(keyString));
// file context
final List<FileDownloadPath> files = filePaths == null ? new ArrayList<FileDownloadPath>() :
transform(filePaths, new Function<String, FileDownloadPath>(){
@Override
public FileDownloadPath apply(String pathStr) {
return AutoBeanUtil.decode(FileDownloadPath.class, pathStr);
}
});
EvaluationResult r = e.evaluate(new ProcessDataRecord() {
@Override
public FileDownloadPath lookupFileReference(Integer index) {
return files.get(index);
}
@Override
public EvaluationResult lookup(String name) {
if (isNullOrEmpty(name)) {
return null;
} else { // normal case, include key selection and property selection
Object payload = Datastore.KEY_RESERVED_NAME.equals(name) ? entity.getKey() : entity.getProperty(name);
return new EvaluationResult(entity.getKey(), name, null, payload);
}
}
@Override
public Iterable<EvaluationResult> asIterable() {
// should not be called
throw new IllegalArgumentException();
}
});
PropertyInfo info = DatastoreUtil.convert(keyString, null, null, r.getPayload(), r.getWarnings());
return info;
} catch (RecognitionException e) {
// can't parse input
logger.log(Level.SEVERE, e.getMessage(), e);
throw new YaacException(null, e.getMessage());
} catch (EntityNotFoundException e) {
logger.log(Level.SEVERE, e.getMessage(), e);
throw new YaacException(ErrorCode.E302, null);
}
}
@Override
public void delete(KeyInfo keyInfo) {
Key key = DatastoreUtil.convert(keyInfo);
DatastoreServiceFactory.getDatastoreService().delete(key);
}
@Override
public String copyToNew(KeyInfo keyInfo) throws YaacException {
try {
Key key = DatastoreUtil.convert(keyInfo);
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
Entity from = datastore.get(key);
Entity to = new Entity(key.getKind()); // copy kind, but not key name
to.setPropertiesFrom(from); // copy all properties
datastore.put(to);
return KeyFactory.keyToString(to.getKey());
} catch (EntityNotFoundException e) {
throw new YaacException(null, "Requested entity doesn't exist, please refresh your browser");
}
}
@Override
public void create(KeyInfo keyInfo) {
Key key = DatastoreUtil.convert(keyInfo);
Entity e = new Entity(key);
DatastoreServiceFactory.getDatastoreService().put(e);
}
}