description,
owners);
log.info("Verifying store: \n" + newStoreDefXml.toString());
StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
// get store def from cluster
log.info("Getting store definition from: " + url + " ( node id " + this.nodeId + " )");
List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
.getValue();
boolean foundStore = false;
// go over all store defs and see if one has the same name as the store
// we're trying
// to build
for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
if(remoteStoreDef.getName().equals(storeName)) {
// if the store already exists, but doesn't match what we want
// to push, we need
// to worry
if(!remoteStoreDef.equals(newStoreDef)) {
// it is possible that the stores actually DO match, but the
// json in the key/value serializers is out of order (eg
// {'a': 'int32', 'b': 'int32'} could have a/b reversed.
// this is just a reflection of the fact that voldemort json
// type defs use hashmaps that are unordered, and pig uses
// bags that are unordered as well. it's therefore
// unpredictable what order the keys will come out of pig.
// let's check to see if the key/value serializers are
// REALLY equal.
SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
if(remoteKeySerializerDef.getName().equals("json")
&& remoteValueSerializerDef.getName().equals("json")
&& remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
&& remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
JsonTypeDefinition remoteKeyDef = JsonTypeDefinition.fromJson(remoteKeySerializerDef.getCurrentSchemaInfo());
JsonTypeDefinition remoteValDef = JsonTypeDefinition.fromJson(remoteValueSerializerDef.getCurrentSchemaInfo());
JsonTypeDefinition localKeyDef = JsonTypeDefinition.fromJson(localKeySerializerDef.getCurrentSchemaInfo());
JsonTypeDefinition localValDef = JsonTypeDefinition.fromJson(localValueSerializerDef.getCurrentSchemaInfo());
if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
// if the key/value serializers are REALLY equal
// (even though the strings may not match), then
// just use the remote stores to GUARANTEE that they
// match, and try again.
keySchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
+ remoteKeySerializerDef.getCurrentSchemaInfo()
+ "</schema-info>\n\t" + keySchemaCompression;
valSchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
+ remoteValueSerializerDef.getCurrentSchemaInfo()
+ "</schema-info>\n\t" + valueSchemaCompression;
newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
replicationFactor,
requiredReads,
requiredWrites,
(preferredReads < 0) ? null
: preferredReads,
(preferredWrites < 0) ? null
: preferredWrites,
keySchema,
valSchema,
description,
owners);
newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
if(!remoteStoreDef.equals(newStoreDef)) {
// if we still get a fail, then we know that the
// store defs don't match for reasons OTHER than
// the key/value serializer
throw new RuntimeException("Your store schema is identical, but the store definition does not match. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
} else {
// if the key/value serializers are not equal (even
// in java, not just json strings), then fail
throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
}
}
foundStore = true;
break;
}
}
// if the store doesn't exist yet, create it
if(!foundStore) {
// New requirement - Make sure the user had description and owner
// specified
if(description.length() == 0) {
throw new RuntimeException("Description field missing in store definition. "
+ "Please add \"push.store.description\" with a line describing your store");
}
if(owners.length() == 0) {
throw new RuntimeException("Owner field missing in store definition. "
+ "Please add \"push.store.owners\" with value being comma-separated list of LinkedIn email ids");
}
log.info("Could not find store " + storeName
+ " on Voldemort. Adding it to all nodes for cluster " + url);
adminClient.storeMgmtOps.addStore(newStoreDef);
}
// don't use newStoreDef because we want to ALWAYS use the JSON
// definition since the store builder assumes that you are using
// JsonTypeSerializer. This allows you to tweak your value/key store xml
// as you see fit, but still uses the json sequence file meta data to
// build the store.
StoreDefinition storeDef = VoldemortUtils.getStoreDef(VoldemortUtils.getStoreDefXml(storeName,
replicationFactor,
requiredReads,
requiredWrites,
(preferredReads < 0) ? null
: preferredReads,