Package voldemort.serialization

Examples of voldemort.serialization.SerializerDefinition


                    // type defs use hashmaps that are unordered, and pig uses
                    // bags that are unordered as well. it's therefore
                    // unpredictable what order the keys will come out of pig.
                    // let's check to see if the key/value serializers are
                    // REALLY equal.
                    SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                    SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                    SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                    SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();

                    if(remoteKeySerializerDef.getName().equals("json")
                       && remoteValueSerializerDef.getName().equals("json")
                       && remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
                       && remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
                        JsonTypeDefinition remoteKeyDef = JsonTypeDefinition.fromJson(remoteKeySerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition remoteValDef = JsonTypeDefinition.fromJson(remoteValueSerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition localKeyDef = JsonTypeDefinition.fromJson(localKeySerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition localValDef = JsonTypeDefinition.fromJson(localValueSerializerDef.getCurrentSchemaInfo());

                        if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
                            // if the key/value serializers are REALLY equal
                            // (even though the strings may not match), then
                            // just use the remote stores to GUARANTEE that they
                            // match, and try again.
                            keySchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                        + remoteKeySerializerDef.getCurrentSchemaInfo()
                                        + "</schema-info>\n\t" + keySchemaCompression;
                            valSchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                        + remoteValueSerializerDef.getCurrentSchemaInfo()
                                        + "</schema-info>\n\t" + valueSchemaCompression;
                            newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
                                                                           replicationFactor,
                                                                           requiredReads,
                                                                           requiredWrites,
View Full Code Here


        this.useNio = true;
    }

    public static List<StoreDefinition> getStoreDef(int nodeId) {
        List<StoreDefinition> defs = new ArrayList<StoreDefinition>();
        SerializerDefinition serDef = new SerializerDefinition("string");
        String storageConfiguration = InMemoryStorageConfiguration.TYPE_NAME;
        if(nodeId == 2) {
            storageConfiguration = SlowStorageConfiguration.TYPE_NAME;
        }
        defs.add(new StoreDefinitionBuilder().setName(STORE_NAME)
View Full Code Here

    }

    public void startFourNodeRO() throws IOException {
        storeDef1 = new StoreDefinitionBuilder().setName("test")
                                                .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                .setKeySerializer(new SerializerDefinition("string"))
                                                .setValueSerializer(new SerializerDefinition("string"))
                                                .setRoutingPolicy(RoutingTier.CLIENT)
                                                .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                .setReplicationFactor(2)
                                                .setPreferredReads(1)
                                                .setRequiredReads(1)
                                                .setPreferredWrites(1)
                                                .setRequiredWrites(1)
                                                .build();
        storeDef2 = new StoreDefinitionBuilder().setName("test2")
                                                .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                .setKeySerializer(new SerializerDefinition("string"))
                                                .setValueSerializer(new SerializerDefinition("string"))
                                                .setRoutingPolicy(RoutingTier.CLIENT)
                                                .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                .setReplicationFactor(3)
                                                .setPreferredReads(1)
                                                .setRequiredReads(1)
View Full Code Here

    }

    public void startFourNodeRORW() throws IOException {
        storeDef1 = new StoreDefinitionBuilder().setName("test")
                                                .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                .setKeySerializer(new SerializerDefinition("string"))
                                                .setValueSerializer(new SerializerDefinition("string"))
                                                .setRoutingPolicy(RoutingTier.CLIENT)
                                                .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                .setReplicationFactor(2)
                                                .setPreferredReads(1)
                                                .setRequiredReads(1)
                                                .setPreferredWrites(1)
                                                .setRequiredWrites(1)
                                                .build();
        storeDef2 = new StoreDefinitionBuilder().setName("test2")
                                                .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                .setKeySerializer(new SerializerDefinition("string"))
                                                .setValueSerializer(new SerializerDefinition("string"))
                                                .setRoutingPolicy(RoutingTier.CLIENT)
                                                .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                .setReplicationFactor(3)
                                                .setPreferredReads(1)
                                                .setRequiredReads(1)
View Full Code Here

                           Lists.newArrayList(storeDef1,
                                              storeDef2,
                                              new StoreDefinitionBuilder()
                                                                          .setName("test3")
                                                                          .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                                          .setKeySerializer(new SerializerDefinition("string"))
                                                                          .setValueSerializer(new SerializerDefinition("string"))
                                                                          .setRoutingPolicy(RoutingTier.CLIENT)
                                                                          .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                                          .setReplicationFactor(2)
                                                                          .setPreferredReads(1)
                                                                          .setRequiredReads(1)
View Full Code Here

                                              storeDef2,
                                              storeDef3,
                                              storeDef4,
                                              new StoreDefinitionBuilder().setName("test5")
                                                                          .setType(ReadOnlyStorageConfiguration.TYPE_NAME)
                                                                          .setKeySerializer(new SerializerDefinition("string"))
                                                                          .setValueSerializer(new SerializerDefinition("string"))
                                                                          .setRoutingPolicy(RoutingTier.CLIENT)
                                                                          .setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
                                                                          .setReplicationFactor(2)
                                                                          .setPreferredReads(1)
                                                                          .setRequiredReads(1)
View Full Code Here

                        // serializers is out of order (eg {'a': 'int32', 'b': 'int32'}  could have a/b reversed.
                        // This is just a reflection of the fact that voldemort json type defs use hashmaps that
                        // are unordered, and pig uses bags that are unordered  as well. it's therefore unpredictable
                        // what order the keys will come out of pig. let's check to see if the key/value
                        // serializers are REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();

                        if(remoteKeySerializerDef.getName().equals("json")
                           && remoteValueSerializerDef.getName().equals("json")
                           && remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
                           && remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
                            JsonTypeDefinition remoteKeyDef = JsonTypeDefinition.fromJson(remoteKeySerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition remoteValDef = JsonTypeDefinition.fromJson(remoteValueSerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition localKeyDef = JsonTypeDefinition.fromJson(localKeySerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition localValDef = JsonTypeDefinition.fromJson(localValueSerializerDef.getCurrentSchemaInfo());

                            if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
                                String compressionPolicy = "";
                                if(hasCompression) {
                                    compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
                                }
                                // if the key/value serializers are REALLY equal (even though the strings may not match), then
                                // just use the remote stores to GUARANTEE that they match, and try again.
                                newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
                                                                               replicationFactor,
                                                                               requiredReads,
                                                                               requiredWrites,
                                                                               props.containsKey("build.preferred.reads") ? props.getInt("build.preferred.reads")
                                                                                                                         : null,
                                                                               props.containsKey("build.preferred.writes") ? props.getInt("build.preferred.writes")
                                                                                                                          : null,
                                                                               "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                                                                       + remoteKeySerializerDef.getCurrentSchemaInfo()
                                                                                       + "</schema-info>\n\t",
                                                                               "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                                                                       + remoteValueSerializerDef.getCurrentSchemaInfo()
                                                                                       + "</schema-info>"
                                                                                       + compressionPolicy
                                                                                       + "\n\t");

                                newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
View Full Code Here

                if(remoteStoreDef.getName().equals(storeName)) {
                    // if the store already exists, but doesn't match what we want to push, we need to worry
                    if(!remoteStoreDef.equals(newStoreDef)) {
                        // let's check to see if the key/value serializers are
                        // REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
                        if(remoteKeySerializerDef.getName().equals(serializerName)
                           && remoteValueSerializerDef.getName().equals(serializerName)) {

                            Schema remoteKeyDef = Schema.parse(remoteKeySerializerDef.getCurrentSchemaInfo());
                            Schema remoteValDef = Schema.parse(remoteValueSerializerDef.getCurrentSchemaInfo());
                            Schema localKeyDef = Schema.parse(localKeySerializerDef.getCurrentSchemaInfo());
                            Schema localValDef = Schema.parse(localValueSerializerDef.getCurrentSchemaInfo());

                            if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {

                                String compressionPolicy = "";
                                if(hasCompression) {
                                    compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
                                }

                                // if the key/value serializers are REALLY equal
                                // (even though the strings may not match), then
                                // just use the remote stores to GUARANTEE that
                                // they
                                // match, and try again.

                                String keySerializerStr = "\n\t\t<type>"
                                                          + remoteKeySerializerDef.getName()
                                                          + "</type>";

                                if(remoteKeySerializerDef.hasVersion()) {

                                    Map<Integer, String> versions = new HashMap<Integer, String>();
                                    for(Map.Entry<Integer, String> entry: remoteKeySerializerDef.getAllSchemaInfoVersions()
                                                                                                .entrySet()) {
                                        keySerializerStr += "\n\t\t <schema-info version=\""
                                                            + entry.getKey() + "\">"
                                                            + entry.getValue()
                                                            + "</schema-info>\n\t";
                                    }

                                } else {
                                    keySerializerStr = "\n\t\t<type>"
                                                       + serializerName
                                                       + "</type>\n\t\t<schema-info version=\"0\">"
                                                       + remoteKeySerializerDef.getCurrentSchemaInfo()
                                                       + "</schema-info>\n\t";
                                }

                                schemaObj.keySchema = keySerializerStr;
                                String valueSerializerStr = "\n\t\t<type>"
                                                            + remoteValueSerializerDef.getName()
                                                            + "</type>";

                                if(remoteValueSerializerDef.hasVersion()) {

                                    Map<Integer, String> versions = new HashMap<Integer, String>();
                                    for(Map.Entry<Integer, String> entry: remoteValueSerializerDef.getAllSchemaInfoVersions()
                                                                                                  .entrySet()) {
                                        valueSerializerStr += "\n\t\t <schema-info version=\""
                                                              + entry.getKey() + "\">"
                                                              + entry.getValue()
                                                              + "</schema-info>\n\t";
                                    }
                                    valueSerializerStr += compressionPolicy + "\n\t";

                                } else {

                                    valueSerializerStr = "\n\t\t<type>"
                                                         + serializerName
                                                         + "</type>\n\t\t<schema-info version=\"0\">"
                                                         + remoteValueSerializerDef.getCurrentSchemaInfo()
                                                         + "</schema-info>" + compressionPolicy
                                                         + "\n\t";

                                }
                                schemaObj.valSchema = valueSerializerStr;
View Full Code Here

                }

                out.write("STORE_NAME: " + storeDefinition.getName() + "\n");

                // k-v serializer
                final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
                final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
                SerializerFactory serializerFactory = new DefaultSerializerFactory();
                @SuppressWarnings("unchecked")
                final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
                @SuppressWarnings("unchecked")
                final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);

                // compression strategy
                final CompressionStrategy keyCompressionStrategy;
                final CompressionStrategy valueCompressionStrategy;
                if(keySerializerDef != null && keySerializerDef.hasCompression()) {
                    keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
                } else {
                    keyCompressionStrategy = null;
                }
                if(valueSerializerDef != null && valueSerializerDef.hasCompression()) {
                    valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
                } else {
                    valueCompressionStrategy = null;
                }

                if(keyCompressionStrategy == null) {
                    out.write("KEY_COMPRESSION_STRATEGY: None\n");
                } else {
                    out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType()
                              + "\n");
                }
                out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n");
                for(Map.Entry<Integer, String> entry: keySerializerDef.getAllSchemaInfoVersions()
                                                                      .entrySet()) {
                    out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey()));
                    out.write("====================================\n");
                    out.write(entry.getValue());
                    out.write("\n====================================\n");
                }
                out.write("\n");
                if(valueCompressionStrategy == null) {
                    out.write("VALUE_COMPRESSION_STRATEGY: None\n");
                } else {
                    out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType()
                              + "\n");
                }
                out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n");
                for(Map.Entry<Integer, String> entry: valueSerializerDef.getAllSchemaInfoVersions()
                                                                        .entrySet()) {
                    out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey()));
                    out.write("====================================\n");
                    out.write(entry.getValue());
                    out.write("\n====================================\n");
View Full Code Here

     *
     * @param storeDefinition the store definition to check on
     */
    public static void validateSchemaAsNeeded(StoreDefinition storeDefinition) {
        logger.info("Validating schema for store:  " + storeDefinition.getName());
        SerializerDefinition keySerDef = storeDefinition.getKeySerializer();
        // validate the key schemas
        try {
            validateIfAvroSchema(keySerDef);
        } catch(Exception e) {
            logger.error("Validating key schema failed for store:  " + storeDefinition.getName());
            throw new VoldemortException("Error validating key schema for store:  "
                                         + storeDefinition.getName() + " " + e.getMessage(), e);
        }

        // validate the value schemas
        SerializerDefinition valueSerDef = storeDefinition.getValueSerializer();
        try {
            validateIfAvroSchema(valueSerDef);
        } catch(Exception e) {
            logger.error("Validating value schema failed for store:  " + storeDefinition.getName());
            throw new VoldemortException("Error validating value schema for store:  "
View Full Code Here

TOP

Related Classes of voldemort.serialization.SerializerDefinition

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.