Package voldemort.store

Examples of voldemort.store.StoreDefinition


                                                              description,
                                                              owners);

        log.info("Verifying store: \n" + newStoreDefXml.toString());

        StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);

        // get store def from cluster
        log.info("Getting store definition from: " + url + " ( node id " + this.nodeId + " )");

        List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
                                                                           .getValue();
        boolean foundStore = false;

        // go over all store defs and see if one has the same name as the store
        // we're trying
        // to build
        for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
            if(remoteStoreDef.getName().equals(storeName)) {
                // if the store already exists, but doesn't match what we want
                // to push, we need
                // to worry
                if(!remoteStoreDef.equals(newStoreDef)) {
                    // it is possible that the stores actually DO match, but the
                    // json in the key/value serializers is out of order (eg
                    // {'a': 'int32', 'b': 'int32'} could have a/b reversed.
                    // this is just a reflection of the fact that voldemort json
                    // type defs use hashmaps that are unordered, and pig uses
                    // bags that are unordered as well. it's therefore
                    // unpredictable what order the keys will come out of pig.
                    // let's check to see if the key/value serializers are
                    // REALLY equal.
                    SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                    SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                    SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                    SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();

                    if(remoteKeySerializerDef.getName().equals("json")
                       && remoteValueSerializerDef.getName().equals("json")
                       && remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
                       && remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
                        JsonTypeDefinition remoteKeyDef = JsonTypeDefinition.fromJson(remoteKeySerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition remoteValDef = JsonTypeDefinition.fromJson(remoteValueSerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition localKeyDef = JsonTypeDefinition.fromJson(localKeySerializerDef.getCurrentSchemaInfo());
                        JsonTypeDefinition localValDef = JsonTypeDefinition.fromJson(localValueSerializerDef.getCurrentSchemaInfo());

                        if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
                            // if the key/value serializers are REALLY equal
                            // (even though the strings may not match), then
                            // just use the remote stores to GUARANTEE that they
                            // match, and try again.
                            keySchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                        + remoteKeySerializerDef.getCurrentSchemaInfo()
                                        + "</schema-info>\n\t" + keySchemaCompression;
                            valSchema = "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                        + remoteValueSerializerDef.getCurrentSchemaInfo()
                                        + "</schema-info>\n\t" + valueSchemaCompression;
                            newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
                                                                           replicationFactor,
                                                                           requiredReads,
                                                                           requiredWrites,
                                                                           (preferredReads < 0) ? null
                                                                                               : preferredReads,
                                                                           (preferredWrites < 0) ? null
                                                                                                : preferredWrites,
                                                                           keySchema,
                                                                           valSchema,
                                                                           description,
                                                                           owners);

                            newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);

                            if(!remoteStoreDef.equals(newStoreDef)) {
                                // if we still get a fail, then we know that the
                                // store defs don't match for reasons OTHER than
                                // the key/value serializer
                                throw new RuntimeException("Your store schema is identical, but the store definition does not match. Have: "
                                                           + newStoreDef
                                                           + "\nBut expected: "
                                                           + remoteStoreDef);
                            }
                        } else {
                            // if the key/value serializers are not equal (even
                            // in java, not just json strings), then fail
                            throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
                                                       + newStoreDef
                                                       + "\nBut expected: "
                                                       + remoteStoreDef);
                        }
                    }
                }

                foundStore = true;
                break;
            }
        }

        // if the store doesn't exist yet, create it
        if(!foundStore) {
            // New requirement - Make sure the user had description and owner
            // specified
            if(description.length() == 0) {
                throw new RuntimeException("Description field missing in store definition. "
                                           + "Please add \"push.store.description\" with a line describing your store");
            }

            if(owners.length() == 0) {
                throw new RuntimeException("Owner field missing in store definition. "
                                           + "Please add \"push.store.owners\" with value being comma-separated list of LinkedIn email ids");

            }

            log.info("Could not find store " + storeName
                     + " on Voldemort. Adding it to all nodes for cluster " + url);
            adminClient.storeMgmtOps.addStore(newStoreDef);
        }

        // don't use newStoreDef because we want to ALWAYS use the JSON
        // definition since the store builder assumes that you are using
        // JsonTypeSerializer. This allows you to tweak your value/key store xml
        // as you see fit, but still uses the json sequence file meta data to
        // build the store.
        StoreDefinition storeDef = VoldemortUtils.getStoreDef(VoldemortUtils.getStoreDefXml(storeName,
                                                                                            replicationFactor,
                                                                                            requiredReads,
                                                                                            requiredWrites,
                                                                                            (preferredReads < 0) ? null
                                                                                                                : preferredReads,
View Full Code Here


        }
        this.fetchOrphaned = request.hasFetchOrphaned() && request.getFetchOrphaned();
    }

    private StoreDefinition getStoreDef(String store, MetadataStore metadataStore) {
        StoreDefinition def = null;
        if(SystemStoreConstants.isSystemStore(store)) {
            def = SystemStoreConstants.getSystemStoreDef(store);
        } else {
            def = metadataStore.getStoreDef(request.getStore());
        }
View Full Code Here

        // TODO (Sid) : Confirm if keeping metadatastore as a class property is
        // the best way to do this. metadataStore is used later in the
        // handleNextPartition() to create object plan.
        this.metadataStore = metadataStore;

        StoreDefinition storeDef = metadataStore.getStoreDef(request.getStoreName());
        boolean isReadOnly = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
        if(!isReadOnly) {
            throw new VoldemortException("Should be fetching partition files only for read-only stores");
        }

        List<Integer> partitionIds = request.getPartitionIdsList();
View Full Code Here

            // Start a new partition
            Integer partitionId = partitionIterator.next();
            int nodeId = metadataStore.getNodeId();
            int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();

            StoreDefinition storeDef = metadataStore.getStoreDef(request.getStoreName());
            StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(),
                                                                     storeDef);
            int getZoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId,
                                                                            nodeId,
                                                                            partitionId);
View Full Code Here

                                           replicationFactor,
                                           requiredReads,
                                           requiredWrites);
        if (!foundStore) {
            try {
                StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
                addStore(description, owners, url, newStoreDef);
            }
            catch(RuntimeException e) {
                log.error("Getting store definition from: " + url + " (node id " + this.nodeId + ")", e);
                System.exit(-1);
View Full Code Here

                                  boolean hasCompression,
                                  int replicationFactor,
                                  int requiredReads,
                                  int requiredWrites) {
        log.info("Verifying store: \n" + newStoreDefXml.toString());
        StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
        log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        try {
            List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
                                                                               .getValue();
            boolean foundStore = false;
            // go over all store defs and see if one has the same name as the store we're trying to build
            for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
                if(remoteStoreDef.getName().equals(storeName)) {
                    // if the store already exists, but doesn't match what we want to push, we need to worry
                    if(!remoteStoreDef.equals(newStoreDef)) {
                        // it is possible that the stores actually DO match, but the json in the key/value
                        // serializers is out of order (eg {'a': 'int32', 'b': 'int32'}  could have a/b reversed.
                        // This is just a reflection of the fact that voldemort json type defs use hashmaps that
                        // are unordered, and pig uses bags that are unordered  as well. it's therefore unpredictable
                        // what order the keys will come out of pig. let's check to see if the key/value
                        // serializers are REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();

                        if(remoteKeySerializerDef.getName().equals("json")
                           && remoteValueSerializerDef.getName().equals("json")
View Full Code Here

                                               requiredWrites,
                                               serializerName,
                                               returnSchemaObj);
        if (!foundStore) {
            try {
                StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
                addStore(description, owners, url, newStoreDef);
            }
            catch(RuntimeException e) {
                log.error("Error in adding store definition from: " + url, e);
                System.exit(-1);
View Full Code Here

                                      int requiredReads,
                                      int requiredWrites,
                                      String serializerName,
                                      KeyValueSchema schemaObj) {
        log.info("Verifying store: \n" + newStoreDefXml.toString());
        StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
        // get store def from cluster
        log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        try {
            List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
                                                                               .getValue();
            boolean foundStore = false;
            // go over all store defs and see if one has the same name as the store we're trying to build
            for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
                if(remoteStoreDef.getName().equals(storeName)) {
                    // if the store already exists, but doesn't match what we want to push, we need to worry
                    if(!remoteStoreDef.equals(newStoreDef)) {
                        // let's check to see if the key/value serializers are
                        // REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
                        if(remoteKeySerializerDef.getName().equals(serializerName)
                           && remoteValueSerializerDef.getName().equals(serializerName)) {
View Full Code Here

    private HashMap<String, String> generateKeysForMasterNode(int numKeys) throws IOException {
        // Assumes master node is 0 and generates keys that goes to master node
        HashMap<String, String> keyValuePairs = new HashMap<String, String>();
        StoreDefinitionsMapper storedDefMapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefs = storedDefMapper.readStoreList(new File(storesxml));
        StoreDefinition testStoreDef = storeDefs.get(0);
        BaseStoreRoutingPlan baseStoreRoutingPlan = new BaseStoreRoutingPlan(cluster, testStoreDef);

        /*
         * Generating simple key values pairs of the form 3:3 where key and
         * value are same but route to the Master node's partition
View Full Code Here

                                                        true,
                                                        null,
                                                        "test/common/voldemort/config/single-store-322.xml",
                                                        new Properties());
        StringReader reader = new StringReader(VoldemortTestConstants.getSingleStore322Xml());
        StoreDefinition storeDef = new StoreDefinitionsMapper().readStoreList(reader).get(0);

        currentRoutingPlan = new BaseStoreRoutingPlan(cluster, storeDef);

        String bootStrapUrl = "";
        for(VoldemortServer server: servers) {
View Full Code Here

TOP

Related Classes of voldemort.store.StoreDefinition

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.