Package voldemort.client.protocol.admin

Examples of voldemort.client.protocol.admin.AdminClient$AdminStoreClient


                ioe.printStackTrace();
            }
        }

        // Get the new cluster.xml
        AdminClient localAdminClient = getAdminClient(newCluster);

        Versioned<String> versionedClusterXML = localAdminClient.metadataMgmtOps.getRemoteMetadata(3,
                                                                                                   MetadataStore.CLUSTER_KEY);

        // Increment the version, let what would be the "donor node" know about
View Full Code Here


    private void allClustersEqual(final List<String> clusterUrls) {
        Validate.notEmpty(clusterUrls, "Clusterurls cannot be null");
        // If only one clusterUrl return immediately
        if (clusterUrls.size() == 1)
            return;
        AdminClient adminClientLhs = new AdminClient(clusterUrls.get(0),
                                                     new AdminClientConfig(),
                                                     new ClientConfig());
        Cluster clusterLhs = adminClientLhs.getAdminClientCluster();
        for (int index = 1; index < clusterUrls.size(); index++) {
            AdminClient adminClientRhs = new AdminClient(clusterUrls.get(index),
                                                         new AdminClientConfig(),
                                                         new ClientConfig());
            Cluster clusterRhs = adminClientRhs.getAdminClientCluster();
            if (!areTwoClustersEqual(clusterLhs, clusterRhs))
                throw new VoldemortException("Cluster " + clusterLhs.getName()
                                             + "is not the same as " + clusterRhs.getName());
        }
    }
View Full Code Here

            catch(RuntimeException e) {
                log.error("Getting store definition from: " + url + " (node id " + this.nodeId + ")", e);
                System.exit(-1);
            }
        }
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        // don't use newStoreDef because we want to ALWAYS use the JSON definition since the store
        // builder assumes that you are using JsonTypeSerializer. This allows you to tweak your
        // value/key store xml  as you see fit, but still uses the json sequence file meta data
        // to  build the store.
        storeDefs = ImmutableList.of(VoldemortUtils.getStoreDef(VoldemortUtils.getStoreDefXml(storeName,
                                                                                              replicationFactor,
                                                                                              requiredReads,
                                                                                              requiredWrites,
                                                                                              props.containsKey("build.preferred.reads") ? props.getInt("build.preferred.reads")
                                                                                                                                        : null,
                                                                                              props.containsKey("build.preferred.writes") ? props.getInt("build.preferred.writes")
                                                                                                                                         : null,
                                                                                              keySchema,
                                                                                              valSchema)));
        cluster = adminClient.getAdminClientCluster();
        adminClient.close();
    }
View Full Code Here

                                  int requiredReads,
                                  int requiredWrites) {
        log.info("Verifying store: \n" + newStoreDefXml.toString());
        StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
        log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        try {
            List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
                                                                               .getValue();
            boolean foundStore = false;
            // go over all store defs and see if one has the same name as the store we're trying to build
            for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
                if(remoteStoreDef.getName().equals(storeName)) {
                    // if the store already exists, but doesn't match what we want to push, we need to worry
                    if(!remoteStoreDef.equals(newStoreDef)) {
                        // it is possible that the stores actually DO match, but the json in the key/value
                        // serializers is out of order (eg {'a': 'int32', 'b': 'int32'}  could have a/b reversed.
                        // This is just a reflection of the fact that voldemort json type defs use hashmaps that
                        // are unordered, and pig uses bags that are unordered  as well. it's therefore unpredictable
                        // what order the keys will come out of pig. let's check to see if the key/value
                        // serializers are REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();

                        if(remoteKeySerializerDef.getName().equals("json")
                           && remoteValueSerializerDef.getName().equals("json")
                           && remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
                           && remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
                            JsonTypeDefinition remoteKeyDef = JsonTypeDefinition.fromJson(remoteKeySerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition remoteValDef = JsonTypeDefinition.fromJson(remoteValueSerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition localKeyDef = JsonTypeDefinition.fromJson(localKeySerializerDef.getCurrentSchemaInfo());
                            JsonTypeDefinition localValDef = JsonTypeDefinition.fromJson(localValueSerializerDef.getCurrentSchemaInfo());

                            if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
                                String compressionPolicy = "";
                                if(hasCompression) {
                                    compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
                                }
                                // if the key/value serializers are REALLY equal (even though the strings may not match), then
                                // just use the remote stores to GUARANTEE that they match, and try again.
                                newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
                                                                               replicationFactor,
                                                                               requiredReads,
                                                                               requiredWrites,
                                                                               props.containsKey("build.preferred.reads") ? props.getInt("build.preferred.reads")
                                                                                                                         : null,
                                                                               props.containsKey("build.preferred.writes") ? props.getInt("build.preferred.writes")
                                                                                                                          : null,
                                                                               "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                                                                       + remoteKeySerializerDef.getCurrentSchemaInfo()
                                                                                       + "</schema-info>\n\t",
                                                                               "\n\t\t<type>json</type>\n\t\t<schema-info version=\"0\">"
                                                                                       + remoteValueSerializerDef.getCurrentSchemaInfo()
                                                                                       + "</schema-info>"
                                                                                       + compressionPolicy
                                                                                       + "\n\t");

                                newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
                                if(!remoteStoreDef.equals(newStoreDef)) {
                                    // if we still get a fail, then we know that the store defs don't match for reasons
                                    // OTHER than the key/value serializer
                                    throw new RuntimeException("Your store schema is identical, but the store definition does not match. Have: "
                                                               + newStoreDef + "\nBut expected: " + remoteStoreDef);
                                }
                            } else {
                                // if the key/value serializers are not equal (even in java, not just json strings),
                                // then fail
                                throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
                                                           + newStoreDef + "\nBut expected: " + remoteStoreDef);
                            }
                        } else {
                            throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Have: "
                                                       + newStoreDef + "\nBut expected: " + remoteStoreDef);
                        }
                    }
                    foundStore = true;
                    break;
                }
            }
         return foundStore;
        } finally {
            adminClient.close();
        }
    }
View Full Code Here

        // Generate random data, populate cluster with it.
        HashMap<String, String> testEntries = ServerTestUtils.createRandomKeyValueString(128);
        populateData(testEntries);

        // create admin client and run repair on all nodes
        AdminClient admin = new AdminClient(cluster, new AdminClientConfig(), new ClientConfig());
        for(int i = 0; i < 9; i++) {
            admin.storeMntOps.repairJob(i);
        }

        // wait for the repair to complete
View Full Code Here

            throw new RuntimeException("Owner field missing in store definition. "
                                       + "Please add \"push.store.owners\" with value being comma-separated list of LinkedIn email ids");

        }
        log.info("Could not find store " + storeName + " on Voldemort. Adding it to all nodes ");
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        try {
            adminClient.storeMgmtOps.addStore(newStoreDef);
        }
        catch(VoldemortException ve) {
            throw new RuntimeException("Exception during adding store" + ve.getMessage());
        }
        finally {
            adminClient.close();
        }
    }
View Full Code Here

        Properties props = new Properties();
        props.put("enable.quota.limiting", "true");
        server = ServerTestUtils.startStandAloneVoldemortServer(props,
                                                                "test/common/voldemort/config/single-store.xml");

        adminClient = new AdminClient(server.getMetadataStore().getCluster(),
                                      new AdminClientConfig(),
                                      new ClientConfig());
        String bootStrapUrl = "tcp://" + server.getIdentityNode().getHost() + ":"
                              + server.getIdentityNode().getSocketPort();
        factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootStrapUrl));
View Full Code Here

            catch(RuntimeException e) {
                log.error("Error in adding store definition from: " + url, e);
                System.exit(-1);
            }
        }
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        // don't use newStoreDef because we want to ALWAYS use the JSON definition since the store
        // builder assumes that you are using JsonTypeSerializer. This allows you to tweak your
        // value/key store xml  as you see fit, but still uses the json sequence file meta data
        // to  build the store.
        storeDefs = ImmutableList.of(VoldemortUtils.getStoreDef(VoldemortUtils.getStoreDefXml(storeName,
                                                                                              replicationFactor,
                                                                                              requiredReads,
                                                                                              requiredWrites,
                                                                                              props.containsKey("build.preferred.reads") ? props.getInt("build.preferred.reads")
                                                                                                                                        : null,
                                                                                              props.containsKey("build.preferred.writes") ? props.getInt("build.preferred.writes")
                                                                                                                                         : null,
                                                                                              returnSchemaObj.keySchema,
                                                                                              returnSchemaObj.valSchema)));
        cluster = adminClient.getAdminClientCluster();
        adminClient.close();
    }
View Full Code Here

                                      KeyValueSchema schemaObj) {
        log.info("Verifying store: \n" + newStoreDefXml.toString());
        StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
        // get store def from cluster
        log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
        AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
        try {
            List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
                                                                               .getValue();
            boolean foundStore = false;
            // go over all store defs and see if one has the same name as the store we're trying to build
            for(StoreDefinition remoteStoreDef: remoteStoreDefs) {
                if(remoteStoreDef.getName().equals(storeName)) {
                    // if the store already exists, but doesn't match what we want to push, we need to worry
                    if(!remoteStoreDef.equals(newStoreDef)) {
                        // let's check to see if the key/value serializers are
                        // REALLY equal.
                        SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
                        SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
                        SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
                        SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
                        if(remoteKeySerializerDef.getName().equals(serializerName)
                           && remoteValueSerializerDef.getName().equals(serializerName)) {

                            Schema remoteKeyDef = Schema.parse(remoteKeySerializerDef.getCurrentSchemaInfo());
                            Schema remoteValDef = Schema.parse(remoteValueSerializerDef.getCurrentSchemaInfo());
                            Schema localKeyDef = Schema.parse(localKeySerializerDef.getCurrentSchemaInfo());
                            Schema localValDef = Schema.parse(localValueSerializerDef.getCurrentSchemaInfo());

                            if(remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {

                                String compressionPolicy = "";
                                if(hasCompression) {
                                    compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
                                }

                                // if the key/value serializers are REALLY equal
                                // (even though the strings may not match), then
                                // just use the remote stores to GUARANTEE that
                                // they
                                // match, and try again.

                                String keySerializerStr = "\n\t\t<type>"
                                                          + remoteKeySerializerDef.getName()
                                                          + "</type>";

                                if(remoteKeySerializerDef.hasVersion()) {

                                    Map<Integer, String> versions = new HashMap<Integer, String>();
                                    for(Map.Entry<Integer, String> entry: remoteKeySerializerDef.getAllSchemaInfoVersions()
                                                                                                .entrySet()) {
                                        keySerializerStr += "\n\t\t <schema-info version=\""
                                                            + entry.getKey() + "\">"
                                                            + entry.getValue()
                                                            + "</schema-info>\n\t";
                                    }

                                } else {
                                    keySerializerStr = "\n\t\t<type>"
                                                       + serializerName
                                                       + "</type>\n\t\t<schema-info version=\"0\">"
                                                       + remoteKeySerializerDef.getCurrentSchemaInfo()
                                                       + "</schema-info>\n\t";
                                }

                                schemaObj.keySchema = keySerializerStr;
                                String valueSerializerStr = "\n\t\t<type>"
                                                            + remoteValueSerializerDef.getName()
                                                            + "</type>";

                                if(remoteValueSerializerDef.hasVersion()) {

                                    Map<Integer, String> versions = new HashMap<Integer, String>();
                                    for(Map.Entry<Integer, String> entry: remoteValueSerializerDef.getAllSchemaInfoVersions()
                                                                                                  .entrySet()) {
                                        valueSerializerStr += "\n\t\t <schema-info version=\""
                                                              + entry.getKey() + "\">"
                                                              + entry.getValue()
                                                              + "</schema-info>\n\t";
                                    }
                                    valueSerializerStr += compressionPolicy + "\n\t";

                                } else {

                                    valueSerializerStr = "\n\t\t<type>"
                                                         + serializerName
                                                         + "</type>\n\t\t<schema-info version=\"0\">"
                                                         + remoteValueSerializerDef.getCurrentSchemaInfo()
                                                         + "</schema-info>" + compressionPolicy
                                                         + "\n\t";

                                }
                                schemaObj.valSchema = valueSerializerStr;

                                newStoreDefXml = VoldemortUtils.getStoreDefXml(storeName,
                                                                               replicationFactor,
                                                                               requiredReads,
                                                                               requiredWrites,
                                                                               props.containsKey("build.preferred.reads") ? props.getInt("build.preferred.reads")
                                                                                                                         : null,
                                                                               props.containsKey("build.preferred.writes") ? props.getInt("build.preferred.writes")
                                                                                                                          : null,
                                                                               keySerializerStr,
                                                                               valueSerializerStr);

                                newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);

                                if(!remoteStoreDef.equals(newStoreDef)) {
                                    // if we still get a fail, then we know that the store defs don't match for reasons
                                    // OTHER than the key/value serializer
                                    throw new RuntimeException("Your store schema is identical, but the store definition does not match. Have: "
                                                               + newStoreDef
                                                               + "\nBut expected: "
                                                               + remoteStoreDef);
                                }

                            } else {
                                // if the key/value serializers are not equal (even in java, not just json strings),
                                // then fail
                                throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
                                                           + newStoreDef
                                                           + "\nBut expected: "
                                                           + remoteStoreDef);
                            }
                        } else {
                            throw new RuntimeException("Your store definition does not match the store definition that is already in the cluster. Have: "
                                                       + newStoreDef
                                                       + "\nBut expected: "
                                                       + remoteStoreDef);
                        }
                    }
                    foundStore = true;
                    break;
                }
            }
            return foundStore;
        } finally {
            adminClient.close();
        }
    }
View Full Code Here

            }
            recordCount++;
        }

        // run the prune job
        AdminClient admin = new AdminClient(cluster, new AdminClientConfig(), new ClientConfig());
        for(int nodeid = 0; nodeid < servers.length; nodeid++) {
            admin.storeMntOps.pruneJob(nodeid, "test");
        }
        for(VoldemortServer server: servers) {
            ServerTestUtils.waitForAsyncOperationOnServer(server, "Prune", 5000);
View Full Code Here

TOP

Related Classes of voldemort.client.protocol.admin.AdminClient$AdminStoreClient

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.