Examples of voldemort.routing.StoreRoutingPlan

voldemort.routing.StoreRoutingPlan
This class wraps up a Cluster object and a StoreDefinition. The methods are effectively helper or util style methods for querying the routing plan that will be generated for a given routing strategy upon store and cluster topology information. This object can be expensive to construct. Invocations of getters should be O(1) if possible. Lazy initialization of cached values is ideal... The intermingling of key-based interfaces and partition-based interfaces is ugly in this class. Partition-based interfaces should be in an underlying class and then key-based interfaces should wrap those up. Or, all of these types of class should implement the getMasterPartition(byte[]) interface from RoutingStrategy. Then the caller always does translation from key to partition ID, and the *StoreRouting* classes only offer partition-based interfaces.

     */
    private List<RebalanceTaskInfo> constructBatchPlan() {
        // Construct all store routing plans once.
        HashMap<String, StoreRoutingPlan> currentStoreRoutingPlans = new HashMap<String, StoreRoutingPlan>();
        for(StoreDefinition storeDef: currentStoreDefs) {
            currentStoreRoutingPlans.put(storeDef.getName(), new StoreRoutingPlan(currentCluster,
                                                                                  storeDef));
        }
        HashMap<String, StoreRoutingPlan> finalStoreRoutingPlans = new HashMap<String, StoreRoutingPlan>();
        for(StoreDefinition storeDef: finalStoreDefs) {
            finalStoreRoutingPlans.put(storeDef.getName(), new StoreRoutingPlan(finalCluster,
                                                                                storeDef));
        }


        RebalanceTaskInfoBuilder rpiBuilder = new RebalanceTaskInfoBuilder();
        // For every node in the final cluster ...
        for(Node stealerNode: finalCluster.getNodes()) {
            int stealerZoneId = stealerNode.getZoneId();
            int stealerNodeId = stealerNode.getId();


            // Consider all store definitions ...
            for(StoreDefinition storeDef: finalStoreDefs) {
                StoreRoutingPlan currentSRP = currentStoreRoutingPlans.get(storeDef.getName());
                StoreRoutingPlan finalSRP = finalStoreRoutingPlans.get(storeDef.getName());
                for(int stealerPartitionId: finalSRP.getZoneNAryPartitionIds(stealerNodeId)) {
                    // ... and all nary partition-stores,
                    // now steal what is needed


                    // Optimization for RW stores: Do not steal a partition-store you already host!
                    if (!storeDef.getType().equalsIgnoreCase(ReadOnlyStorageConfiguration.TYPE_NAME)) {

View Full Code Here

                
                for (RebalanceTaskInfo currentPlan: plans) {
                    MetadataStore metadataStore = getServer(currentPlan.getStealerId()).getMetadataStore();
                    int nodeId = metadataStore.getNodeId();
                    int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();
                    StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(),
                                                                             storeDef);
                    File currentDir = new File(((ReadOnlyStorageEngine) getStore(currentPlan.getStealerId(),
                                                                                 storeName)).getCurrentDirPath());
                    if (currentPlan.getPartitionStores().contains(storeDef.getName())) {
                        for (Integer partitionId: currentPlan.getStoreToPartitionIds().get(storeName)) {
                            int zoneNary = -1;
                            // If computing zoneNary for a partition throws an exception
                            // it means we don't want to consider that partition.
                            try {
                                zoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId,
                                                                                         nodeId,
                                                                                         partitionId);
                            } catch (VoldemortException ve) {
                                continue;
                            }

View Full Code Here

            Integer partitionId = partitionIterator.next();
            int nodeId = metadataStore.getNodeId();
            int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();


            StoreDefinition storeDef = metadataStore.getStoreDef(request.getStoreName());
            StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(),
                                                                     storeDef);
            int getZoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId,
                                                                            nodeId,
                                                                            partitionId);


            currentPair = Pair.create(getZoneNary, partitionId);
            currentChunkId = 0;

View Full Code Here

            Cluster cluster = adminClient.getAdminClientCluster();
            List<StoreDefinition> storeDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList()
                                                                         .getValue();
            StoreDefinition storeDef = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs,
                                                                                       storeName);
            StoreRoutingPlan routingPlan = new StoreRoutingPlan(cluster, storeDef);
            BaseStoreRoutingPlan bRoutingPlan = new BaseStoreRoutingPlan(cluster, storeDef);


            final int COLUMN_WIDTH = 30;


            for(String keyStr: keyStrings) {
                byte[] key = ByteUtils.fromHexString(keyStr);
                System.out.println("Key :" + keyStr);
                System.out.println("Replicating Partitions :"
                                   + routingPlan.getReplicatingPartitionList(key));
                System.out.println("Replicating Nodes :");
                List<Integer> nodeList = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key));
                for(int i = 0; i < nodeList.size(); i++) {
                    System.out.println(nodeList.get(i) + "\t"
                                       + cluster.getNodeById(nodeList.get(i)).getHost());
                }


                System.out.println("Zone Nary information :");
                HashMap<Integer, Integer> zoneRepMap = storeDef.getZoneReplicationFactor();


                for(Zone zone: cluster.getZones()) {
                    System.out.println("\tZone #" + zone.getId());
                    int numReplicas = -1;
                    if(zoneRepMap == null) {
                        // non zoned cluster
                        numReplicas = storeDef.getReplicationFactor();
                    } else {
                        // zoned cluster
                        if(!zoneRepMap.containsKey(zone.getId())) {
                            Utils.croak("Repfactor for Zone " + zone.getId()
                                        + " not found in storedef");
                        }
                        numReplicas = zoneRepMap.get(zone.getId());
                    }


                    String FormatString = "%s %s %s\n";
                    System.out.format(FormatString,
                                      Utils.paddedString("REPLICA#", COLUMN_WIDTH),
                                      Utils.paddedString("PARTITION", COLUMN_WIDTH),
                                      Utils.paddedString("NODE", COLUMN_WIDTH));
                    for(int i = 0; i < numReplicas; i++) {
                        Integer nodeId = bRoutingPlan.getNodeIdForZoneNary(zone.getId(), i, key);
                        Integer partitionId = routingPlan.getNodesPartitionIdForKey(nodeId, key);
                        System.out.format(FormatString,
                                          Utils.paddedString(i + "", COLUMN_WIDTH),
                                          Utils.paddedString(partitionId.toString(), COLUMN_WIDTH),
                                          Utils.paddedString(nodeId
                                                                     + "("

View Full Code Here

    @Override
    public void operate() throws Exception {
        for(StoreDefinition storeDef: metadataStore.getStoreDefList()) {
            if(isWritableStore(storeDef)) {
                // Lets generate routing strategy for this storage engine
                StoreRoutingPlan routingPlan = new StoreRoutingPlan(metadataStore.getCluster(),
                                                                    storeDef);
                logger.info("Repairing store " + storeDef.getName());
                StorageEngine<ByteArray, byte[], byte[]> engine = storeRepo.getStorageEngine(storeDef.getName());
                iterator = engine.keys();


                long itemsScanned = 0;
                long numDeletedKeys = 0;
                while(iterator.hasNext()) {
                    ByteArray key = iterator.next();


                    if(!routingPlan.checkKeyBelongsToNode(key.get(), metadataStore.getNodeId())) {
                        /**
                         * Blow away the entire key with all its versions..
                         */
                        engine.delete(key, null);
                        numDeletedKeys = this.numKeysUpdatedThisRun.incrementAndGet();

View Full Code Here

                        return null;
                    }
                }, true);


                final CountDownLatch latch = new CountDownLatch(srcCluster.getNumberOfPartitions());
                StoreRoutingPlan storeInstance = new StoreRoutingPlan(srcCluster,
                                                                      srcStoreDefMap.get(store));


                // submit work on every partition that is to be forklifted
                for(Integer partitionId: partitionList) {
                    if(this.mode == ForkLiftTaskMode.global_resolution) {

View Full Code Here

        List<StoreDefinition> storeDefs = storeDefinitions.getValue();
        StoreDefinition storeDefinition = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs,
                                                                                          storeName);
        logger.info("Store definition for store " + storeName + " has been determined.");


        storeInstance = new StoreRoutingPlan(cluster, storeDefinition);


        stats = new Stats(progressBar);


        this.perServerQPSLimit = perServerQPSLimit;
        this.putThrottlers = new ConcurrentHashMap<Integer, EventThrottler>();

View Full Code Here

        // Constructing a StoreRoutingPlan has the (desirable in this
        // case) side-effect of verifying that the store definition is congruent
        // with the cluster definition. If there are issues, exceptions are
        // thrown.
        for(StoreDefinition storeDefinition: storeDefs) {
            new StoreRoutingPlan(cluster, storeDefinition);
        }
        return;
    }

View Full Code Here

            throw new VoldemortException("Unknown store " + storeName);
        }


        if(isWritableStore(storeDef)) {
            // Lets generate routing strategy for this storage engine
            StoreRoutingPlan routingPlan = new StoreRoutingPlan(metadataStore.getCluster(),
                                                                storeDef);
            logger.info("Pruning store " + storeDef.getName());
            StorageEngine<ByteArray, byte[], byte[]> engine = storeRepo.getStorageEngine(storeDef.getName());
            iterator = engine.keys();


            long itemsScanned = 0;
            long numPrunedKeys = 0;
            while(iterator.hasNext()) {
                ByteArray key = iterator.next();


                KeyLockHandle<byte[]> lockHandle = null;
                try {
                    lockHandle = engine.getAndLock(key);
                    List<Versioned<byte[]>> vals = lockHandle.getValues();
                    List<Integer> keyReplicas = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key.get()));
                    MutableBoolean didPrune = new MutableBoolean(false);
                    List<Versioned<byte[]>> prunedVals = pruneNonReplicaEntries(vals,
                                                                                keyReplicas,
                                                                                didPrune);
                    // Only write something back if some pruning actually

View Full Code Here

        for(Integer zoneId: zoneIds) {
            zoneIdToPartitionStoreCount.put(zoneId, 0);
        }


        for(StoreDefinition storeDefinition: uniqueStores.keySet()) {
            StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(cluster, storeDefinition);


            // High level information about the store def exemplar
            builder.append(Utils.NEWLINE)
                   .append("Store exemplar: " + storeDefinition.getName())
                   .append(Utils.NEWLINE)

View Full Code Here

0 1

TOP

Related Classes of voldemort.routing.StoreRoutingPlan

voldemort.client.rebalance.AdminRebalanceTest

voldemort.client.rebalance.RebalanceBatchPlan

voldemort.cluster.Node

voldemort.performance.StoreRoutingPlanPerf

voldemort.server.protocol.admin.FetchPartitionFileStreamRequestHandler

voldemort.server.protocol.admin.FetchStreamRequestHandler

voldemort.server.storage.prunejob.VersionedPutPruneJob

voldemort.server.storage.repairjob.RepairJob

voldemort.server.storage.VersionedPutPruningTest

voldemort.tools.admin.command.AdminCommandDebug$SubCommandDebugRoute

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.