Examples of voldemort.utils.EventThrottler

voldemort.utils.EventThrottler
A class to throttle Events to a certain rate This class takes a maximum rate in events/sec and a minimum interval in ms at which to check the rate. The rate is checked every time the interval ellapses, and if the events rate exceeds the maximum, the call will block long enough to equalize it. This is generalized IoThrottler as it existed before, you can use it to throttle on Bytes read/write,number of entries scanned etc.

            // clean up will purge everything older than last 2 seconds
            Runnable cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine,
                                                                                new ScanPermitWrapper(1),
                                                                                2 * Time.MS_PER_SECOND,
                                                                                SystemTime.INSTANCE,
                                                                                new EventThrottler(1),
                                                                                null);


            // and will run every 5 seconds starting now
            scheduler.schedule("cleanup-freq-test", cleanupJob, now, 5 * Time.MS_PER_SECOND);

View Full Code Here

        // now run cleanup
        new DataCleanupJob<ByteArray, byte[], byte[]>(engine,
                                                      new ScanPermitWrapper(1),
                                                      Time.MS_PER_DAY,
                                                      time,
                                                      new EventThrottler(1),
                                                      null).run();


        // Check that all the later keys are there AND the key updated later
        assertContains("a", "d", "e", "f");
    }

View Full Code Here

                       String kerberosUser,
                       int retryCount,
                       long retryDelayMs) {
        if(maxBytesPerSecond != null) {
            this.maxBytesPerSecond = maxBytesPerSecond;
            this.throttler = new EventThrottler(this.maxBytesPerSecond);
        } else if(dynThrottleLimit != null && dynThrottleLimit.getRate() != 0) {
            this.maxBytesPerSecond = dynThrottleLimit.getRate();
            this.throttler = new DynamicEventThrottler(dynThrottleLimit);
            this.globalThrottleLimit = dynThrottleLimit;
            logger.info("Initializing Dynamic Event throttler with rate : "

View Full Code Here

                             long minBytesPerSecond,
                             int retryCount,
                             long retryDelayMs) {
        if(maxBytesPerSecond != null) {
            this.maxBytesPerSecond = maxBytesPerSecond;
            this.throttler = new EventThrottler(this.maxBytesPerSecond);
        } else if(dynThrottleLimit != null && dynThrottleLimit.getRate() != 0) {
            this.maxBytesPerSecond = dynThrottleLimit.getRate();
            this.throttler = new DynamicEventThrottler(dynThrottleLimit);
            this.globalThrottleLimit = dynThrottleLimit;
            logger.info("Initializing Dynamic Event throttler with rate : "

View Full Code Here

        super();
        this.request = request;
        this.errorCodeMapper = errorCodeMapper;
        this.storageEngine = storageEngine;
        this.metadataStore = metadataStore;
        throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
        filter = (request.hasFilter()) ? AdminServiceRequestHandler.getFilterFromRequest(request.getFilter(),
                                                                                         voldemortConfig,
                                                                                         networkClassLoader)
                                      : new DefaultVoldemortFilter();
        startTime = System.currentTimeMillis();

View Full Code Here

        } else {
            this.initialCluster = metadataStore.getCluster();
        }
        this.storeInstance = new StoreRoutingPlan(this.initialCluster, this.storeDef);


        this.throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
        if(request.hasFilter()) {
            this.filter = AdminServiceRequestHandler.getFilterFromRequest(request.getFilter(),
                                                                          voldemortConfig,
                                                                          networkClassLoader);
        } else {

View Full Code Here

        this.bucketToNumChunks = storageEngine.getChunkedFileSet().getChunkIdToNumChunks();
        this.blockSize = voldemortConfig.getAllProps()
                                        .getLong("partition.buffer.size.bytes",
                                                 voldemortConfig.getAdminSocketBufferSize());
        this.storeDir = new File(storageEngine.getCurrentDirPath());
        this.throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
        if(voldemortConfig.isJmxEnabled()) {
            this.streamStats = storeRepository.getStreamingStats(storageEngine.getName());
        } else {
            this.streamStats = null;
        }

View Full Code Here

                                                                                voldemortConfig.getClientMaxConnectionsPerNode());
                    try {
                        StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository,
                                                                                                  storeName);


                        EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());


                        if(isReadOnlyStore) {
                            ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                            String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                            logger.info("Fetching files for RO store '" + storeName
                                        + "' from node " + nodeId + " ( " + partitionIds + " )");
                            updateStatus("Fetching files for RO store '" + storeName
                                         + "' from node " + nodeId + " ( " + partitionIds + " )");


                            adminClient.readonlyOps.fetchPartitionFiles(nodeId,
                                                                        storeName,
                                                                        partitionIds,
                                                                        destinationDir,
                                                                        readOnlyStorageEngine.getChunkedFileSet()
                                                                                             .getChunkIdToNumChunks()
                                                                                             .keySet(),
                                                                        running);


                        } else {
                            logger.info("Fetching entries for RW store '" + storeName
                                        + "' from node " + nodeId + " ( " + partitionIds + " )");
                            updateStatus("Fetching entries for RW store '" + storeName
                                         + "' from node " + nodeId + " ( " + partitionIds + " ) ");


                            if(partitionIds.size() > 0) {
                                Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId,
                                                                                                                                     storeName,
                                                                                                                                     partitionIds,
                                                                                                                                     filter,
                                                                                                                                     false,
                                                                                                                                     initialCluster,
                                                                                                                                     0);
                                long numTuples = 0;
                                long startTime = System.currentTimeMillis();
                                long startNs = System.nanoTime();
                                while(running.get() && entriesIterator.hasNext()) {


                                    Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                    if(streamingStats != null) {
                                        streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES,
                                                                         Utils.elapsedTimeNs(startNs,
                                                                                             System.nanoTime()));
                                    }
                                    ByteArray key = entry.getFirst();
                                    Versioned<byte[]> value = entry.getSecond();
                                    startNs = System.nanoTime();
                                    try {
                                        /**
                                         * TODO This also needs to be fixed to
                                         * use the atomic multi version puts
                                         */
                                        storageEngine.put(key, value, null);
                                    } catch(ObsoleteVersionException e) {
                                        // log and ignore
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    } finally {
                                        if(streamingStats != null) {
                                            streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                            streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES,
                                                                             Utils.elapsedTimeNs(startNs,
                                                                                                 System.nanoTime()));
                                        }
                                    }


                                    long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                                    throttler.maybeThrottle(key.length() + valueSize(value));
                                    if((numTuples % 100000) == 0 && numTuples > 0) {
                                        logger.info(numTuples + " entries copied from node "
                                                    + nodeId + " for store '" + storeName + "'c");
                                        updateStatus(numTuples + " entries copied from node "
                                                     + nodeId + " for store '" + storeName

View Full Code Here

                                                                                      storeName);
            VoldemortFilter filter = (request.hasFilter()) ? getFilterFromRequest(request.getFilter(),
                                                                                  voldemortConfig,
                                                                                  networkClassLoader)
                                                          : new DefaultVoldemortFilter();
            EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
            iterator = storageEngine.entries();
            long deleteSuccess = 0;
            logger.info("Deleting entries for RW store " + storeName + " from node "
                        + metadataStore.getNodeId() + " ( " + storeName + " )");


            while(iterator.hasNext()) {
                Pair<ByteArray, Versioned<byte[]>> entry = iterator.next();


                ByteArray key = entry.getFirst();
                Versioned<byte[]> value = entry.getSecond();
                throttler.maybeThrottle(key.length() + valueSize(value));
                if(StoreRoutingPlan.checkKeyBelongsToNode(key.get(),
                                                          metadataStore.getNodeId(),
                                                          request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster()))
                                                                                     : metadataStore.getCluster(),
                                                          metadataStore.getStoreDef(storeName))

View Full Code Here


        logger.info("Scheduling data retention cleanup job for store '" + storeDef.getName()
                    + "' at " + startTime + " with retention scan throttle rate:" + maxReadRate
                    + " Entries/second.");


        EventThrottler throttler = new EventThrottler(maxReadRate);


        Runnable cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine,
                                                                            scanPermitWrapper,
                                                                            storeDef.getRetentionDays()
                                                                                    * Time.MS_PER_DAY,

View Full Code Here

0 1

TOP

Related Classes of voldemort.utils.EventThrottler

voldemort.client.protocol.admin.BaseStreamingClient

voldemort.scheduled.DataCleanupJobTest

voldemort.server.protocol.admin.AdminServiceRequestHandler

voldemort.server.protocol.admin.FetchPartitionFileStreamRequestHandler

voldemort.server.protocol.admin.FetchStreamRequestHandler

voldemort.server.protocol.admin.UpdatePartitionEntriesStreamRequestHandler

voldemort.server.protocol.hadoop.RestHadoopFetcher

voldemort.server.scheduler.slop.BlockingSlopPusherJob

voldemort.server.scheduler.slop.StreamingSlopPusherJob

voldemort.server.scheduler.slop.StreamingSlopPusherJob$SlopIterator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.