Package voldemort.server.scheduler.slop

Examples of voldemort.server.scheduler.slop.StreamingSlopPusherJob


                                      routedStoreFactory.toNonblockingStore(slopStorageEngine));
            slopStores.put(nodeId, storageEngine);

            MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster,
                                                                              Lists.newArrayList(storeDef));
            StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(storeRepo,
                                                                       metadataStore,
                                                                       failureDetector,
                                                                       ServerTestUtils.createServerConfigWithDefs(false,
                                                                                                                  nodeId,
                                                                                                                  TestUtils.createTempDir()
View Full Code Here


                                                                                  "test-consistent",
                                                                                  "test-consistent-with-pref-list");

        populateSlops(0, slopStoreNode0, entrySet1, entrySet2);

        StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(),
                                                                   getVoldemortServer(0).getMetadataStore(),
                                                                   new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                               .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                         metadataStore,
                                                                                                                                                                         configs[0]))),
                                                                   configs[0],
                                                                   new ScanPermitWrapper(1));

        pusher.run();

        // Give some time for the slops to go over
        Thread.sleep(2000);

        // Now check if the slops went through and also got deleted
View Full Code Here

        // Generate two sub lists to send in batches
        List<Versioned<Slop>> firstSet = entrySet.subList(0, 7), secondSet = entrySet.subList(7, 15);
        populateSlops(0, slopStoreNode0, firstSet);

        StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(),
                                                                   getVoldemortServer(0).getMetadataStore(),
                                                                   new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                               .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                         metadataStore,
                                                                                                                                                                         configs[0]))),
                                                                   configs[0],
                                                                   new ScanPermitWrapper(1));

        pusher.run();

        populateSlops(0, slopStoreNode0, secondSet);

        pusher.run();

        // Give some time for the slops to go over
        Thread.sleep(2000);

        for(String storeName: Lists.newArrayList("test-replication-memory",
View Full Code Here

                                                                                 "test-consistent",
                                                                                 "test-consistent-with-pref-list");

        populateSlops(0, slopStoreNode0, entrySet);

        StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(),
                                                                   getVoldemortServer(0).getMetadataStore(),
                                                                   new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                               .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                         metadataStore,
                                                                                                                                                                         configs[0]))),
                                                                   configs[0],
                                                                   new ScanPermitWrapper(1));

        pusher.run();

        // Give some time for the slops to go over
        Thread.sleep(2000);

        // Now check if the slops went through and also got deleted
View Full Code Here

        // Populated the slop stores
        populateSlops(0, slopStoreNode0, entrySetNode0);
        populateSlops(1, slopStoreNode1, entrySetNode1);

        StreamingSlopPusherJob pusher0 = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(),
                                                                    getVoldemortServer(0).getMetadataStore(),
                                                                    new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                                .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                          metadataStore,
                                                                                                                                                                          configs[0]))),
                                                                    configs[0],
                                                                    new ScanPermitWrapper(1)), pusher1 = new StreamingSlopPusherJob(getVoldemortServer(1).getStoreRepository(),
                                                                                                                                    getVoldemortServer(1).getMetadataStore(),
                                                                                                                                    new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                                                                                                .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                                                                                          metadataStore,
                                                                                                                                                                                                                                          configs[1]))),
                                                                                                                                    configs[1],
                                                                                                                                    new ScanPermitWrapper(1));

        pusher0.run();
        pusher1.run();

        // Give some time for the slops to go over
        Thread.sleep(2000);

        // Now check if the slops worked
View Full Code Here

                                                                                  "test-consistent",
                                                                                  "test-consistent-with-pref-list");

        populateSlops(0, slopStoreNode0, entrySet1, entrySet2);

        StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(),
                                                                   getVoldemortServer(0).getMetadataStore(),
                                                                   new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster)
                                                                                                                               .setStoreVerifier(new ServerStoreVerifier(socketStoreFactory,
                                                                                                                                                                         metadataStore,
                                                                                                                                                                         configs[0]))),
                                                                   configs[0],
                                                                   new ScanPermitWrapper(1));

        pusher.run();

        // Give some time for the slops to go over
        Thread.sleep(10000);

        // Now check if the slops went through and also got deleted
        Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
        while(entryIterator.hasNext()) {
            Versioned<Slop> versionedSlop = entryIterator.next();
            Slop nextSlop = versionedSlop.getValue();
            StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository()
                                                                                  .getStorageEngine(nextSlop.getStoreName());
            if(nextSlop.getOperation().equals(Slop.Operation.PUT)) {
                assertNotSame("entry should be present at store",
                              0,
                              store.get(nextSlop.getKey(), null).size());
                assertEquals("entry value should match",
                             new String(nextSlop.getValue()),
                             new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
            } else if(nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
                assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null)
                                                                 .size());
            }

            // did it get deleted correctly
            assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null)
                                                                   .size());
        }

        entryIterator = entrySet1.listIterator();
        while(entryIterator.hasNext()) {
            Versioned<Slop> versionedSlop = entryIterator.next();
            Slop nextSlop = versionedSlop.getValue();
            // did it get deleted correctly
            assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null)
                                                                   .size());
        }

        // Check counts
        SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
        assertEquals(slopEngine.getOutstandingTotal(), 50);
        assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
        assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));

        // now replace server 1 with a new host and start it
        cluster = ServerTestUtils.updateClusterWithNewHost(cluster, 1);
        startServers(1);

        // update the meatadata store with the new cluster on node 0 and 2 (the
        // two servers that are running)
        servers[0].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
        servers[2].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);

        // Give some time for the pusher job to figure out that server1 is up
        Thread.sleep(35000);

        // start the pusher job again
        pusher.run();

        // Give some time for the slops to go over
        Thread.sleep(10000);

        // make sure the slot for server 1 is pushed to the new host
View Full Code Here

                                                   .compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0) ? new BlockingSlopPusherJob(storeRepository,
                                                                                                                                 metadata,
                                                                                                                                 failureDetector,
                                                                                                                                 voldemortConfig,
                                                                                                                                 scanPermitWrapper)
                                                                                                    : new StreamingSlopPusherJob(storeRepository,
                                                                                                                                 metadata,
                                                                                                                                 failureDetector,
                                                                                                                                 voldemortConfig,
                                                                                                                                 scanPermitWrapper),
                                   nextRun,
View Full Code Here

TOP

Related Classes of voldemort.server.scheduler.slop.StreamingSlopPusherJob

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.