Examples of org.apache.bookkeeper.meta.LedgerUnderreplicationManager

org.apache.bookkeeper.meta.LedgerUnderreplicationManager
Interface for marking ledgers which need to be rereplicated

     * Test no auditing during restart if disabled
     */
    @Test(timeout=600000) // 10 minutes
    public void testAuditingDuringRollingRestart() throws Exception {
        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
        final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();


        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        for (int i = 0; i < 10; i++) {
            lh.asyncAddEntry("foobar".getBytes(), new TestCallbacks.AddCallbackFuture(), null);
        }
        lh.addEntry("foobar".getBytes());
        lh.close();


        assertEquals("shouldn't be anything under replicated",
                     underReplicationManager.pollLedgerToRereplicate(), -1);
        underReplicationManager.disableLedgerReplication();


        InetSocketAddress auditor = AuditorElector.getCurrentAuditor(baseConf, zkc);
        ServerConfiguration conf = killBookie(auditor);
        Thread.sleep(2000);
        startBookie(conf);
        Thread.sleep(2000); // give it time to run
        assertEquals("shouldn't be anything under replicated", -1,
                underReplicationManager.pollLedgerToRereplicate());
    }

View Full Code Here

     */
    @Test(timeout = 30000)
    public void testMultipleManagersShouldBeAbleToTakeAndReleaseLock()
            throws Exception {
        String missingReplica1 = "localhost:3181";
        final LedgerUnderreplicationManager m1 = lmf1
                .newLedgerUnderreplicationManager();
        final LedgerUnderreplicationManager m2 = lmf2
                .newLedgerUnderreplicationManager();
        Long ledgerA = 0xfeadeefdacL;
        m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
        final int iterationCount = 100;
        final CountDownLatch latch1 = new CountDownLatch(iterationCount);
        final CountDownLatch latch2 = new CountDownLatch(iterationCount);
        Thread thread1 = new Thread() {
            @Override
            public void run() {
                takeLedgerAndRelease(m1, latch1, iterationCount);
            }
        };


        Thread thread2 = new Thread() {
            @Override
            public void run() {
                takeLedgerAndRelease(m2, latch2, iterationCount);
            }
        };
        thread1.start();
        thread2.start();


        // wait until at least one thread completed
        while (!latch1.await(50, TimeUnit.MILLISECONDS)
                && !latch2.await(50, TimeUnit.MILLISECONDS)) {
            Thread.sleep(50);
        }


        m1.close();
        m2.close();


        // After completing 'lock acquire,release' job, it should notify below
        // wait
        latch1.await();
        latch2.await();

View Full Code Here

     * able to getLedgerToRereplicate(). This calls will enter into infinite
     * waiting until enabling rereplication process
     */
    @Test(timeout = 20000)
    public void testDisableLedegerReplication() throws Exception {
        final LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();


        // simulate few urLedgers before disabling
        final Long ledgerA = 0xfeadeefdacL;
        final String missingReplica = "localhost:3181";


        // disabling replication
        replicaMgr.disableLedgerReplication();
        LOG.info("Disabled Ledeger Replication");


        try {
            replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
        } catch (UnavailableException e) {
            LOG.debug("Unexpected exception while marking urLedger", e);
            fail("Unexpected exception while marking urLedger" + e.getMessage());
        }

View Full Code Here

     * should continue getLedgerToRereplicate() task
     */
    @Test(timeout = 20000)
    public void testEnableLedegerReplication() throws Exception {
        isLedgerReplicationDisabled = true;
        final LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();


        // simulate few urLedgers before disabling
        final Long ledgerA = 0xfeadeefdacL;
        final String missingReplica = "localhost:3181";
        try {
            replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
        } catch (UnavailableException e) {
            LOG.debug("Unexpected exception while marking urLedger", e);
            fail("Unexpected exception while marking urLedger" + e.getMessage());
        }


        // disabling replication
        replicaMgr.disableLedgerReplication();
        LOG.debug("Disabled Ledeger Replication");


        String znodeA = getUrLedgerZnode(ledgerA);
        final CountDownLatch znodeLatch = new CountDownLatch(2);
        String urledgerA = StringUtils.substringAfterLast(znodeA, "/");
        String urLockLedgerA = basePath + "/locks/" + urledgerA;
        zkc1.exists(urLockLedgerA, new Watcher(){
            @Override
            public void process(WatchedEvent event) {
                if (event.getType() == EventType.NodeCreated) {
                    znodeLatch.countDown();
                    LOG.debug("Recieved node creation event for the zNodePath:"
                            + event.getPath());
                }
                
            }});
        // getLedgerToRereplicate is waiting until enable rereplication
        Thread thread1 = new Thread() {
            @Override
            public void run() {
                try {
                    Long lA = replicaMgr.getLedgerToRereplicate();
                    assertEquals("Should be the ledger I just marked", lA,
                            ledgerA);
                    isLedgerReplicationDisabled = false;
                    znodeLatch.countDown();
                } catch (UnavailableException e) {
                    LOG.debug("Unexpected exception while marking urLedger", e);
                    isLedgerReplicationDisabled = false;
                }
            }
        };
        thread1.start();


        try {
            znodeLatch.await(5, TimeUnit.SECONDS);
            assertTrue("Ledger replication is not disabled!",
                    isLedgerReplicationDisabled);
            assertEquals("Failed to disable ledger replication!", 2, znodeLatch
                    .getCount());


            replicaMgr.enableLedgerReplication();
            znodeLatch.await(5, TimeUnit.SECONDS);
            LOG.debug("Enabled Ledeger Replication");
            assertTrue("Ledger replication is not disabled!",
                    !isLedgerReplicationDisabled);
            assertEquals("Failed to disable ledger replication!", 0, znodeLatch

View Full Code Here

     * Test that the hierarchy gets cleaned up as ledgers
     * are marked as fully replicated
     */
    @Test(timeout=60000)
    public void testHierarchyCleanup() throws Exception {
        final LedgerUnderreplicationManager replicaMgr = lmf1
            .newLedgerUnderreplicationManager();
        // 4 ledgers, 2 in the same hierarchy
        long[] ledgers = { 0x00000000deadbeefL, 0x00000000deadbeeeL,
                           0x00000000beefcafeL, 0x00000000cafed00dL };


        for (long l : ledgers) {
            replicaMgr.markLedgerUnderreplicated(l, "localhost:3181");
        }
        // can't simply test top level as we are limited to ledger
        // ids no larger than an int
        String testPath = urLedgerPath + "/0000/0000";
        List<String> children = zkc1.getChildren(testPath, false);
        assertEquals("Wrong number of hierarchies", 3, children.size());


        int marked = 0;
        while (marked < 3) {
            long l = replicaMgr.getLedgerToRereplicate();
            if (l != ledgers[0]) {
                replicaMgr.markLedgerReplicated(l);
                marked++;
            } else {
                replicaMgr.releaseUnderreplicatedLedger(l);
            }
        }
        children = zkc1.getChildren(testPath, false);
        assertEquals("Wrong number of hierarchies", 1, children.size());


        long l = replicaMgr.getLedgerToRereplicate();
        assertEquals("Got wrong ledger", ledgers[0], l);
        replicaMgr.markLedgerReplicated(l);


        children = zkc1.getChildren(urLedgerPath, false);
        assertEquals("All hierarchies should be cleaned up", 0, children.size());
    }

View Full Code Here

     * Test that as the hierarchy gets cleaned up, it doesn't interfere
     * with the marking of other ledgers as underreplicated
     */
    @Test(timeout = 90000)
    public void testHierarchyCleanupInterference() throws Exception {
        final LedgerUnderreplicationManager replicaMgr1 = lmf1
            .newLedgerUnderreplicationManager();
        final LedgerUnderreplicationManager replicaMgr2 = lmf2
            .newLedgerUnderreplicationManager();


        final int iterations = 1000;
        final AtomicBoolean threadFailed = new AtomicBoolean(false);
        Thread markUnder = new Thread() {
                public void run() {
                    long l = 1;
                    try {
                        for (int i = 0; i < iterations; i++) {
                            replicaMgr1.markLedgerUnderreplicated(l, "localhost:3181");
                            l += 10000;
                        }
                    } catch (Exception e) {
                        LOG.error("markUnder Thread failed with exception", e);
                        threadFailed.set(true);
                        return;
                    }
                }
            };
        final AtomicInteger processed = new AtomicInteger(0);
        Thread markRepl = new Thread() {
                public void run() {
                    try {
                        for (int i = 0; i < iterations; i++) {
                            long l = replicaMgr2.getLedgerToRereplicate();
                            replicaMgr2.markLedgerReplicated(l);
                            processed.incrementAndGet();
                        }
                    } catch (Exception e) {
                        LOG.error("markRepl Thread failed with exception", e);
                        threadFailed.set(true);

View Full Code Here

    private void verifyMarkLedgerUnderreplicated(Collection<String> missingReplica)
            throws KeeperException, InterruptedException,
            CompatibilityException, UnavailableException {
        Long ledgerA = 0xfeadeefdacL;
        String znodeA = getUrLedgerZnode(ledgerA);
        LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();
        for (String replica : missingReplica) {
            replicaMgr.markLedgerUnderreplicated(ledgerA, replica);
        }


        String urLedgerA = getData(znodeA);
        UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat
                .newBuilder();

View Full Code Here


        ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);


        LedgerManagerFactory mFactory = LedgerManagerFactory
                .newLedgerManagerFactory(baseClientConf, zkc);
        LedgerUnderreplicationManager underReplicationManager = mFactory
                .newLedgerUnderreplicationManager();
        rw.start();
        try {


            underReplicationManager.markLedgerUnderreplicated(lh.getId(),
                    replicaToKill.toString());
            while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
                    .getId(), basePath)) {
                Thread.sleep(100);
            }
            killAllBookies(lh, newBkAddr);
            // Should be able to read the entries from 0-9
            verifyRecoveredLedgers(lh, 0, 9);
            lh = bkc.openLedgerNoRecovery(lh.getId(),
                    BookKeeper.DigestType.CRC32, TESTPASSWD);
            assertFalse("Ledger must have been closed by RW", ClientUtil
                    .isLedgerOpen(lh));
        } finally {
            rw.shutdown();
            underReplicationManager.close();
        }


    }

View Full Code Here


        ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);


        LedgerManagerFactory mFactory = LedgerManagerFactory
                .newLedgerManagerFactory(baseClientConf, zkc);
        LedgerUnderreplicationManager underReplicationManager = mFactory
                .newLedgerUnderreplicationManager();


        rw.start();
        try {


            underReplicationManager.markLedgerUnderreplicated(lh.getId(),
                    replicaToKill.toString());
            while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
                    .getId(), basePath)) {
                Thread.sleep(100);
            }


            killAllBookies(lh, newBkAddr);


            // Should be able to read the entries from 0-9
            verifyRecoveredLedgers(lh, 0, 9);
            lh = bkc.openLedgerNoRecovery(lh.getId(),
                    BookKeeper.DigestType.CRC32, TESTPASSWD);


            // Ledger should be still in open state
            assertTrue("Ledger must have been closed by RW", ClientUtil
                    .isLedgerOpen(lh));
        } finally {
            rw.shutdown();
            underReplicationManager.close();
        }


    }

View Full Code Here

        ledgers.add(0xffffbeefL);
        ledgers.add(0xfacebeefL);
        String missingReplica = "localhost:3181";


        int count = 0;
        LedgerUnderreplicationManager m = lmf1.newLedgerUnderreplicationManager();
        Iterator<Long> iter = ledgers.iterator();
        while (iter.hasNext()) {
            m.markLedgerUnderreplicated(iter.next(), missingReplica);
            count++;
        }


        List<Future<Long>> futures = new ArrayList<Future<Long>>();
        for (int i = 0; i < count; i++) {
            futures.add(getLedgerToReplicate(m));
        }


        for (Future<Long> f : futures) {
            Long l = f.get(5, TimeUnit.SECONDS);
            assertTrue(ledgers.remove(l));
        }


        Future<Long> f = getLedgerToReplicate(m);
        try {
            f.get(5, TimeUnit.SECONDS);
            fail("Shouldn't be able to find a ledger to replicate");
        } catch (TimeoutException te) {
            // correct behaviour
        }
        Long newl = 0xfefefefefefeL;
        m.markLedgerUnderreplicated(newl, missingReplica);
        assertEquals("Should have got the one just added", newl, f.get(5, TimeUnit.SECONDS));
    }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.bookkeeper.meta.LedgerUnderreplicationManager

org.apache.bookkeeper.bookie.BookieShell$AutoRecoveryCmd

org.apache.bookkeeper.bookie.BookieShell$ListUnderreplicatedCmd

org.apache.bookkeeper.replication.AuditorPeriodicBookieCheckTest

org.apache.bookkeeper.replication.AuditorPeriodicCheckTest

org.apache.bookkeeper.replication.AuditorRollingRestartTest

org.apache.bookkeeper.replication.TestLedgerUnderreplicationManager

org.apache.bookkeeper.replication.TestReplicationWorker

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.