/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.bookkeeper.replication;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
import org.apache.bookkeeper.proto.DataFormats.UnderreplicatedLedgerFormat;
import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
import org.apache.bookkeeper.test.ZooKeeperUtil;
import org.apache.bookkeeper.util.BookKeeperConstants;
import org.apache.bookkeeper.util.ZkUtils;
import org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase;
import org.apache.commons.lang.StringUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.Watcher.Event.EventType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.protobuf.TextFormat;
/**
* Test the zookeeper implementation of the ledger replication manager
*/
public class TestLedgerUnderreplicationManager {
static final Logger LOG = LoggerFactory.getLogger(TestLedgerUnderreplicationManager.class);
ZooKeeperUtil zkUtil = null;
ServerConfiguration conf = null;
ExecutorService executor = null;
LedgerManagerFactory lmf1 = null;
LedgerManagerFactory lmf2 = null;
ZooKeeper zkc1 = null;
ZooKeeper zkc2 = null;
String basePath;
String urLedgerPath;
boolean isLedgerReplicationDisabled = true;
@Before
public void setupZooKeeper() throws Exception {
zkUtil = new ZooKeeperUtil();
zkUtil.startServer();
conf = new ServerConfiguration()
.setAllowLoopback(true)
.setZkServers(zkUtil.getZooKeeperConnectString());
executor = Executors.newCachedThreadPool();
ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000);
zkc1 = ZkUtils.createConnectedZookeeperClient(
zkUtil.getZooKeeperConnectString(), w);
w = new ZooKeeperWatcherBase(10000);
zkc2 = ZkUtils.createConnectedZookeeperClient(
zkUtil.getZooKeeperConnectString(), w);
lmf1 = LedgerManagerFactory.newLedgerManagerFactory(conf, zkc1);
lmf2 = LedgerManagerFactory.newLedgerManagerFactory(conf, zkc2);
basePath = conf.getZkLedgersRootPath() + '/'
+ BookKeeperConstants.UNDER_REPLICATION_NODE;
urLedgerPath = basePath
+ BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH;
}
@After
public void teardownZooKeeper() throws Exception {
if (zkUtil != null) {
zkUtil.killServer();
zkUtil = null;
}
if (executor != null) {
executor = null;
}
if (zkc1 != null) {
zkc1.close();
zkc1 = null;
}
if (zkc2 != null) {
zkc2.close();
zkc2 = null;
}
if (lmf1 != null) {
lmf1.uninitialize();
lmf1 = null;
}
if (lmf2 != null) {
lmf2.uninitialize();
lmf2 = null;
}
}
private Future<Long> getLedgerToReplicate(final LedgerUnderreplicationManager m) {
return executor.submit(new Callable<Long>() {
public Long call() {
try {
return m.getLedgerToRereplicate();
} catch (Exception e) {
LOG.error("Error getting ledger id", e);
return -1L;
}
}
});
}
/**
* Test basic interactions with the ledger underreplication
* manager.
* Mark some ledgers as underreplicated.
* Ensure that getLedgerToReplicate will block until it a ledger
* becomes available.
*/
@Test(timeout=60000)
public void testBasicInteraction() throws Exception {
Set<Long> ledgers = new HashSet<Long>();
ledgers.add(0xdeadbeefL);
ledgers.add(0xbeefcafeL);
ledgers.add(0xffffbeefL);
ledgers.add(0xfacebeefL);
String missingReplica = "localhost:3181";
int count = 0;
LedgerUnderreplicationManager m = lmf1.newLedgerUnderreplicationManager();
Iterator<Long> iter = ledgers.iterator();
while (iter.hasNext()) {
m.markLedgerUnderreplicated(iter.next(), missingReplica);
count++;
}
List<Future<Long>> futures = new ArrayList<Future<Long>>();
for (int i = 0; i < count; i++) {
futures.add(getLedgerToReplicate(m));
}
for (Future<Long> f : futures) {
Long l = f.get(5, TimeUnit.SECONDS);
assertTrue(ledgers.remove(l));
}
Future<Long> f = getLedgerToReplicate(m);
try {
f.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// correct behaviour
}
Long newl = 0xfefefefefefeL;
m.markLedgerUnderreplicated(newl, missingReplica);
assertEquals("Should have got the one just added", newl, f.get(5, TimeUnit.SECONDS));
}
/**
* Test locking for ledger unreplication manager.
* If there's only one ledger marked for rereplication,
* and one client has it, it should be locked; another
* client shouldn't be able to get it. If the first client dies
* however, the second client should be able to get it.
*/
@Test(timeout=60000)
public void testLocking() throws Exception {
String missingReplica = "localhost:3181";
LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
Long ledger = 0xfeadeefdacL;
m1.markLedgerUnderreplicated(ledger, missingReplica);
Future<Long> f = getLedgerToReplicate(m1);
Long l = f.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I just marked", ledger, l);
f = getLedgerToReplicate(m2);
try {
f.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// correct behaviour
}
zkc1.close(); // should kill the lock
zkc1 = null;
l = f.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I marked", ledger, l);
}
/**
* Test that when a ledger has been marked as replicated, it
* will not be offered to anther client.
* This test checked that by marking two ledgers, and acquiring
* them on a single client. It marks one as replicated and then
* the client is killed. We then check that another client can
* acquire a ledger, and that it's not the one that was previously
* marked as replicated.
*/
@Test(timeout=60000)
public void testMarkingAsReplicated() throws Exception {
String missingReplica = "localhost:3181";
LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
Long ledgerA = 0xfeadeefdacL;
Long ledgerB = 0xdefadebL;
m1.markLedgerUnderreplicated(ledgerA, missingReplica);
m1.markLedgerUnderreplicated(ledgerB, missingReplica);
Future<Long> fA = getLedgerToReplicate(m1);
Future<Long> fB = getLedgerToReplicate(m1);
Long lA = fA.get(5, TimeUnit.SECONDS);
Long lB = fB.get(5, TimeUnit.SECONDS);
assertTrue("Should be the ledgers I just marked",
(lA.equals(ledgerA) && lB.equals(ledgerB))
|| (lA.equals(ledgerB) && lB.equals(ledgerA)));
Future<Long> f = getLedgerToReplicate(m2);
try {
f.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// correct behaviour
}
m1.markLedgerReplicated(lA);
zkc1.close(); // should kill the lock
zkc1 = null;
Long l = f.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I marked", lB, l);
}
/**
* Test releasing of a ledger
* A ledger is released when a client decides it does not want
* to replicate it (or cannot at the moment).
* When a client releases a previously acquired ledger, another
* client should then be able to acquire it.
*/
@Test(timeout=60000)
public void testRelease() throws Exception {
String missingReplica = "localhost:3181";
LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
Long ledgerA = 0xfeadeefdacL;
Long ledgerB = 0xdefadebL;
m1.markLedgerUnderreplicated(ledgerA, missingReplica);
m1.markLedgerUnderreplicated(ledgerB, missingReplica);
Future<Long> fA = getLedgerToReplicate(m1);
Future<Long> fB = getLedgerToReplicate(m1);
Long lA = fA.get(5, TimeUnit.SECONDS);
Long lB = fB.get(5, TimeUnit.SECONDS);
assertTrue("Should be the ledgers I just marked",
(lA.equals(ledgerA) && lB.equals(ledgerB))
|| (lA.equals(ledgerB) && lB.equals(ledgerA)));
Future<Long> f = getLedgerToReplicate(m2);
try {
f.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// correct behaviour
}
m1.markLedgerReplicated(lA);
m1.releaseUnderreplicatedLedger(lB);
Long l = f.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I marked", lB, l);
}
/**
* Test that when a failure occurs on a ledger, while the ledger
* is already being rereplicated, the ledger will still be in the
* under replicated ledger list when first rereplicating client marks
* it as replicated.
*/
@Test(timeout=60000)
public void testManyFailures() throws Exception {
String missingReplica1 = "localhost:3181";
String missingReplica2 = "localhost:3182";
LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
Long ledgerA = 0xfeadeefdacL;
m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
Future<Long> fA = getLedgerToReplicate(m1);
Long lA = fA.get(5, TimeUnit.SECONDS);
m1.markLedgerUnderreplicated(ledgerA, missingReplica2);
assertEquals("Should be the ledger I just marked",
lA, ledgerA);
m1.markLedgerReplicated(lA);
Future<Long> f = getLedgerToReplicate(m1);
lA = f.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I had marked previously",
lA, ledgerA);
}
/**
* Test that when a ledger is marked as underreplicated with
* the same missing replica twice, only marking as replicated
* will be enough to remove it from the list.
*/
@Test(timeout=60000)
public void test2reportSame() throws Exception {
String missingReplica1 = "localhost:3181";
LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
Long ledgerA = 0xfeadeefdacL;
m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
m2.markLedgerUnderreplicated(ledgerA, missingReplica1);
// verify duplicate missing replica
UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat
.newBuilder();
String znode = getUrLedgerZnode(ledgerA);
byte[] data = zkc1.getData(znode, false, null);
TextFormat.merge(new String(data, Charset.forName("UTF-8")), builderA);
List<String> replicaList = builderA.getReplicaList();
assertEquals("Published duplicate missing replica : " + replicaList, 1,
replicaList.size());
assertTrue("Published duplicate missing replica : " + replicaList,
replicaList.contains(missingReplica1));
Future<Long> fA = getLedgerToReplicate(m1);
Long lA = fA.get(5, TimeUnit.SECONDS);
assertEquals("Should be the ledger I just marked",
lA, ledgerA);
m1.markLedgerReplicated(lA);
Future<Long> f = getLedgerToReplicate(m2);
try {
f.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// correct behaviour
}
}
/**
* Test that multiple LedgerUnderreplicationManagers should be able to take
* lock and release for same ledger
*/
@Test(timeout = 30000)
public void testMultipleManagersShouldBeAbleToTakeAndReleaseLock()
throws Exception {
String missingReplica1 = "localhost:3181";
final LedgerUnderreplicationManager m1 = lmf1
.newLedgerUnderreplicationManager();
final LedgerUnderreplicationManager m2 = lmf2
.newLedgerUnderreplicationManager();
Long ledgerA = 0xfeadeefdacL;
m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
final int iterationCount = 100;
final CountDownLatch latch1 = new CountDownLatch(iterationCount);
final CountDownLatch latch2 = new CountDownLatch(iterationCount);
Thread thread1 = new Thread() {
@Override
public void run() {
takeLedgerAndRelease(m1, latch1, iterationCount);
}
};
Thread thread2 = new Thread() {
@Override
public void run() {
takeLedgerAndRelease(m2, latch2, iterationCount);
}
};
thread1.start();
thread2.start();
// wait until at least one thread completed
while (!latch1.await(50, TimeUnit.MILLISECONDS)
&& !latch2.await(50, TimeUnit.MILLISECONDS)) {
Thread.sleep(50);
}
m1.close();
m2.close();
// After completing 'lock acquire,release' job, it should notify below
// wait
latch1.await();
latch2.await();
}
/**
* Test verifies failures of bookies which are resembling each other.
*
* BK servers named like*********************************************
* 1.cluster.com, 2.cluster.com, 11.cluster.com, 12.cluster.com
* *******************************************************************
*
* BKserver IP:HOST like*********************************************
* localhost:3181, localhost:318, localhost:31812
* *******************************************************************
*/
@Test(timeout=60000)
public void testMarkSimilarMissingReplica() throws Exception {
List<String> missingReplica = new ArrayList<String>();
missingReplica.add("localhost:3181");
missingReplica.add("localhost:318");
missingReplica.add("localhost:31812");
missingReplica.add("1.cluster.com");
missingReplica.add("2.cluster.com");
missingReplica.add("11.cluster.com");
missingReplica.add("12.cluster.com");
verifyMarkLedgerUnderreplicated(missingReplica);
}
/**
* Test multiple bookie failures for a ledger and marked as underreplicated
* one after another.
*/
@Test(timeout=60000)
public void testManyFailuresInAnEnsemble() throws Exception {
List<String> missingReplica = new ArrayList<String>();
missingReplica.add("localhost:3181");
missingReplica.add("localhost:3182");
verifyMarkLedgerUnderreplicated(missingReplica);
}
/**
* Test disabling the ledger re-replication. After disabling, it will not be
* able to getLedgerToRereplicate(). This calls will enter into infinite
* waiting until enabling rereplication process
*/
@Test(timeout = 20000)
public void testDisableLedegerReplication() throws Exception {
final LedgerUnderreplicationManager replicaMgr = lmf1
.newLedgerUnderreplicationManager();
// simulate few urLedgers before disabling
final Long ledgerA = 0xfeadeefdacL;
final String missingReplica = "localhost:3181";
// disabling replication
replicaMgr.disableLedgerReplication();
LOG.info("Disabled Ledeger Replication");
try {
replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
} catch (UnavailableException e) {
LOG.debug("Unexpected exception while marking urLedger", e);
fail("Unexpected exception while marking urLedger" + e.getMessage());
}
Future<Long> fA = getLedgerToReplicate(replicaMgr);
try {
fA.get(5, TimeUnit.SECONDS);
fail("Shouldn't be able to find a ledger to replicate");
} catch (TimeoutException te) {
// expected behaviour, as the replication is disabled
isLedgerReplicationDisabled = false;
}
assertTrue("Ledger replication is not disabled!",
!isLedgerReplicationDisabled);
}
/**
* Test enabling the ledger re-replication. After enableLedegerReplication,
* should continue getLedgerToRereplicate() task
*/
@Test(timeout = 20000)
public void testEnableLedegerReplication() throws Exception {
isLedgerReplicationDisabled = true;
final LedgerUnderreplicationManager replicaMgr = lmf1
.newLedgerUnderreplicationManager();
// simulate few urLedgers before disabling
final Long ledgerA = 0xfeadeefdacL;
final String missingReplica = "localhost:3181";
try {
replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
} catch (UnavailableException e) {
LOG.debug("Unexpected exception while marking urLedger", e);
fail("Unexpected exception while marking urLedger" + e.getMessage());
}
// disabling replication
replicaMgr.disableLedgerReplication();
LOG.debug("Disabled Ledeger Replication");
String znodeA = getUrLedgerZnode(ledgerA);
final CountDownLatch znodeLatch = new CountDownLatch(2);
String urledgerA = StringUtils.substringAfterLast(znodeA, "/");
String urLockLedgerA = basePath + "/locks/" + urledgerA;
zkc1.exists(urLockLedgerA, new Watcher(){
@Override
public void process(WatchedEvent event) {
if (event.getType() == EventType.NodeCreated) {
znodeLatch.countDown();
LOG.debug("Recieved node creation event for the zNodePath:"
+ event.getPath());
}
}});
// getLedgerToRereplicate is waiting until enable rereplication
Thread thread1 = new Thread() {
@Override
public void run() {
try {
Long lA = replicaMgr.getLedgerToRereplicate();
assertEquals("Should be the ledger I just marked", lA,
ledgerA);
isLedgerReplicationDisabled = false;
znodeLatch.countDown();
} catch (UnavailableException e) {
LOG.debug("Unexpected exception while marking urLedger", e);
isLedgerReplicationDisabled = false;
}
}
};
thread1.start();
try {
znodeLatch.await(5, TimeUnit.SECONDS);
assertTrue("Ledger replication is not disabled!",
isLedgerReplicationDisabled);
assertEquals("Failed to disable ledger replication!", 2, znodeLatch
.getCount());
replicaMgr.enableLedgerReplication();
znodeLatch.await(5, TimeUnit.SECONDS);
LOG.debug("Enabled Ledeger Replication");
assertTrue("Ledger replication is not disabled!",
!isLedgerReplicationDisabled);
assertEquals("Failed to disable ledger replication!", 0, znodeLatch
.getCount());
} finally {
thread1.interrupt();
}
}
/**
* Test that the hierarchy gets cleaned up as ledgers
* are marked as fully replicated
*/
@Test(timeout=60000)
public void testHierarchyCleanup() throws Exception {
final LedgerUnderreplicationManager replicaMgr = lmf1
.newLedgerUnderreplicationManager();
// 4 ledgers, 2 in the same hierarchy
long[] ledgers = { 0x00000000deadbeefL, 0x00000000deadbeeeL,
0x00000000beefcafeL, 0x00000000cafed00dL };
for (long l : ledgers) {
replicaMgr.markLedgerUnderreplicated(l, "localhost:3181");
}
// can't simply test top level as we are limited to ledger
// ids no larger than an int
String testPath = urLedgerPath + "/0000/0000";
List<String> children = zkc1.getChildren(testPath, false);
assertEquals("Wrong number of hierarchies", 3, children.size());
int marked = 0;
while (marked < 3) {
long l = replicaMgr.getLedgerToRereplicate();
if (l != ledgers[0]) {
replicaMgr.markLedgerReplicated(l);
marked++;
} else {
replicaMgr.releaseUnderreplicatedLedger(l);
}
}
children = zkc1.getChildren(testPath, false);
assertEquals("Wrong number of hierarchies", 1, children.size());
long l = replicaMgr.getLedgerToRereplicate();
assertEquals("Got wrong ledger", ledgers[0], l);
replicaMgr.markLedgerReplicated(l);
children = zkc1.getChildren(urLedgerPath, false);
assertEquals("All hierarchies should be cleaned up", 0, children.size());
}
/**
* Test that as the hierarchy gets cleaned up, it doesn't interfere
* with the marking of other ledgers as underreplicated
*/
@Test(timeout = 90000)
public void testHierarchyCleanupInterference() throws Exception {
final LedgerUnderreplicationManager replicaMgr1 = lmf1
.newLedgerUnderreplicationManager();
final LedgerUnderreplicationManager replicaMgr2 = lmf2
.newLedgerUnderreplicationManager();
final int iterations = 1000;
final AtomicBoolean threadFailed = new AtomicBoolean(false);
Thread markUnder = new Thread() {
public void run() {
long l = 1;
try {
for (int i = 0; i < iterations; i++) {
replicaMgr1.markLedgerUnderreplicated(l, "localhost:3181");
l += 10000;
}
} catch (Exception e) {
LOG.error("markUnder Thread failed with exception", e);
threadFailed.set(true);
return;
}
}
};
final AtomicInteger processed = new AtomicInteger(0);
Thread markRepl = new Thread() {
public void run() {
try {
for (int i = 0; i < iterations; i++) {
long l = replicaMgr2.getLedgerToRereplicate();
replicaMgr2.markLedgerReplicated(l);
processed.incrementAndGet();
}
} catch (Exception e) {
LOG.error("markRepl Thread failed with exception", e);
threadFailed.set(true);
return;
}
}
};
markRepl.setDaemon(true);
markUnder.setDaemon(true);
markRepl.start();
markUnder.start();
markUnder.join();
assertFalse("Thread failed to complete", threadFailed.get());
int lastProcessed = 0;
while (true) {
markRepl.join(10000);
if (!markRepl.isAlive()) {
break;
}
assertFalse("markRepl thread not progressing", lastProcessed == processed.get());
}
assertFalse("Thread failed to complete", threadFailed.get());
List<String> children = zkc1.getChildren(urLedgerPath, false);
for (String s : children) {
LOG.info("s: {}", s);
}
assertEquals("All hierarchies should be cleaned up", 0, children.size());
}
private void verifyMarkLedgerUnderreplicated(Collection<String> missingReplica)
throws KeeperException, InterruptedException,
CompatibilityException, UnavailableException {
Long ledgerA = 0xfeadeefdacL;
String znodeA = getUrLedgerZnode(ledgerA);
LedgerUnderreplicationManager replicaMgr = lmf1
.newLedgerUnderreplicationManager();
for (String replica : missingReplica) {
replicaMgr.markLedgerUnderreplicated(ledgerA, replica);
}
String urLedgerA = getData(znodeA);
UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat
.newBuilder();
for (String replica : missingReplica) {
builderA.addReplica(replica);
}
List<String> replicaList = builderA.getReplicaList();
for (String replica : missingReplica) {
assertTrue("UrLedger:" + urLedgerA
+ " doesn't contain failed bookie :" + replica, replicaList
.contains(replica));
}
}
private String getData(String znode) {
try {
byte[] data = zkc1.getData(znode, false, null);
return new String(data);
} catch (KeeperException e) {
LOG.error("Exception while reading data from znode :" + znode);
} catch (InterruptedException e) {
LOG.error("Exception while reading data from znode :" + znode);
}
return "";
}
private String getUrLedgerZnode(long ledgerId) {
return ZkLedgerUnderreplicationManager.getUrLedgerZnode(urLedgerPath, ledgerId);
}
private void takeLedgerAndRelease(final LedgerUnderreplicationManager m,
final CountDownLatch latch, int numberOfIterations) {
for (int i = 0; i < numberOfIterations; i++) {
try {
long ledgerToRereplicate = m.getLedgerToRereplicate();
m.releaseUnderreplicatedLedger(ledgerToRereplicate);
} catch (UnavailableException e) {
LOG.error("UnavailableException when "
+ "taking or releasing lock", e);
}
latch.countDown();
}
}
}