package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestRaidDfs;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.raid.Codec;
import org.apache.hadoop.raid.RaidNode;
import org.apache.hadoop.raid.RaidUtils;
import org.apache.hadoop.raid.Utils;
import org.junit.Test;
import junit.framework.Assert;
import junit.framework.TestCase;
public class TestDirectoryRaidBlockPlacement extends TestCase {
private Configuration conf = null;
private MiniDFSCluster cluster = null;
private FSNamesystem namesystem = null;
private BlockPlacementPolicyRaid policy = null;
private FileSystem fs = null;
String[] rack1 = {"/rack1", "/rack1", "/rack1"};
String[] host1 = {"host1.rack1.com", "host2.rack1.com", "host3.rack1.com"};
final static Log LOG =
LogFactory.getLog(TestDirectoryRaidBlockPlacement.class);
protected void setupCluster() throws IOException {
conf = new Configuration();
conf.setLong("dfs.blockreport.intervalMsec", 1000L);
conf.set("dfs.replication.pending.timeout.sec", "2");
conf.setLong("dfs.block.size", 1L);
conf.set("dfs.block.replicator.classname",
"org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid");
Utils.loadTestCodecs(conf, 5, 5, 1, 3, "/raid", "/raidrs", false, true);
conf.setInt("io.bytes.per.checksum", 1);
cluster = new MiniDFSCluster(conf, 3, true, rack1, host1);
cluster.waitActive();
namesystem = cluster.getNameNode().getNamesystem();
Assert.assertTrue("BlockPlacementPolicy type is not correct.",
namesystem.replicator instanceof BlockPlacementPolicyRaid);
policy = (BlockPlacementPolicyRaid) namesystem.replicator;
fs = cluster.getFileSystem();
}
protected void closeCluster() throws IOException {
if (null != cluster) {
cluster.shutdown();
}
}
@Test
public void testGetCompanionBLocks() throws IOException {
try {
setupCluster();
String[] files = new String[] {"/dir/file1", "/dir/file2", "/dir/file3"};
Codec codec = Codec.getCodec("rs");
for (String file : files) {
TestRaidDfs.createTestFile(fs, new Path(file), 3, 2, 8192L);
}
FileStatus stat = fs.getFileStatus(new Path("/dir"));
RaidNode.doDirRaid(conf, stat, new Path(codec.parityDirectory), codec,
new RaidNode.Statistics(),
RaidUtils.NULL_PROGRESSABLE, false, 1, 1);
Collection<LocatedBlock> companionBlocks;
for (int i=0; i<2; i++) {
for (int j=0; j<2; j++) {
companionBlocks = getCompanionBlocks(
namesystem, policy, getBlocks(namesystem,
files[i]).get(j).getBlock());
Assert.assertEquals(8, companionBlocks.size());
}
}
companionBlocks = getCompanionBlocks(
namesystem, policy, getBlocks(namesystem,
files[2]).get(0).getBlock());
Assert.assertEquals(8, companionBlocks.size());
companionBlocks = getCompanionBlocks(
namesystem, policy, getBlocks(namesystem,
files[2]).get(1).getBlock());
Assert.assertEquals(4, companionBlocks.size());
String parityFile = "/raidrs/dir";
for (int i=0; i<3; i++) {
companionBlocks = getCompanionBlocks(
namesystem, policy, getBlocks(namesystem,
parityFile).get(i).getBlock());
Assert.assertEquals(8, companionBlocks.size());
}
for (int i=3; i<6; i++) {
companionBlocks = getCompanionBlocks(
namesystem, policy, getBlocks(namesystem,
parityFile).get(i).getBlock());
Assert.assertEquals(4, companionBlocks.size());
}
} finally {
closeCluster();
}
}
private Collection<LocatedBlock> getCompanionBlocks(
FSNamesystem namesystem, BlockPlacementPolicyRaid policy,
Block block) throws IOException {
INodeFile inode = namesystem.blocksMap.getINode(block);
BlockPlacementPolicyRaid.FileInfo info =
policy.getFileInfo(inode.getFullPathName());
return policy.getCompanionBlocks(inode.getFullPathName(), info, block);
}
private List<LocatedBlock> getBlocks(FSNamesystem namesystem, String file)
throws IOException {
FileStatus stat = namesystem.getFileInfo(file);
return namesystem.getBlockLocations(
file, 0, stat.getLen()).getLocatedBlocks();
}
}