// Copyright (C) 2011 Splunk Inc.
//
// Splunk Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.splunk.shuttl.prototype.symlink;
import static org.testng.Assert.*;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.splunk.shuttl.archiver.model.IllegalIndexException;
import com.splunk.shuttl.archiver.model.LocalBucket;
import com.splunk.shuttl.archiver.thaw.SplunkIndexesLayer;
import com.splunk.shuttl.testutil.TUtilsBucket;
import com.splunk.shuttl.testutil.TUtilsEndToEnd;
import com.splunk.shuttl.testutil.TUtilsFile;
import com.splunk.shuttl.testutil.TUtilsFunctional;
/**
* Using results from {@link HadoopFileLocationPrototypeTest}, it should be
* possible to symlink all the files in a bucket to a Splunk index's thaw
* directory. It should then be possible to search for the data in Splunk.
*/
public class BucketBlockSymlinkPrototypeTest {
Path testDataPath;
FileSystem hadoopFileSystem;
private LocalBucket realBucket;
private File thawLocation;
@BeforeMethod
public void setUp() {
realBucket = TUtilsBucket.createRealBucket();
testDataPath = new Path("/testData");
}
@AfterMethod
public void tearDown() throws IOException {
if (hadoopFileSystem != null)
hadoopFileSystem.delete(testDataPath, true);
cleanThawDirectory();
}
private void cleanThawDirectory() {
if (thawLocation != null)
if (thawLocation.listFiles() != null)
for (File f : thawLocation.listFiles())
FileUtils.deleteQuietly(f);
}
@Test(groups = { "prototype" })
@Parameters(value = { "hadoop.host", "hadoop.port", "splunk.username",
"splunk.password", "splunk.host", "splunk.mgmtport" })
public void test(String hadoopHost, String hadoopPort, String splunkUsername,
String splunkPassword, String splunkHost, String splunkPort)
throws IOException {
Path bucketPathOnHadoop = copyRealBucketToHadoop(hadoopHost, hadoopPort);
thawLocation = getShuttlThawLocation(splunkUsername, splunkPassword,
splunkHost, splunkPort);
assertTrue(TUtilsFile.isDirectoryEmpty(thawLocation));
File thawBucket = new File(thawLocation, realBucket.getName());
assertTrue(thawBucket.mkdirs());
FileStatus[] pathsInBucketOnHadoop = hadoopFileSystem
.listStatus(bucketPathOnHadoop);
for (FileStatus fs : pathsInBucketOnHadoop)
if (fs.isDir())
handleRawdataDirectory(fs, thawBucket);
else
createSymlinkToPathInDir(fs.getPath(), thawBucket);
}
private void handleRawdataDirectory(FileStatus fs, File thawBucket)
throws IOException {
Path rawdataOnHadoop = fs.getPath();
String rawdataName = rawdataOnHadoop.getName();
assertEquals("rawdata", rawdataName);
File rawdataInThaw = new File(thawBucket, rawdataName);
assertTrue(rawdataInThaw.mkdirs()); // Create the rawdata directory
FileStatus[] lsInRawdata = hadoopFileSystem.listStatus(fs.getPath());
for (FileStatus fs2 : lsInRawdata) {
if (fs2.isDir())
throw new IllegalStateException("Cannot be another "
+ "dir in the rawdata dir");
else
createSymlinkToPathInDir(fs2.getPath(), rawdataInThaw);
}
}
private void createSymlinkToPathInDir(Path path, File dir) throws IOException {
File fileInDir = new File(dir, path.getName());
DistributedFileSystem dfs = (DistributedFileSystem) hadoopFileSystem;
ClientProtocol namenode = dfs.getClient().namenode;
String pathOnHadoop = path.toUri().getPath();
LocatedBlocks blockLocations = namenode.getBlockLocations(pathOnHadoop, 0,
Long.MAX_VALUE);
List<LocatedBlock> locatedBlocks = blockLocations.getLocatedBlocks();
if (!locatedBlocks.isEmpty()) {
doSymlinkPathInDir(fileInDir, blockLocations, locatedBlocks);
} else {
// Means that they don't have a block and that they are empty files. Just
// create them.
assertTrue(fileInDir.createNewFile());
}
}
private void doSymlinkPathInDir(File fileInDir, LocatedBlocks blockLocations,
List<LocatedBlock> locatedBlocks) throws IOException {
assertEquals(1, locatedBlocks.size());
LocatedBlock locatedBlock = blockLocations.get(0);
assertEquals(1, locatedBlock.getLocations().length);
DatanodeInfo datanodeInfo = locatedBlock.getLocations()[0];
ClientDatanodeProtocol createClientDatanodeProtocolProxy = HadoopFileLocationPrototypeTest
.createClientDatanodeProtocolProxy(datanodeInfo,
hadoopFileSystem.getConf(), 1000);
BlockLocalPathInfo blockLocalPathInfo = createClientDatanodeProtocolProxy
.getBlockLocalPathInfo(locatedBlock.getBlock(),
locatedBlock.getBlockToken());
String absolutePathToBlock = blockLocalPathInfo.getBlockPath();
assertTrue(new File(absolutePathToBlock).exists());
FileUtil.symLink(absolutePathToBlock, fileInDir.getAbsolutePath());
}
private Path copyRealBucketToHadoop(String hadoopHost, String hadoopPort)
throws IOException {
hadoopFileSystem = TUtilsFunctional.getHadoopFileSystem(hadoopHost,
hadoopPort);
Path bucketPathOnHadoop = new Path(testDataPath, realBucket.getName());
hadoopFileSystem.copyFromLocalFile(new Path(realBucket.getDirectory()
.toURI()), bucketPathOnHadoop);
return bucketPathOnHadoop;
}
private File getShuttlThawLocation(String splunkUsername,
String splunkPassword, String splunkHost, String splunkPort)
throws IllegalIndexException {
SplunkIndexesLayer splunkIndexesLayer = new SplunkIndexesLayer(
TUtilsEndToEnd.getLoggedInService(splunkHost, splunkPort,
splunkUsername, splunkPassword));
return splunkIndexesLayer.getThawLocation(TUtilsEndToEnd.REAL_SPLUNK_INDEX);
}
}