/*
* Copyright 2013 NGDATA nv
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ngdata.hbaseindexer.mr;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.net.ServerSocket;
import java.util.List;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import com.ngdata.hbaseindexer.conf.DefaultIndexerComponentFactory;
import com.ngdata.hbaseindexer.model.api.IndexerDefinition;
import com.ngdata.hbaseindexer.model.api.IndexerDefinitionBuilder;
import com.ngdata.hbaseindexer.model.impl.IndexerModelImpl;
import com.ngdata.sep.util.io.Closer;
import com.ngdata.sep.util.zookeeper.ZkUtil;
import com.ngdata.sep.util.zookeeper.ZooKeeperItf;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.util.ToolRunner;
import org.joda.time.format.DateTimeFormat;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
public class HBaseIndexingOptionsTest {
private static MiniZooKeeperCluster ZK_CLUSTER;
private static File ZK_DIR;
private static int ZK_CLIENT_PORT;
private static ZooKeeperItf ZK;
private static IndexerModelImpl INDEXER_MODEL;
private Configuration conf;
private HBaseIndexingOptions opts;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
ZK_DIR = new File(System.getProperty("java.io.tmpdir") + File.separator + "hbaseindexer.zktest");
ZK_CLIENT_PORT = getFreePort();
ZK_CLUSTER = new MiniZooKeeperCluster();
ZK_CLUSTER.setDefaultClientPort(ZK_CLIENT_PORT);
ZK_CLUSTER.startup(ZK_DIR);
ZK = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
INDEXER_MODEL = new IndexerModelImpl(ZK, "/ngdata/hbaseindexer");
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
INDEXER_MODEL.stop();
Closer.close(ZK);
if (ZK_CLUSTER != null) {
ZK_CLUSTER.shutdown();
}
FileUtils.deleteDirectory(ZK_DIR);
}
private static int getFreePort() {
ServerSocket socket = null;
try {
socket = new ServerSocket(0);
return socket.getLocalPort();
} catch (IOException e) {
throw new RuntimeException("Error finding a free port", e);
} finally {
if (socket != null) {
try {
socket.close();
} catch (IOException e) {
throw new RuntimeException("Error closing ServerSocket used to detect a free port.", e);
}
}
}
}
private void addAndWaitForIndexer(IndexerDefinition indexerDef) throws Exception {
long startTime = System.currentTimeMillis();
INDEXER_MODEL.addIndexer(indexerDef);
// Wait max 5 seconds
while (System.currentTimeMillis() - startTime < 15000) {
if (INDEXER_MODEL.hasIndexer(indexerDef.getName())) {
return;
}
Thread.sleep(200);
}
throw new RuntimeException("Failed to add indexer: " + indexerDef);
}
@Before
public void setUp() throws /*ZkConnectException,*/ InterruptedException, /*KeeperException,*/ IOException {
conf = new Configuration();
opts = new HBaseIndexingOptions(conf);
HBaseAdmin hBaseAdmin = Mockito.mock(HBaseAdmin.class);
Mockito.when(hBaseAdmin.listTables("record")).thenReturn(new HTableDescriptor[]{
new HTableDescriptor("record")
});
opts.hBaseAdmin = hBaseAdmin;
}
@Test
public void testIsDirectWrite_True() {
opts.reducers = 0;
assertTrue(opts.isDirectWrite());
}
@Test
public void testIsDirectWrite_False() {
opts.reducers = 1;
assertFalse(opts.isDirectWrite());
// Different negative values have different meanings in core search MR job
opts.reducers = -1;
assertFalse(opts.isDirectWrite());
opts.reducers = -2;
assertFalse(opts.isDirectWrite());
}
@Test
public void testEvaluateOutputDir_DirectWrite() {
opts.outputDir = null;
opts.reducers = 0;
opts.zkHost = "myzkhost/solr";
// Should have no effect
opts.evaluateOutputDir();
assertNull(opts.outputDir);
assertFalse(opts.isGeneratedOutputDir());
}
@Test
public void testEvaluateOutputDir_DryRun() {
opts.outputDir = null;
opts.isDryRun = true;
opts.zkHost = "myzkhost/solr";
// Should have no effect
opts.evaluateOutputDir();
assertNull(opts.outputDir);
assertFalse(opts.isGeneratedOutputDir());
}
@Test(expected=IllegalStateException.class)
public void testEvaluateOutputDir_DirectWrite_NoSolrZkHostDefined() {
opts.outputDir = null;
opts.reducers = 0;
opts.zkHost = null;
opts.evaluateOutputDir();
}
@Test
public void testEvaluateOutputDir_DirectWrite_WithZkBasedIndexer() {
opts.outputDir = null;
opts.reducers = 0;
opts.zkHost = null;
opts.hbaseIndexerName = "indexer";
opts.hbaseIndexerZkHost = "indexerhost";
// Should have no effect
opts.evaluateOutputDir();
assertNull(opts.outputDir);
assertFalse(opts.isGeneratedOutputDir());
}
@Test
public void testEvaluateOutputDir_GoLive() {
opts.outputDir = null;
opts.reducers = 2;
opts.goLive = true;
opts.evaluateOutputDir();
Path outputPath = opts.outputDir;
assertEquals(new Path("/tmp"), outputPath.getParent());
assertTrue(opts.isGeneratedOutputDir());
}
@Test
public void testEvaluateOutputDir_GoLive_UserDefinedOutputDir() {
opts.outputDir = new Path("/outputdir");
opts.reducers = 2;
opts.goLive = true;
opts.evaluateOutputDir();
Path outputPath = opts.outputDir;
assertEquals(new Path("/outputdir"), outputPath);
assertFalse(opts.isGeneratedOutputDir());
}
@Test
public void testEvaluateOutputDir_GoLive_AlternateTempDirViaConfig() {
opts.outputDir = null;
opts.reducers = 2;
opts.goLive = true;
conf.set("hbase.search.mr.tmpdir", "/othertmp");
opts.evaluateOutputDir();
Path outputPath = opts.outputDir;
assertEquals(new Path("/othertmp"), outputPath.getParent());
assertTrue(opts.isGeneratedOutputDir());
}
@Test
public void testEvaluateOutputDir_WriteToFile() {
opts.outputDir = new Path("/output/path");
opts.reducers = 2;
// Should have no effect
opts.evaluateOutputDir();
assertEquals(new Path("/output/path"), opts.outputDir);
assertFalse(opts.isGeneratedOutputDir());
}
@Test(expected=IllegalStateException.class)
public void testEvaluateOutputDir_NoOutputDirButNotDirectWriteMode() {
opts.outputDir = null;
opts.reducers = 1;
opts.evaluateOutputDir();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateOutputDir_OutputDirSetAlongWithDirectWriteMode() {
opts.outputDir = new Path("/some/path");
opts.reducers = 0;
opts.evaluateOutputDir();
}
@Test
public void testEvaluateIndexingSpecification_PullSolrZkHostFromIndexerDefinition() throws Exception {
IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
.name("customsolr")
.configuration(Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")))
.connectionParams(ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"))
.build();
addAndWaitForIndexer(indexerDef);
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = "customsolr";
opts.zkHost = "customhost/solr";
opts.collection = null;
opts.evaluateIndexingSpecification();
INDEXER_MODEL.deleteIndexerInternal("customsolr");
assertEquals("customhost/solr", opts.zkHost);
assertEquals("mycollection", opts.collection);
}
@Test
public void testIndexingSpecification_PullSolrCollectionFromIndexerDefinition() throws Exception {
IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
.name("customcollection")
.configuration(Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")))
.connectionParams(ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"))
.build();
addAndWaitForIndexer(indexerDef);
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = "customcollection";
opts.zkHost = null;
opts.collection = "customcollection";
opts.evaluateIndexingSpecification();
INDEXER_MODEL.deleteIndexerInternal("customcollection");
assertEquals("myZkHost/solr", opts.zkHost);
assertEquals("customcollection", opts.collection);
}
@Test
public void testEvaluateScan_StartRowDefined() throws Exception {
// Set up the dependencies for the indexing specification
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "mock_indexer.xml").toURI());
opts.zkHost = "myzkhost";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.hbaseStartRow = "starthere";
opts.evaluateScan();
assertArrayEquals(Bytes.toBytes("starthere"), opts.getScans().get(0).getStartRow());
}
@Test
public void testEvaluateScan_EndRowDefined() throws Exception {
// Set up the dependencies for the indexing specification
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "mock_indexer.xml").toURI());
opts.zkHost = "myzkhost";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.hbaseEndRow = "endhere";
opts.evaluateScan();
assertArrayEquals(Bytes.toBytes("endhere"), opts.getScans().get(0).getStopRow());
}
@Test
public void testEvaluateScan_StartTimeDefined() throws Exception {
// Set up the dependencies for the indexing specification
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "mock_indexer.xml").toURI());
opts.zkHost = "myzkhost";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.hbaseStartTimeString = "220777";
opts.evaluateScan();
assertEquals(220777L, opts.getScans().get(0).getTimeRange().getMin());
}
@Test
public void testEvaluateScan_EndTimeDefined() throws Exception {
// Set up the dependencies for the indexing specification
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "mock_indexer.xml").toURI());
opts.zkHost = "myzkhost";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.hbaseEndTimeString = "220777";
opts.evaluateScan();
assertEquals(220777L, opts.getScans().get(0).getTimeRange().getMax());
}
@Test
public void testEvaluateScan_CheckFamilyMap() throws Exception{
// Check that the Scan only scans values referred to via the
// ResultToSolrMapper#getGet(byte[]) method
// Set up the dependencies for the indexing specification
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "mock_indexer.xml").toURI());
opts.zkHost = "myzkhost";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
opts.evaluateScan();
Scan scan = opts.getScans().get(0);
assertTrue(scan.getFamilyMap().containsKey(Bytes.toBytes("info")));
// Should be firstname, lastname, age
assertEquals(3, scan.getFamilyMap().get(Bytes.toBytes("info")).size());
}
@Test(expected=IllegalStateException.class)
public void testEvaluateGoLiveArgs_NoZkHostOrSolrHomeDir() {
opts.solrHomeDir = null;
opts.zkHost = null;
opts.evaluateGoLiveArgs();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateGoLiveArgs_GoLiveButNoZkAndNoShards() {
opts.goLive = true;
opts.zkHost = null;
opts.shardUrls = null;
opts.evaluateGoLiveArgs();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateGoLiveArgs_ZkHostButNoCollection() {
opts.zkHost = "myzkhost";
opts.collection = null;
opts.evaluateGoLiveArgs();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateGoLiveArgs_EmptyShardUrls() {
opts.solrHomeDir = new File("/solrhome");
opts.shardUrls = ImmutableList.of();
opts.evaluateGoLiveArgs();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateGoLiveArgs_ZeroShards() {
opts.solrHomeDir = new File("/solrhome");
opts.shards = 0;
opts.evaluateGoLiveArgs();
}
@Test
public void testEvaluateGoLiveArgs_AutoSetShardCount() {
opts.solrHomeDir = new File("/solrhome");
opts.shardUrls = ImmutableList.<List<String>>of(ImmutableList.of("shard_a"), ImmutableList.of("shard_b"));
opts.shards = null;
opts.evaluateGoLiveArgs();
assertEquals(2, (int)opts.shards);
}
@Test
public void testEvaluateIndexingSpecification_AllFromZooKeeper() throws Exception {
IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
.name("userindexer")
.configuration(Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")))
.connectionParams(ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"))
.build();
addAndWaitForIndexer(indexerDef);
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = "userindexer";
opts.evaluateIndexingSpecification();
INDEXER_MODEL.deleteIndexerInternal("userindexer");
IndexingSpecification expectedSpec = new IndexingSpecification(
"record", "userindexer", null,
Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")), ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"));
assertEquals(expectedSpec, opts.getIndexingSpecification());
}
@Test
public void testEvaluateIndexingSpecification_AllFromCmdline() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.hbaseTableName = "mytable";
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.zkHost = "myZkHost/solr";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
IndexingSpecification expectedSpec = new IndexingSpecification(
"mytable", HBaseIndexingOptions.DEFAULT_INDEXER_NAME, DefaultIndexerComponentFactory.class.getName(),
Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")), ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"));
assertEquals(expectedSpec, opts.getIndexingSpecification());
}
@Test
public void testEvaluateIndexingSpecification_SolrClassic() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.hbaseTableName = "mytable";
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.zkHost = null;
opts.collection = null;
opts.solrHomeDir = new File(".");
opts.evaluateIndexingSpecification();
IndexingSpecification expectedSpec = new IndexingSpecification(
"mytable", HBaseIndexingOptions.DEFAULT_INDEXER_NAME, DefaultIndexerComponentFactory.class.getName(),
Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")), ImmutableMap.of(
"solr.mode", "classic",
"solr.home", opts.solrHomeDir.getAbsolutePath()));
assertEquals(expectedSpec, opts.getIndexingSpecification());
}
@Test(expected=IllegalStateException.class)
public void testEvaluateIndexingSpecification_IndexerZkSuppliedButNoIndexerNameSupplied() throws Exception {
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = null;
opts.evaluateIndexingSpecification();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateIndexingSpecification_NonExistantIndexerSupplied() throws Exception {
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = "NONEXISTANT";
opts.evaluateIndexingSpecification();
}
@Test
public void testEvaluateIndexingSpecification_TableNameFromXmlFile() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.zkHost = "myZkHost/solr";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
IndexingSpecification expectedSpec = new IndexingSpecification(
"record", HBaseIndexingOptions.DEFAULT_INDEXER_NAME, DefaultIndexerComponentFactory.class.getName(),
Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")), ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"));
assertEquals(expectedSpec, opts.getIndexingSpecification());
}
@Test(expected=IllegalStateException.class)
public void testEvaluateIndexingSpecification_NoIndexXmlSpecified() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.hbaseIndexerConfigFile = null;
opts.hbaseTableName = "mytable";
opts.zkHost = "myZkHost/solr";
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateIndexingSpecification_NoZkHostSpecified() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.zkHost = null;
opts.hbaseTableName = "mytable";
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.collection = "mycollection";
opts.evaluateIndexingSpecification();
}
@Test(expected=IllegalStateException.class)
public void testEvaluateIndexingSpecification_NoCollectionSpecified() throws Exception {
opts.hbaseIndexerZkHost = null;
opts.collection = null;
opts.hbaseTableName = "mytable";
opts.hbaseIndexerConfigFile = new File(Resources.getResource(getClass(), "user_indexer.xml").toURI());
opts.zkHost = "myZkHost/solr";
opts.evaluateIndexingSpecification();
}
@Test
public void testEvaluateIndexingSpecification_CombinationOfCmdlineAndZk() throws Exception {
// Create an indexer -- verify INDEXER_ADDED event
IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
.name("userindexer")
.configuration(Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")))
.connectionParams(ImmutableMap.of(
"solr.zk", "myZkHost/solr",
"solr.collection", "mycollection"))
.build();
addAndWaitForIndexer(indexerDef);
opts.hbaseIndexerZkHost = "localhost:" + ZK_CLIENT_PORT;
opts.hbaseIndexerName = "userindexer";
opts.hbaseTableName = "mytable";
opts.zkHost = "myOtherZkHost/solr";
opts.evaluateIndexingSpecification();
INDEXER_MODEL.deleteIndexerInternal("userindexer");
IndexingSpecification expectedSpec = new IndexingSpecification(
"mytable", "userindexer", null,
Resources.toByteArray(Resources.getResource(getClass(), "user_indexer.xml")), ImmutableMap.of(
"solr.zk", "myOtherZkHost/solr",
"solr.collection", "mycollection"));
assertEquals(expectedSpec, opts.getIndexingSpecification());
}
@Test
public void testEvaluateTimestamp_NoFormatSupplied() {
assertEquals(
Long.valueOf(12345),
HBaseIndexingOptions.evaluateTimestamp("12345", null));
}
@Test(expected=IllegalStateException.class)
public void testEvaluateTimestamp_NoFormat_NonParseableLong() {
HBaseIndexingOptions.evaluateTimestamp("abc", null);
}
@Test
public void testEvaluateTimestamp_CustomTimestampFormat() {
assertEquals(
Long.valueOf(DateTimeFormat.forPattern("yyyy/MM/dd HH:mm:ss").parseMillis("2013/10/20 00:39:00")),
HBaseIndexingOptions.evaluateTimestamp("2013/10/20 00:39:00", "yyyy/MM/dd HH:mm:ss"));
}
@Test(expected=IllegalStateException.class)
public void testEvaluateTimestamp_InvalidTimestampFormat() {
HBaseIndexingOptions.evaluateTimestamp("2013/10/20 00:39", "not a timestamp format");
}
@Test(expected=IllegalStateException.class)
public void testEvaluateTimestamp_TimestampNotAccordingToFormat() {
HBaseIndexingOptions.evaluateTimestamp("invalid timestamp data", "yyyy/MM/dd HH:mm");
}
@Test
public void testEvaluateTimestamp_NullTimestamp() {
assertNull(HBaseIndexingOptions.evaluateTimestamp(null, null));
}
@Test
public void testHelp() throws Exception {
String[] args = new String[] {"--help"};
assertEquals(0, ToolRunner.run(new Configuration(), new HBaseMapReduceIndexerTool(), args));
}
@Test
public void testHelpWithNonSolrCloud() throws Exception {
String[] args = new String[] {"--help", "--show-non-solr-cloud"};
assertEquals(0, ToolRunner.run(new Configuration(), new HBaseMapReduceIndexerTool(), args));
}
}