package org.apache.blur.thrift;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.xml.parsers.FactoryConfigurationError;
import org.apache.blur.BlurConfiguration;
import org.apache.blur.analysis.FieldManager;
import org.apache.blur.analysis.FieldTypeDefinition;
import org.apache.blur.log.Log;
import org.apache.blur.log.LogFactory;
import org.apache.blur.manager.clusterstatus.ClusterStatus;
import org.apache.blur.server.TableContext;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.blur.thrift.generated.ColumnDefinition;
import org.apache.blur.thrift.generated.Level;
import org.apache.blur.thrift.generated.Metric;
import org.apache.blur.thrift.generated.Schema;
import org.apache.blur.thrift.generated.Selector;
import org.apache.blur.thrift.generated.ShardState;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.trace.Trace;
import org.apache.blur.trace.TraceStorage;
import org.apache.blur.utils.BlurUtil;
import org.apache.blur.utils.MemoryReporter;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;
import org.apache.zookeeper.ZooKeeper;
public abstract class TableAdmin implements Iface {
private static final Log LOG = LogFactory.getLog(TableAdmin.class);
protected ZooKeeper _zookeeper;
protected ClusterStatus _clusterStatus;
protected BlurConfiguration _configuration;
protected int _maxRecordsPerRowFetchRequest = 1000;
protected void checkSelectorFetchSize(Selector selector) {
if (selector == null) {
return;
}
int maxRecordsToFetch = selector.getMaxRecordsToFetch();
if (maxRecordsToFetch > _maxRecordsPerRowFetchRequest) {
LOG.warn("Max records to fetch is too high [{0}] max [{1}] in Selector [{2}]", maxRecordsToFetch,
_maxRecordsPerRowFetchRequest, selector);
selector.setMaxRecordsToFetch(_maxRecordsPerRowFetchRequest);
}
}
@Override
public Map<String, Metric> metrics(Set<String> metrics) throws BlurException, TException {
try {
Map<String, Metric> metricsMap = MemoryReporter.getMetrics();
if (metrics == null) {
return metricsMap;
} else {
Map<String, Metric> result = new HashMap<String, Metric>();
for (String n : metrics) {
Metric metric = metricsMap.get(n);
if (metric != null) {
result.put(n, metric);
}
}
return result;
}
} catch (Exception e) {
LOG.error("Unknown error while trying to get metrics [{0}] ", e, metrics);
throw new BException(e.getMessage(), e);
}
}
@Override
public boolean isInSafeMode(String cluster) throws BlurException, TException {
try {
return _clusterStatus.isInSafeMode(true, cluster);
} catch (Exception e) {
LOG.error("Unknown error during safe mode check of [cluster={0}]", e, cluster);
throw new BException(e.getMessage(), e);
}
}
@Override
public final void createTable(TableDescriptor tableDescriptor) throws BlurException, TException {
try {
TableContext.clear(tableDescriptor.getName());
BlurUtil.validateTableName(tableDescriptor.getName());
assignClusterIfNull(tableDescriptor);
_clusterStatus.createTable(tableDescriptor);
} catch (Exception e) {
LOG.error("Unknown error during create of [table={0}, tableDescriptor={1}]", e, tableDescriptor.name,
tableDescriptor);
throw new BException(e.getMessage(), e);
}
if (tableDescriptor.isEnabled()) {
enableTable(tableDescriptor.getName());
}
}
private void assignClusterIfNull(TableDescriptor tableDescriptor) throws BlurException, TException {
if (tableDescriptor.getCluster() == null) {
List<String> shardClusterList = shardClusterList();
if (shardClusterList != null && shardClusterList.size() == 1) {
String cluster = shardClusterList.get(0);
tableDescriptor.setCluster(cluster);
LOG.info("Assigning table [{0}] to the single default cluster [{1}]", tableDescriptor.getName(), cluster);
}
}
}
@Override
public final void disableTable(String table) throws BlurException, TException {
try {
TableContext.clear(table);
String cluster = _clusterStatus.getCluster(false, table);
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
_clusterStatus.disableTable(cluster, table);
waitForTheTableToDisable(cluster, table);
waitForTheTableToDisengage(cluster, table);
} catch (Exception e) {
LOG.error("Unknown error during disable of [table={0}]", e, table);
throw new BException(e.getMessage(), e);
}
}
@Override
public final void enableTable(String table) throws BlurException, TException {
try {
TableContext.clear(table);
String cluster = _clusterStatus.getCluster(false, table);
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
_clusterStatus.enableTable(cluster, table);
waitForTheTableToEnable(cluster, table);
waitForTheTableToEngage(cluster, table);
} catch (Exception e) {
LOG.error("Unknown error during enable of [table={0}]", e, table);
throw new BException(e.getMessage(), e);
}
}
private void waitForTheTableToEnable(String cluster, String table) throws BlurException {
LOG.info("Waiting for shards to engage on table [" + table + "]");
while (true) {
if (_clusterStatus.isEnabled(false, cluster, table)) {
return;
}
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
LOG.error("Unknown error while enabling table [" + table + "]", e);
throw new BException("Unknown error while enabling table [" + table + "]", e);
}
}
}
/**
* This method only works on controllers, if called on shard servers it will
* only wait itself to finish not the whole cluster.
*/
private void waitForTheTableToEngage(String cluster, String table) throws BlurException, TException {
TableDescriptor describe = describe(table);
int shardCount = describe.shardCount;
LOG.info("Waiting for shards to engage on table [" + table + "]");
while (true) {
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
LOG.error("Unknown error while engaging table [" + table + "]", e);
throw new BException("Unknown error while engaging table [" + table + "]", e);
}
try {
Map<String, Map<String, ShardState>> shardServerLayoutState = shardServerLayoutState(table);
int countNumberOfOpen = 0;
int countNumberOfOpening = 0;
for (Entry<String, Map<String, ShardState>> shardEntry : shardServerLayoutState.entrySet()) {
Map<String, ShardState> value = shardEntry.getValue();
for (ShardState state : value.values()) {
if (state == ShardState.OPEN) {
countNumberOfOpen++;
} else if (state == ShardState.OPENING) {
countNumberOfOpening++;
} else {
LOG.warn("Unexpected state of [{0}] for shard [{1}].", state, shardEntry.getKey());
}
}
}
LOG.info("Opening - Shards Open [{0}], Shards Opening [{1}] of table [{2}]", countNumberOfOpen,
countNumberOfOpening, table);
if (countNumberOfOpen == shardCount && countNumberOfOpening == 0) {
return;
}
} catch (BlurException e) {
LOG.info("Stilling waiting", e);
} catch (TException e) {
LOG.info("Stilling waiting", e);
}
}
}
/**
* This method only works on controllers, if called on shard servers it will
* only wait itself to finish not the whole cluster.
*/
private void waitForTheTableToDisengage(String cluster, String table) throws BlurException, TException {
LOG.info("Waiting for shards to disengage on table [" + table + "]");
while (true) {
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
LOG.error("Unknown error while disengaging table [" + table + "]", e);
throw new BException("Unknown error while disengaging table [" + table + "]", e);
}
try {
Map<String, Map<String, ShardState>> shardServerLayoutState = shardServerLayoutState(table);
int countNumberOfOpen = 0;
int countNumberOfClosing = 0;
for (Entry<String, Map<String, ShardState>> shardEntry : shardServerLayoutState.entrySet()) {
Map<String, ShardState> value = shardEntry.getValue();
for (ShardState state : value.values()) {
if (state == ShardState.OPEN) {
countNumberOfOpen++;
} else if (state == ShardState.CLOSING) {
countNumberOfClosing++;
} else if (state == ShardState.CLOSED) {
LOG.info("Shard [{0}] of table [{1}] now reporting closed.", shardEntry.getKey(), table);
} else {
LOG.warn("Unexpected state of [{0}] for shard [{1}].", state, shardEntry.getKey());
}
}
}
LOG.info("Closing - Shards Open [{0}], Shards Closing [{1}] of table [{2}]", countNumberOfOpen,
countNumberOfClosing, table);
if (countNumberOfOpen == 0 && countNumberOfClosing == 0) {
return;
}
} catch (BlurException e) {
LOG.info("Stilling waiting", e);
} catch (TException e) {
LOG.info("Stilling waiting", e);
}
}
}
private void waitForTheTableToDisable(String cluster, String table) throws BlurException, TException {
LOG.info("Waiting for shards to disable on table [" + table + "]");
while (true) {
if (!_clusterStatus.isEnabled(false, cluster, table)) {
return;
}
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
LOG.error("Unknown error while enabling table [" + table + "]", e);
throw new BException("Unknown error while enabling table [" + table + "]", e);
}
}
}
@Override
public final void removeTable(String table, boolean deleteIndexFiles) throws BlurException, TException {
try {
TableContext.clear(table);
String cluster = _clusterStatus.getCluster(false, table);
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
_clusterStatus.removeTable(cluster, table, deleteIndexFiles);
} catch (Exception e) {
LOG.error("Unknown error during remove of [table={0}]", e, table);
throw new BException(e.getMessage(), e);
}
}
public boolean isTableEnabled(boolean useCache, String cluster, String table) {
return _clusterStatus.isEnabled(useCache, cluster, table);
}
public void checkTable(String table) throws BlurException {
if (table == null) {
throw new BException("Table cannot be null.");
}
String cluster = _clusterStatus.getCluster(true, table);
if (cluster == null) {
throw new BException("Table [" + table + "] does not exist");
}
checkTable(cluster, table);
}
public void checkTable(String cluster, String table) throws BlurException {
if (inSafeMode(true, table)) {
throw new BException("Cluster for [" + table + "] is in safe mode");
}
if (tableExists(true, cluster, table)) {
if (isTableEnabled(true, cluster, table)) {
return;
}
throw new BException("Table [" + table + "] exists, but is not enabled");
} else {
throw new BException("Table [" + table + "] does not exist");
}
}
public void checkForUpdates(String table) throws BlurException {
String cluster = _clusterStatus.getCluster(true, table);
if (cluster == null) {
throw new BException("Table [" + table + "] does not exist");
}
checkForUpdates(cluster, table);
}
public void checkForUpdates(String cluster, String table) throws BlurException {
if (_clusterStatus.isReadOnly(true, cluster, table)) {
throw new BException("Table [" + table + "] in cluster [" + cluster + "] is read only.");
}
}
@Override
public final List<String> controllerServerList() throws BlurException, TException {
try {
return _clusterStatus.getOnlineControllerList();
} catch (Exception e) {
LOG.error("Unknown error while trying to get a controller list.", e);
throw new BException("Unknown error while trying to get a controller list.", e);
}
}
@Override
public final List<String> shardServerList(String cluster) throws BlurException, TException {
try {
return _clusterStatus.getShardServerList(cluster);
} catch (Exception e) {
LOG.error("Unknown error while trying to get a shard server list.", e);
throw new BException("Unknown error while trying to get a shard server list.", e);
}
}
@Override
public final List<String> shardClusterList() throws BlurException, TException {
try {
return _clusterStatus.getClusterList(true);
} catch (Exception e) {
LOG.error("Unknown error while trying to get a cluster list.", e);
throw new BException("Unknown error while trying to get a cluster list.", e);
}
}
@Override
public final TableDescriptor describe(final String table) throws BlurException, TException {
try {
String cluster = _clusterStatus.getCluster(true, table);
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table);
TableContext tableContext = TableContext.create(tableDescriptor);
return tableContext.getDescriptor();
} catch (Exception e) {
LOG.error("Unknown error while trying to describe a table [" + table + "].", e);
throw new BException("Unknown error while trying to describe a table [" + table + "].", e);
}
}
@Override
public final List<String> tableListByCluster(String cluster) throws BlurException, TException {
try {
return _clusterStatus.getTableList(true, cluster);
} catch (Exception e) {
LOG.error("Unknown error while trying to get a table list by cluster [" + cluster + "].", e);
throw new BException("Unknown error while trying to get a table list by cluster [" + cluster + "].", e);
}
}
@Override
public final List<String> tableList() throws BlurException, TException {
try {
return _clusterStatus.getTableList(true);
} catch (Exception e) {
LOG.error("Unknown error while trying to get a table list.", e);
throw new BException("Unknown error while trying to get a table list.", e);
}
}
@Override
public boolean addColumnDefinition(String table, ColumnDefinition columnDefinition) throws BlurException, TException {
if (table == null) {
throw new BException("Table cannot be null.");
}
if (columnDefinition == null) {
throw new BException("ColumnDefinition cannot be null.");
}
TableDescriptor tableDescriptor = describe(table);
TableContext context = TableContext.create(tableDescriptor);
FieldManager fieldManager = context.getFieldManager();
String family = columnDefinition.getFamily();
if (family == null) {
throw new BException("Family in ColumnDefinition [{0}] cannot be null.", columnDefinition);
}
String columnName = columnDefinition.getColumnName();
if (columnName == null) {
throw new BException("ColumnName in ColumnDefinition [{0}] cannot be null.", columnDefinition);
}
String subColumnName = columnDefinition.getSubColumnName();
boolean fieldLessIndexed = columnDefinition.isFieldLessIndexed();
String fieldType = columnDefinition.getFieldType();
if (fieldType == null) {
throw new BException("FieldType in ColumnDefinition [{0}] cannot be null.", columnDefinition);
}
boolean sortable = columnDefinition.isSortable();
Map<String, String> props = columnDefinition.getProperties();
try {
return fieldManager.addColumnDefinition(family, columnName, subColumnName, fieldLessIndexed, fieldType, sortable,
props);
} catch (IOException e) {
throw new BException(
"Unknown error while trying to addColumnDefinition on table [{0}] with columnDefinition [{1}]", e, table,
columnDefinition);
}
}
@Override
public List<String> traceList() throws BlurException, TException {
TraceStorage storage = Trace.getStorage();
try {
return storage.getTraceIds();
} catch (Exception e) {
throw new BException("Unknown error while trying to get traceList", e);
}
}
@Override
public List<String> traceRequestList(String traceId) throws BlurException, TException {
TraceStorage storage = Trace.getStorage();
try {
return storage.getRequestIds(traceId);
} catch (Exception e) {
throw new BException("Unknown error while trying to get traceRequestList for traceId [{0}]", e, traceId);
}
}
@Override
public String traceRequestFetch(String traceId, String requestId) throws BlurException, TException {
TraceStorage storage = Trace.getStorage();
try {
return storage.getRequestContentsJson(traceId, requestId);
} catch (Exception e) {
throw new BException("Unknown error while trying to get traceRequestList for traceId [{0}] requestId [{1}]", e,
traceId, requestId);
}
}
@Override
public void traceRemove(String traceId) throws BlurException, TException {
TraceStorage storage = Trace.getStorage();
try {
storage.removeTrace(traceId);
} catch (Exception e) {
throw new BException("Unknown error while trying to get remove trace [{0}]", e, traceId);
}
}
protected boolean inSafeMode(boolean useCache, String table) throws BlurException {
String cluster = _clusterStatus.getCluster(useCache, table);
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
return _clusterStatus.isInSafeMode(useCache, cluster);
}
public boolean tableExists(boolean useCache, String cluster, String table) {
return _clusterStatus.exists(useCache, cluster, table);
}
public ClusterStatus getClusterStatus() {
return _clusterStatus;
}
public void setClusterStatus(ClusterStatus clusterStatus) {
_clusterStatus = clusterStatus;
}
public void setZookeeper(ZooKeeper zookeeper) {
_zookeeper = zookeeper;
}
public void setConfiguration(BlurConfiguration config) {
_configuration = config;
}
@Override
public Map<String, String> configuration() throws BlurException, TException {
return _configuration.getProperties();
}
public int getMaxRecordsPerRowFetchRequest() {
return _maxRecordsPerRowFetchRequest;
}
public void setMaxRecordsPerRowFetchRequest(int _maxRecordsPerRowFetchRequest) {
this._maxRecordsPerRowFetchRequest = _maxRecordsPerRowFetchRequest;
}
@Override
public Schema schema(String table) throws BlurException, TException {
checkTable(table);
try {
TableContext tableContext = getTableContext(table);
FieldManager fieldManager = tableContext.getFieldManager();
fieldManager.loadFromStorage();
Schema schema = new Schema().setTable(table);
schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>());
Set<String> fieldNames = fieldManager.getFieldNames();
INNER: for (String fieldName : fieldNames) {
FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName);
if (fieldTypeDefinition == null) {
continue INNER;
}
String columnName = fieldTypeDefinition.getColumnName();
String columnFamily = fieldTypeDefinition.getFamily();
String subColumnName = fieldTypeDefinition.getSubColumnName();
Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily);
if (map == null) {
map = new HashMap<String, ColumnDefinition>();
schema.putToFamilies(columnFamily, map);
}
if (subColumnName == null) {
map.put(columnName, getColumnDefinition(fieldTypeDefinition));
} else {
map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition));
}
}
return schema;
} catch (Exception e) {
throw new BException("Unknown error while trying to get schema for table [{0}]", table);
}
}
private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition) {
ColumnDefinition columnDefinition = new ColumnDefinition();
columnDefinition.setFamily(fieldTypeDefinition.getFamily());
columnDefinition.setColumnName(fieldTypeDefinition.getColumnName());
columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName());
columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed());
columnDefinition.setFieldType(fieldTypeDefinition.getFieldType());
columnDefinition.setSortable(fieldTypeDefinition.isSortEnable());
columnDefinition.setProperties(fieldTypeDefinition.getProperties());
return columnDefinition;
}
private TableContext getTableContext(final String table) {
return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true, table), table));
}
@Override
public void ping() throws TException {
}
@Override
public void logging(String classNameOrLoggerName, Level level) throws BlurException, TException {
Logger logger;
if (classNameOrLoggerName == null) {
logger = LogManager.getRootLogger();
} else {
logger = LogManager.getLogger(classNameOrLoggerName);
}
if (logger == null) {
throw new BException("Logger [{0}] not found.", classNameOrLoggerName);
}
org.apache.log4j.Level current = logger.getLevel();
org.apache.log4j.Level newLevel = getLevel(level);
LOG.info("Changing Logger [{0}] from logging level [{1}] to [{2}]", logger, current, newLevel);
logger.setLevel(newLevel);
}
@Override
public void resetLogging() throws BlurException, TException {
try {
reloadConfig();
} catch (MalformedURLException e) {
throw new BException("Unknown error while trying to reload log4j config.");
} catch (FactoryConfigurationError e) {
throw new BException("Unknown error while trying to reload log4j config.");
}
}
private void reloadConfig() throws MalformedURLException, FactoryConfigurationError, BException {
String blurHome = System.getenv("BLUR_HOME");
if (blurHome != null) {
File blurHomeFile = new File(blurHome);
if (blurHomeFile.exists()) {
File log4jFile = new File(new File(blurHomeFile, "conf"), "log4j.xml");
if (log4jFile.exists()) {
LOG.info("Reseting log4j config from [{0}]", log4jFile);
LogManager.resetConfiguration();
DOMConfigurator.configure(log4jFile.toURI().toURL());
return;
}
}
}
URL url = TableAdmin.class.getResource("/log4j.xml");
if (url != null) {
LOG.info("Reseting log4j config from classpath resource [{0}]", url);
LogManager.resetConfiguration();
DOMConfigurator.configure(url);
return;
}
throw new BException("Could not locate log4j file to reload, doing nothing.");
}
private org.apache.log4j.Level getLevel(Level level) throws BlurException {
switch (level) {
case ALL:
return org.apache.log4j.Level.ALL;
case DEBUG:
return org.apache.log4j.Level.DEBUG;
case ERROR:
return org.apache.log4j.Level.ERROR;
case FATAL:
return org.apache.log4j.Level.FATAL;
case INFO:
return org.apache.log4j.Level.INFO;
case TRACE:
return org.apache.log4j.Level.TRACE;
case OFF:
return org.apache.log4j.Level.OFF;
case WARN:
return org.apache.log4j.Level.WARN;
default:
throw new BException("Level [{0}] not found.", level);
}
}
}