return addEs(name, new ThreadPoolExecutor(min, max, timeout, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamingThreadFactory(name)));
}
public TabletServerResourceManager(Instance instance, FileSystem fs) {
this.conf = new ServerConfiguration(instance);
final AccumuloConfiguration acuConf = conf.getConfiguration();
long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM);
boolean usingNativeMap = acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.loadedNativeLibraries();
long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
_iCache = new LruBlockCache(iCacheSize, blockSize);
_dCache = new LruBlockCache(dCacheSize, blockSize);
Runtime runtime = Runtime.getRuntime();
if (!usingNativeMap && maxMemory + dCacheSize + iCacheSize > runtime.maxMemory()) {
throw new IllegalArgumentException(String.format(
"Maximum tablet server map memory %,d and block cache sizes %,d is too large for this JVM configuration %,d", maxMemory, dCacheSize + iCacheSize,
runtime.maxMemory()));
}
runtime.gc();
// totalMemory - freeMemory = memory in use
// maxMemory - memory in use = max available memory
if (!usingNativeMap && maxMemory > runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())) {
log.warn("In-memory map may not fit into local memory space.");
}
minorCompactionThreadPool = createEs(Property.TSERV_MINC_MAXCONCURRENT, "minor compactor");
// make this thread pool have a priority queue... and execute tablets with the most
// files first!
majorCompactionThreadPool = createEs(Property.TSERV_MAJC_MAXCONCURRENT, "major compactor", new CompactionQueue());
rootMajorCompactionThreadPool = createEs(0, 1, 300, "md root major compactor");
defaultMajorCompactionThreadPool = createEs(0, 1, 300, "md major compactor");
splitThreadPool = createEs(1, "splitter");
defaultSplitThreadPool = createEs(0, 1, 60, "md splitter");
defaultMigrationPool = createEs(0, 1, 60, "metadata tablet migration");
migrationPool = createEs(Property.TSERV_MIGRATE_MAXCONCURRENT, "tablet migration");
// not sure if concurrent assignments can run safely... even if they could there is probably no benefit at startup because
// individual tablet servers are already running assignments concurrently... having each individual tablet server run
// concurrent assignments would put more load on the metadata table at startup
assignmentPool = createEs(1, "tablet assignment");
assignMetaDataPool = createEs(0, 1, 60, "metadata tablet assignment");
readAheadThreadPool = createEs(Property.TSERV_READ_AHEAD_MAXCONCURRENT, "tablet read ahead");
defaultReadAheadThreadPool = createEs(Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT, "metadata tablets read ahead");
tabletResources = new HashSet<TabletResourceManager>();
int maxOpenFiles = acuConf.getCount(Property.TSERV_SCAN_MAX_OPENFILES);
fileManager = new FileManager(conf, fs, maxOpenFiles, _dCache, _iCache);
try {
Class<? extends MemoryManager> clazz = AccumuloVFSClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class);
memoryManager = clazz.newInstance();
memoryManager.init(conf);
log.debug("Loaded memory manager : " + memoryManager.getClass().getName());
} catch (Exception e) {
log.error("Failed to find memory manger in config, using default", e);