package com.cloudhopper.datastore.kyoto;
/*
* #%L
* ch-datastore
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.commons.util.HexUtil;
import com.cloudhopper.commons.util.StringUtil;
import com.cloudhopper.datastore.*;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import kyotocabinet.DB;
import kyotocabinet.Error;
/**
* Kyoto Cabinet backed implementation of a DataStore.
*
* @author garth
*/
public class KyotoDataStore extends BaseDataStore implements AscendingIteratorSupport {
private final static Logger logger = LoggerFactory.getLogger(KyotoDataStore.class);
// path used to generate a full path to the database file
private File path;
// the underlying b-tree database
private DB db;
// for performance, the database is not synchronized on every put() and take()
// if -1, sync() is disabled
// if 0, sync() is called on ever put() and take()
// if > 0, the amount of ms between a sync() call is made, set to 5 seconds by default
private long synchronizeInterval;
private Timer synchronizeTimer;
private TimerTask synchronizeTimerTask;
public KyotoDataStore() {
super("KC-DB");
// defaults
synchronizeInterval = 5000;
}
/**
* Returns the underlying B-Tree database. If not opened, this will return
* null.
* @return Null if not open, otherwise the B-Tree database.
*/
public DB getDatabase() {
return db;
}
/**
* Gets the interval between sync() calls to underlying Kyoto cabinet db.
* For performance, by default, the database is not synchronized on every
* put() and take(). If -1, sync() is disabled. If 0, sync() is called on
* every put() and take(). Throughput will be dramatically decreased, but
* it'll guarantee the data makes it into the file. If > 0, its the length
* of ms between a sync() call is made, set to 5 seconds by default
* @return The synchronization interval in milliseconds
*/
public long getSynchronizeInterval() {
return synchronizeInterval;
}
/**
*
* @throws QueueStoreException Thrown if the data store is already open
* when this method is called.
*/
public void setSynchronizeInterval(long value) {
this.synchronizeInterval = value;
}
/**
* Kyoto cabinet determines its database type by the file name extension (.kch
* for Hash and .kct for B+ tree). DO NOT CHANGE THIS.
*/
private File getDatabaseFile() {
return new File(getDirectory(), getName() + ".kct");
}
@Override
public void doDelete() throws DataStoreFatalException {
File databaseFile = getDatabaseFile();
try {
databaseFile.delete();
} catch (Exception e) {
throw new DataStoreFatalException("Unable to delete " + databaseFile.getPath(), e);
}
}
/**
http://fallabs.com/kyotocabinet/spex.html
The file hash database supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "zcomp", and "zkey". The file tree database supports all parameters of the file hash database and "psiz", "rcomp", "pccap" in addition.
MISSING:
tune_options : sets the optional features.
The optional features by `tune_options' is useful to reduce the size of the database file at the expense of scalability or time efficiency. If `HashDB::TSMALL' is specified, the width of record addressing is reduced from 6 bytes to 4 bytes. As the result, the footprint for each record is reduced from 16 bytes to 12 bytes. However, it limits the maximum size of the database file up to 16GB (2GB multiplied by the alignment). If `HashDB::TLINEAR' is specified, the data structure of the collision chain of hash table is changed from binary tree to linear linked list. In that case, the footprint of each record is reduced from 16 bytes to 10 bytes although the time efficiency becomes sensitive to the number of the hash buckets. If `HashDB::TCOMPRESS' is specified, the value of each record is compressed implicitly when stored in the file. If the value is bigger than 1KB or more, compression is effective.
tune_compressor : set the data compressor.
The default compression algorithm of the `HashDB::TCOMPRESS' option is "Deflate" by ZLIB. If you want to use another algorithm, call `tune_compressor' to set a functor which implements compression and decompression functions.
tune_comparator : sets the record comparator.
The default record comparator is the lexical ordering function. That is, records in the B+ tree database are placed in the lexical order of each key. If you want to use another ordering, call `tune_comparator' to set a functor which implements the ordering function.
*/
private final Map<String,String> kprops = new HashMap<String,String>();
/**
* tune_alignment : sets the power of the alignment of record size.
* The default alignment power is 3, which means the address of each record is
* aligned to a multiple of 8 (1<<3) bytes. If you trust that the database is
* constructed at a time and not updated often, call `tune_alignment' to set the
* alignment power 0, which means 1 (1<<0) byte. If the typical size of each record
* is expected to be larger than 1KB, tune the alignment 8 or more.
*/
public void setAlignmentPower(int value) {
kprops.put("apow", Integer.toString(value));
}
/**
* tune_fbp : sets the power of the capacity of the free block pool.
* The tuning of the free block pool by `tune_fbp' does not have to be modified in most
* cases. The default is 10, which means the capacity of the free block pool is 1024
* (1<<10).
*/
public void setFreeBlockPoolPower(int value) {
kprops.put("fpow", Integer.toString(value));
}
/**
* tune_buckets : sets the number of buckets of the hash table.
* The default tuning of the bucket number is about one million. If you intend to store
* more records, call `tune_buckets' to set the bucket number. The suggested ratio of
* the bucket number is the twice of the total number of records and it is okay from
* 100% to 400%. If the ratio decreases smaller than 100%, the time efficiency will
* decrease gradually. If you set the bucket number, setting the `HashDB::TLINEAR'
* option is recommended to improve time and space efficiency.
*/
public void setNumBuckets(int value) {
kprops.put("bnum", Integer.toString(value));
}
/**
* tune_defrag : sets the unit step number of auto defragmentation.
* By default, auto defragmentation is disabled. If the existing records in the database
* are modified (removed or modified with varying the size), fragmentation of available
* regions proceeds gradually. In that case, call `tune_defrag' to enable auto
* defragmentation and set the unit step number. The suggested unit step number is 8,
* which means that a set of defragmentation operations is performed each 8 updating
* operations. The more the unit is, space efficiency becomes higher but time efficiency
* becomes lower.
*/
public void setDefragmentationUnit(int value) {
kprops.put("dfunit", Integer.toString(value));
}
/**
* tune_map : sets the size of the internal memory-mapped region.
* The default tuning of the size of the internal memory-mapped region is 64MB. If the
* database size is expected to be larger than 64MB, call `tune_map to set the map size
* larger than the expected size of the database. Although the capacity of the RAM on
* the machine limits the map size, increasing the map size is effective to improve
* performance.
*/
public void setMmapSize(int value) {
kprops.put("msiz", Integer.toString(value));
}
/**
* tune_page : sets the size of each page.
* The tuning of the page size by `tune_page' does not have to be modified in most cases.
* The default is 8192, which is the twice of the typical page size of popular
* environments. If the size of each node exceeds the parameter, the node is divided into
* two.
*/
public void setPageSize(int value) {
kprops.put("psiz", Integer.toString(value));
}
/**
* tune_page_cache : sets the capacity size of the page cache.
* The default tuning of the capacity size of the page cache is 64MB. If your machine has
* abundant RAM, call `tune_page_cache' to load all nodes on the page cache. If the RAM is
* not abundant, it is better to keep the default page cache size and assign the RAM for
* the internal memory-mapped region by `tune_map'.
*/
public void setPageCacheSize(int value) {
kprops.put("pccap", Integer.toString(value));
}
private String createKyotoPath(File file, Map<String, String> tuningParameters) {
StringBuilder p = new StringBuilder(path.getAbsolutePath());
for (Map.Entry<String,String> e : tuningParameters.entrySet()) {
p.append('#');
p.append(e.getKey()).append("=").append(e.getValue());
}
return p.toString();
}
@Override
public void doOpen() throws DataStoreFatalException {
// create a kyoto cabinet backed queue
db = new DB();
// just leave comparison as LEXICAL, don't set it to anything else
// try to open the database
path = getDatabaseFile();
// if (!db.open(path.getAbsolutePath(), DB.OWRITER | DB.OCREATE)) {
String kyotoPath = createKyotoPath(path, kprops);
logger.debug("Using kyoto path {}", kyotoPath);
if (!db.open(kyotoPath, DB.OWRITER | DB.OCREATE)) {
// error occurred while opening, throw error
Error error = db.error();
db = null;
throw new DataStoreFatalException("Unable to open " + toString() + " @ " + path + " {code=" + error.code() + ", message=" + error.message() + "}");
}
if (this.synchronizeInterval < 0) {
logger.debug("Automatic sync disabled {synchronizeInterval < 0} for " + toString());
} else if (this.synchronizeInterval == 0) {
logger.debug("Automatic sync enabled for every transaction {synchronizeInterval == 0} for " + toString());
} else {
// schedule task to print results every 5 seconds
synchronizeTimer = new Timer("KyotoDataStoreSync-" + getName(), true);
synchronizeTimerTask = new TimerTask() {
@Override
public void run() {
try {
if (db != null) {
logger.trace("Synchronized data store [name=" + getName() + "]");
db.synchronize(false, null);
} else {
// cancel this task!
logger.warn("Cancelling synchronizeTimerTask since db was null, this may be normal if data store shutting down");
this.cancel();
}
} catch (Exception e) {
logger.error("", e);
}
}
};
synchronizeTimer.schedule(synchronizeTimerTask, synchronizeInterval, synchronizeInterval);
logger.debug("Automatic sync enabled every " + synchronizeInterval + " ms for " + toString());
}
}
@Override
public void doClose() throws DataStoreFatalException {
try {
// make sure sync thread is cancelled
if (synchronizeTimerTask != null) {
try {
synchronizeTimerTask.cancel();
} catch (Exception e) {
// ignore this
}
synchronizeTimerTask = null;
}
if (synchronizeTimer != null) {
try {
synchronizeTimer.cancel();
} catch (Exception e) {
// ignore this
}
synchronizeTimer = null;
}
if (db != null) {
if (!db.close()) {
// error occurred while opening, throw error
Error error = db.error();
throw new DataStoreFatalException("Unable to properly close DataStore {code=" + error.code() + ", message=" + error.message() + "}");
}
}
} finally {
db = null;
}
}
@Override
public void doSetRecord(byte[] key, byte[] value) throws DataStoreFatalException {
// db.put = returns true if value put in and didn't replace anything
// returns false if it failed OR if it had to replace a value
if (!db.set(key, value)) {
Error error = db.error();
// this code may be 0 if the value was replaced
if (error.code() == Error.DUPREC) {
// value replaced, ignoring
} else {
throw new DataStoreFatalException("Unable to set value and key [0x" + HexUtil.toHexString(key) + "] [code=" + error.code() + ", message=" + error.message() + "]");
}
}
Error error = db.error();
if (error != null && error.code() != 0) logger.trace("Error on set {}=>{} {} {}", HexUtil.toHexString(key), value.length, error.code(), error);
// if synchronize interval is zero, sync() on every put() and take()
if (this.synchronizeInterval == 0) {
// syncing the b-tree database
db.synchronize(false, null);
}
}
@Override
public byte[] doGetRecord(byte[] key) throws RecordNotFoundException, DataStoreFatalException {
byte[] value = db.get(key);
Error error = db.error();
if (error != null && error.code() != 0) logger.trace("Error on get {} {} {}", HexUtil.toHexString(key), error.code(), error);
if (value == null) {
throw new RecordNotFoundException("The record for key [0x" + HexUtil.toHexString(key) + "] was not found");
}
return value;
}
@Override
public void doDeleteRecord(byte[] key) throws RecordNotFoundException, DataStoreFatalException {
if (!db.remove(key)) {
Error error = db.error();
// this code may be 0 if the value was replaced
if (error.code() == 0) {
// value deleted, ignoring error code
} else if (error.code() == Error.NOREC) {
//[code=22, message=no record found]
throw new RecordNotFoundException("The record for key [0x" + HexUtil.toHexString(key) + "] was not found");
} else {
throw new DataStoreFatalException("Unable to delete record with key [0x" + HexUtil.toHexString(key) + "] [code=" + error.code() + ", message=" + error.message() + "]");
}
}
Error error = db.error();
if (error != null && error.code() != 0) logger.trace("Error on remove {} {} {}", HexUtil.toHexString(key), error.code(), error);
// if synchronize interval is zero, sync() on every put() and take()
if (this.synchronizeInterval == 0) {
// syncing the b-tree database
db.synchronize(false, null);
}
}
public DataStoreIterator doGetAscendingIterator() throws DataStoreFatalException {
return new KyotoDataStoreIterator(getDatabase());
}
}