/*
* Copyright (C) 2006 http://www.chaidb.org
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
package org.chaidb.db.log;
import org.apache.log4j.Logger;
import org.chaidb.db.DBState;
import org.chaidb.db.exception.ChaiDBException;
import org.chaidb.db.exception.ErrorCode;
import org.chaidb.db.helper.AbstractDaemonThread;
import org.chaidb.db.helper.ByteTool;
import org.chaidb.db.helper.Config;
import org.chaidb.db.helper.DaemonThreadManager;
import org.chaidb.db.index.btree.BTreeSpec;
import org.chaidb.db.log.logrecord.TxnCkpLogRecord;
import org.chaidb.db.log.logrecord.TxnFuzzyCkpLogRecord;
import org.chaidb.db.transaction.TransactionImpl;
import org.chaidb.db.transaction.TransactionManager;
import java.io.File;
import java.util.Date;
import java.util.Stack;
/**
* LogManagerImpl implemens interface LogManager. Comments of all methods
* implemented from LogManager refer to those in interface.
* Inner classes:
* Class RecordBuffer(WriteBuffer): saving new log record from BTree or TXN
* Class ReadCache(ReadBuffer): saving old log record read from log file
*/
public class LogManagerImpl implements LogManager {
private static final Logger logger = Logger.getLogger(LogManagerImpl.class);
/**
* The lsn LSN is the file offset that we're about to write and which we will return to the user.
*/
private Lsn curLsn;
/**
* Lsn that we last read. Used in get method
*/
private Lsn rLsn = null;
/**
* offset of the last lsn. In fact, we shoule keep in mind that it is consistent with the last
* record in log file. When records are written successfully, prevOffset ought to be updated in
* config file.
*/
private int prevOffset;
/**
* Last checkpoint's lsn
*/
private Lsn lastChkpt;
/**
* Current checkpoint's lsn
*/
private Lsn chkpt;
/**
* Current checkpoint's time
*/
private Date chkptTime;
/**
* Bytes to log since the last checkpoint.
*/
volatile private int chkptBytes;
/**
* Megabytes to log since the last checkpoint.
*/
volatile private int chkptMbytes;
/**
* Log Writer
*/
private LogFile lgWr;
/**
* Transaction Manager
*/
private TransactionManager txnMgr;
/**
* In memory log record buffer.
*/
private RecordBuffer buffer;
private Stack bufferStack = new Stack();
private final byte[] BUFFER_WRITER_OBJ = new byte[0];
/**
* In memory log record cache for read.
*/
private ReadCache readBuf;
/**
* lastest lsn while last flush,added by marriane 2001-12-3 for archive
*/
private Lsn lastLsnInFlush;
/* added by marriane 2002-1-8 for judge normal or shutdown checkpoint */
private boolean shutDown = false;
/**
* Max log file number
*/
private static final int MAX_LOG_FILE_NUMBER = Short.MAX_VALUE;//(int)Math.pow(10,LogFile.LOG_NAME_NUM);
// private static final int BTREE_FILEPATH_CACHE_SIZE = 1024;
private static final int LOG_FILE_MIN_SIZE = 1024 * 1024;//min size is 1M
public static final int LOG_FILE_MAX_SIZE = getLogFileSize();
static final int LOG_BUFFER_SIZE = Config.getConfig("datalog.write.buffer.size", 3 * 1024 * 1024); /* 3M */
static final int LOG_READBUFFER_MAX_SIZE = Config.getConfig("datalog.read.buffer.size", 64 * 1024); /* 64 KB. */
/* Threashhold of rec_buf to start to write records to disk */
static final double THRESHOLD = Config.getConfig("datalog.threshold", 0.9); //0.9 is optional.
/* it must be large than 3*BTreeSpec.PAGE_SIZE for read large record
like BTreeMergeLogRecord */
public static final int READ_LOG_CHUNK_SIZE = Config.getConfig("datalog.read.chunk", 3 * BTreeSpec.PAGE_SIZE);
/* max log record size */
static final int MAX_LOG_RECORD_SIZE = 3 * BTreeSpec.PAGE_SIZE;
private static LogManagerImpl logManagerImpl = null;
/* This object is used to sync operations and vars of checkpoint */
public static final byte[] CHECKPOINT_SYNC_OBJ = new byte[0];
/**
* get the largest file length in datalog directory
*
* @return long largest file length
*/
private static int getLargetFileLengthInDir() {
int largestLen = LOG_FILE_MIN_SIZE;
String logFilePath = DefaultLogFile.getInstance().getLogFilePath();
File dirFile = new File(logFilePath);
File[] fileArr = dirFile.listFiles();
if (fileArr != null) {
for (int i = 0; i < fileArr.length; i++) {
File tempFile = fileArr[i];
if (tempFile.isDirectory()) {
continue;
} else if (tempFile.getName().startsWith("log.")) {
int len = (int) tempFile.length();
if (len > largestLen) {
largestLen = len;
}
}
}
}
return largestLen;
}
private static int getLogFileSize() {
int size = Config.getConfig("datalog.file.size", 200) * 1024 * 1024; /* 200 MB. */
int largestFileLen = getLargetFileLengthInDir() - Lsn.getLsnLength();
if (size < LOG_FILE_MIN_SIZE || size < largestFileLen) {
logger.fatal("'datalog.file.size' in chaidb.conf is incorrect. " + "It's value must between the min value and the max value, " + "and should not less then the existed biggest datalog file. " + "Server will now exit, please give a reasonable value and try again.");
System.exit(-1);
}
return size;
}
/**
* Create a LogManager and initialize it.
*/
public LogManagerImpl() {
try {
initialize();
} catch (ChaiDBException e) {
logger.error("LogManagerImpl construct error:" + e.getMessage());
}
}
public static LogManagerImpl getInstance() {
if (logManagerImpl == null) {
logManagerImpl = new LogManagerImpl();
}
return logManagerImpl;
}
//only be called by CatastrophicTxnRecoverImpl.recoverToCheckpoint
public void setCurLsn(Lsn curLsn, int prevOffset) {
this.curLsn = new Lsn(curLsn);
this.prevOffset = prevOffset;
}
/**
* Restore LogManager's environment from configure file,in the form of key/value pairs,
* such as buffersize=32768. In fact this file is a property list, being abide by property file.
*
* @see java.util.Properties
*/
public synchronized void initialize() throws ChaiDBException {
lgWr = DefaultLogFile.getInstance();
bufferStack.push(new RecordBuffer());
bufferStack.push(new RecordBuffer());
buffer = (RecordBuffer) bufferStack.pop();
readBuf = new ReadCache(lgWr);
int fileId;
Lsn lastLsn = null;
int[] endOffset = new int[1];
lastLsn = DefaultLogFile.getInstance().getLastLsnInDir(endOffset);
fileId = lastLsn.getFileId();
prevOffset = lastLsn == null ? FIRSTREC_PREVOFFSET : lastLsn.getOffset();
int off = 0;
if ((prevOffset == FIRSTREC_PREVOFFSET) && (fileId == FIRSTREC_FILEID)) {
off = FIRSTREC_PREVOFFSET;
} else {
/* it's not correct that we get next log record offset from log file
length,because maybe log file has some error message,and end not
correctly,we must get off from lastest complete log record.
*/
off = endOffset[0];
if (DefaultLogFile.getNewestFileIdFromStateFile() != fileId) {
DefaultLogFile.writeNewestFileIdToFile(fileId);
}
}
curLsn = new Lsn(fileId, off);
lastLsnInFlush = lastLsn;
lastChkpt = new Lsn(FIRSTREC_LSN);
BufferWriter bufWriter = new BufferWriter(this);
bufWriter.start();
}
/**
* set latest checkpoint lsn when do recovery
*
* @param newLsn
*/
public void setLastCheckpointLsn(Lsn newLsn) {
synchronized (CHECKPOINT_SYNC_OBJ) {
this.lastChkpt = new Lsn(newLsn);
}
}
/**
* set latest checkpoint(chkpt) and second-to-last checkpoint(lastChkpt)
* value from log file while system restart
*/
public void setCheckpointValueFromDisk() throws ChaiDBException {
synchronized (CHECKPOINT_SYNC_OBJ) {
chkpt = new Lsn(FIRSTREC_LSN);
lastChkpt = new Lsn(FIRSTREC_LSN);
try {
byte[] stateValue = DBState.getInstance().getLatestCheckPoint();
int file_id = (int) (ByteTool.bytesToShort(stateValue, 0, true));
int off_set = ByteTool.bytesToInt(stateValue, 2, true);
if (file_id >= FIRSTREC_FILEID && off_set != FIRSTREC_PREVOFFSET) {
chkpt = new Lsn(file_id, off_set);
LogRecord cursorLogRecord = get(chkpt);
TxnCkpLogRecord logRec = (TxnCkpLogRecord) cursorLogRecord;
lastChkpt = logRec.getLastCkpLsn();
}
} catch (Exception e) {
logger.error(e);
Lsn cursorLsn = new Lsn(lastLsnInFlush);
LogRecord cursorLogRecord = null;
boolean isEnd = false;
while (!isEnd) {
try {
cursorLogRecord = get(cursorLsn);
} catch (ChaiDBException e1) {
throw e1;
}
if (cursorLogRecord.getType() == LogRecord.LOG_TXN_CHECKPOINT || cursorLogRecord.getType() == LogRecord.LOG_TXN_FUZZY_CHECKPOINT) {
chkpt = cursorLsn;
TxnCkpLogRecord logRec = (TxnCkpLogRecord) cursorLogRecord;
lastChkpt = logRec.getLastCkpLsn();//modified by marriane 2002-10-16
isEnd = true;
}
if (cursorLsn.getOffset() == LogManager.FIRSTREC_PREVOFFSET) {
isEnd = true;
}
cursorLsn = new Lsn(cursorLsn.getFileId(), cursorLogRecord.getHeader().getPrevOffset());
}//while
} //try
} //sync
}
public boolean isShutDown() {
return shutDown;
}
/**
* close log file
*/
public void closeLogFile() throws ChaiDBException {
lgWr.closeLogFile();
}
public void setTxnManager(TransactionManager tm) {
txnMgr = tm;
}
public LogFile getLogWriter() {
return lgWr;
}
/**
* get lastest lsn while last flush,added by marriane 2001-12-3 for archive
* or get lastest lsn in log file while system initialize,for recover
*/
public Lsn getLastLsnInFlush() {
synchronized (BUFFER_WRITER_OBJ) {
return new Lsn(lastLsnInFlush);
}
}
public void setLastLsnInFlush(Lsn lsn) {
synchronized (BUFFER_WRITER_OBJ) {
this.lastLsnInFlush = new Lsn(lsn);
}
}
/**
* This method just be invoked by doCheckpoint routine located in the Checkpoint class.
* After the doCheckpoint routine insertted the log record of checkpoint type into log, the method should be invoked immediately.
*/
public void initChkptBytes() {
chkptBytes = chkptMbytes = 0;
}
public int compare(Lsn lsn0, Lsn lsn1) {
return lsn0.compare(lsn1);
}
public int getChkptBytes() {
return chkptBytes;
}
public int getChkptMBytes() {
return chkptMbytes;
}
/**
* get second-to-last checkpoint
* added for archive
*
* @return lsn
*/
public Lsn getSecondToLastCheckpoint() {
synchronized (CHECKPOINT_SYNC_OBJ) {
return lastChkpt;
}
}
/**
* get latest checkpoint lsn
* added for transaction manager while system restart
*
* @return Lsn
*/
public Lsn getLatestCheckpoint() {
synchronized (CHECKPOINT_SYNC_OBJ) {
return chkpt;
}
}
private static final int _M = 1024 * 1024;
/**
* Accumulate bytes since the last chkpt
*
* @param bytes bytes to be accumulated
*/
void addChkptBytes(int bytes) {
synchronized (CHECKPOINT_SYNC_OBJ) {
chkptBytes += bytes;
if (chkptBytes > _M) {
int mega = chkptBytes / (_M);
chkptMbytes += mega;
//Modified by Ben
chkptBytes -= _M * mega;
}
}
}
/**
* put a log record to log write buffer,if buffer get into threshold,
* flush all log record in write buffer to disk.
*
* @param data
* @param flags
* @return Lsn current log record's lsn
*/
public Lsn put(LogRecord data, int flags) throws ChaiDBException {
Lsn result;
if (flags == LOG_CHECKPOINT) {
synchronized (this) {
result = _putCheckPoint(data);
}//sync
} else {
TransactionImpl txn = txnMgr.getTxn(data.getTxnId());
synchronized (this) {
result = _put(flags, data, txn);
}//sync
}
return result;
}
private static Lsn _INVALID_LSN = new Lsn(INVALID_LSN_FILE_ID, INVALID_LSN_OFFSET);
private Lsn _put(int flags, LogRecord data, TransactionImpl txn) throws ChaiDBException {
if (shutDown) {
String detail = "ChaiDB has been shut down,couldn't put log to buffer any longer";
throw new ChaiDBException(ErrorCode.DB_SERVER_SHUTDOWN, detail);
}
if (flags == LOG_CURLSN) {
return curLsn;//it's not used now
}
//modified by marriane 2001-9-27 for integrate with Transaction Manger
if (txn == null) {
String details = "the transaction id is " + data.getTxnId() + ".";
throw new ChaiDBException(ErrorCode.TXN_NOT_EXIST, details);
}
Lsn lastLsn = _put(data, txn);
/* set begin lsn and last lsn of transaction */
if (txn.getBeginLsn().compare(_INVALID_LSN) == 0) {
txn.setBeginLsn(lastLsn);
}
txn.setLastLsn(lastLsn);
if (flags == LOG_FLUSH) {
_flush();
}
return lastLsn;
}
private Lsn _put(LogRecord data, TransactionImpl txn) throws ChaiDBException {
int recordLen = data.getHeader().getLength();
/* To see if record is larger than buffer in memory. */
if (recordLen > LOG_BUFFER_SIZE) {
String details = "The record length is " + recordLen + ".";
throw new ChaiDBException(ErrorCode.LOG_RECORD_TOO_LARGE, details);
}
//long record will be put to next file by Kurt 2004-7-19
if (recordLen + curLsn.getOffset() > LOG_FILE_MAX_SIZE) {
int fileid = curLsn.getFileId();
if (fileid == MAX_LOG_FILE_NUMBER) {
/* reset file id from NEW_LOG_FILEID */
curLsn.setFileId(FIRSTREC_FILEID);
} else {
curLsn.setFileId(fileid + 1);
}
prevOffset = -prevOffset;
curLsn.setOffset(0);
}
if (txn != null) {
//store the previous
data.setPrevLsn(txn.getLastLsn());
}
data.getHeader().setPrevOffset(prevOffset);
byte[] recordAsBytes = new byte[recordLen];
data.toBytes(recordAsBytes, 0);
//the flag indicating if records need writting to disk
if (!(buffer.put(curLsn, recordAsBytes, recordLen))) {
synchronized (BUFFER_WRITER_OBJ) {
while (bufferStack.size() == 0) {
try {
BUFFER_WRITER_OBJ.wait();
} catch (InterruptedException e) {
logger.warn("LogManager is interrupted.");
}
}
BufferWriter bufWriter = (BufferWriter) DaemonThreadManager.getInstance().getThread(BufferWriter.THREAD_NAME);
bufWriter.flushBuffer(buffer);
buffer = (RecordBuffer) bufferStack.pop();
}
buffer.put(curLsn, recordAsBytes, recordLen);
}
addChkptBytes(recordLen);
/** copy the curLsn before updating it */
Lsn lastLsn = new Lsn(curLsn);
prevOffset = curLsn.getOffset();
curLsn.setOffset(prevOffset + recordLen);
return lastLsn;
}
private Lsn _putCheckPoint(LogRecord data) throws ChaiDBException {
if (shutDown) {
String detail = "ChaiDB has been shut down,couldn't put log to buffer any longer";
throw new ChaiDBException(ErrorCode.DB_SERVER_SHUTDOWN, detail);
}
// saveBTreeFileName(data.getBTreeFileName());
TxnFuzzyCkpLogRecord lr = (TxnFuzzyCkpLogRecord) data;
if (lr.getSmallestLsn() == null) {
lr.setSmallestLsn(curLsn);
}
if (lr.getFlag() == TxnCkpLogRecord.SHUTDOWN_CHECKPOINT) {
shutDown = true;
}
Lsn lastLsn = _put(data, null);
_flush();
synchronized (CHECKPOINT_SYNC_OBJ) {
lastChkpt = chkpt;
chkpt = lastLsn;
if (chkptTime == null) chkptTime = new Date();
else chkptTime.setTime(System.currentTimeMillis());
}
return lastLsn;
}
public boolean needCheckpoint() {
synchronized (this) {
synchronized (CHECKPOINT_SYNC_OBJ) {
Lsn lastLsn = new Lsn(curLsn.getFileId(), prevOffset);
if (lastLsn.compare(chkpt) > 0) {
return true;
}
}
}
return false;
}
public LogRecord get(Lsn lsn) throws ChaiDBException {
return get(lsn, null);
}
/**
* get a log record by lsn
* first step,search it in write buffer,if not in write buffer,
* second step,get it from read buffer,if not in read buffer either,
* program will read a chunk log record from log file begin with input lsn.
*
* @param lsn
* @return LogRecord
*/
public LogRecord get(Lsn lsn, Lsn stopLsn) throws ChaiDBException {
/* if this lsn.offset is negative, it is the negative value of the
last logrecord's offset in privous file. */
if (lsn.getOffset() < 0) {
lsn.setFileId(lsn.getFileId() - 1);
lsn.setOffset(-lsn.getOffset());
}
LogRecord log = null;
synchronized (BUFFER_WRITER_OBJ) {
log = buffer.get(lsn);//get log from write buffer
}
if (log == null) {
flush();
log = readBuf.get(lsn, stopLsn);//get log from read buffer
}
if (log == null) {
String details = "The LSN value is " + lsn.toHexString() + ".";
throw new ChaiDBException(ErrorCode.LOG_RECORD_NOT_EXIST, details);
} else {
return log;
}
}
/**
* wirte log record in write buffer to log file(s)
* flush log record begin from input lsn,and other log records' lsn
* less than input lsn will all be flushed to disk.
*/
public synchronized void flush() throws ChaiDBException {
_flush();
}
/**
* The sync-free version of flush. It is called by other methods
* of LogManagerImpl, which has been synchronized already.
* WHY: to reduce time.
*/
private final void _flush() throws ChaiDBException {
synchronized (BUFFER_WRITER_OBJ) {
/**
* if buffer stack is empty, it means the OTHER buffer is being
* flushed by the daemon thread. So we should wait for its
* finishing here.
*/
while (bufferStack.size() == 0) {
try {
BUFFER_WRITER_OBJ.wait();
} catch (InterruptedException e) {
logger.warn("Log flush process is interrupted.");
return;
}
}
buffer.writeToDisk(this);
}
}
public void clear() {
try {
closeLogFile();
} catch (ChaiDBException e) {
logger.error("closeLogFile() error:" + e.getMessage());
}
bufferStack = null;
readBuf = null;
buffer = null;
}
/**
* dump buffer information to String as XML format
*
* @return String
*/
public synchronized String dump() {
StringBuffer buf = new StringBuffer("<LogDump>");
buf.append("<LogManagerImpl>");
buf.append("<currentLsn>" + "<fileId>" + curLsn.getFileId() + "</fileId>" + "<offset>" + curLsn.getOffset() + "</offset></currentLsn>");
buf.append("<latestWriteLsn>" + "<fileId>" + lastLsnInFlush.getFileId() + "</fileId>" + "<offset>" + lastLsnInFlush.getOffset() + "</offset></latestWriteLsn>");
buf.append("<previousOffset>" + prevOffset + "</previousOffset>");
if (lastChkpt != null)
buf.append("<lastCheckpoint>" + "<fileId>" + lastChkpt.getFileId() + "</fileId>" + "<offset>" + lastChkpt.getOffset() + "</offset></lastCheckpoint>");
if (chkpt != null)
buf.append("<currentCheckpoint>" + "<fileId>" + chkpt.getFileId() + "</fileId>" + "<offset>" + chkpt.getOffset() + "</offset></currentCheckpoint>");
if (chkptTime != null)
buf.append("<currentCheckpointTime>" + chkptTime.toString() + "</currentCheckpointTime>");
buf.append("<bytes2LastCheckpoint>" + chkptBytes + "</bytes2LastCheckpoint>");
buf.append("<megaBytes2LastCheckpoint>" + chkptMbytes + "</megaBytes2LastCheckpoint>");
if (rLsn != null)
buf.append("<lastReadLsn>" + "<fileId>" + rLsn.getFileId() + "</fileId>" + "<offset>" + rLsn.getOffset() + "</offset></lastReadLsn>");
buf.append("</LogManagerImpl>");
buf.append(buffer.dump());
buf.append(readBuf.dump());
buf.append("</LogDump>");
return buf.toString();
}
class BufferWriter extends AbstractDaemonThread {
private RecordBuffer _buffer;
private LogManagerImpl _logManager;
public static final String THREAD_NAME = "Log Buffer Writer Thread";
public BufferWriter(LogManagerImpl logManager) {
super(THREAD_NAME);
_logManager = logManager;
}
public void flushBuffer(RecordBuffer buffer) {
synchronized (BUFFER_WRITER_OBJ) {
_buffer = buffer;
BUFFER_WRITER_OBJ.notify();
}
}
protected void runningBody() {
try {
synchronized (BUFFER_WRITER_OBJ) {
if (_buffer == null) {
try {
BUFFER_WRITER_OBJ.wait();
} catch (InterruptedException ie) {
return;
}
}
try {
_buffer.writeToDisk(_logManager);
} catch (Throwable e) {
logger.fatal(e);
} finally {
synchronized (BUFFER_WRITER_OBJ) {
bufferStack.push(_buffer);
_buffer = null;
BUFFER_WRITER_OBJ.notify();
}
}
}//sync(this)
} catch (Throwable t) {
logger.fatal(t);
}
}
}
}