/*
* @(#)$Id: Paged.java 3619 2008-03-26 07:23:03Z yui $
*
* Copyright 2006-2008 Makoto YUI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Contributors:
* Makoto YUI - ported from Apache Xindice with some modification
*/
/*
* Copyright 1999-2004 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xbird.storage.index;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.ref.Reference;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Map;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import xbird.storage.DbException;
import xbird.util.concurrent.reference.ReferenceMap;
import xbird.util.concurrent.reference.ReferenceType;
import xbird.util.io.FastMultiByteArrayOutputStream;
/**
*
* <DIV lang="en"></DIV>
* <DIV lang="ja"></DIV>
*
* @author Makoto YUI (yuin405+xbird@gmail.com)
*/
@NotThreadSafe
public abstract class Paged {
private static final Log LOG = LogFactory.getLog(Paged.class);
public static final int DEFAULT_PAGESIZE = 1024 * 4; // 4KB page
protected static final byte UNUSED = 0;
protected static final byte OVERFLOW = 126;
/** Page ID of non-existent page */
protected static final int NO_PAGE = -1;
//--------------------------------------------
private final Map<Long, Reference<Page>> _pages = new ReferenceMap<Long, Reference<Page>>(ReferenceType.WEAK, ReferenceType.SOFT, 64);
private final FileHeader _fileHeader;
protected final File _file;
//--------------------------------------------
// resources
private boolean _opened = false;
private RandomAccessFile _raf = null;
private FileChannel _fc = null;
//--------------------------------------------
public Paged(File file) {
this(file, DEFAULT_PAGESIZE);
}
public Paged(File file, int pageSize) {
this._fileHeader = createFileHeader(pageSize);
this._file = file;
}
public File getFile() {
return _file;
}
/** create index resources and close it. */
public boolean create() throws DbException {
return create(true);
}
public boolean create(boolean close) throws DbException {
ensureResourceOpen();
try {
_fileHeader.write();
} catch (IOException e) {
throw new DbException(e);
}
if(close) {
close();
} else {
this._opened = true;
}
return true;
}
public boolean open() throws DbException {
ensureResourceOpen();
if(exists()) {
try {
_fileHeader.read();
} catch (IOException e) {
throw new DbException(e);
}
this._opened = true;
return true;
} else {
this._opened = false;
return false;
}
}
protected final RandomAccessFile ensureResourceOpen() throws DbException {
if(_raf == null) {
try {
this._raf = new RandomAccessFile(_file, "rw");
} catch (FileNotFoundException e) {
throw new DbException(e);
}
}
if(_fc == null) {
this._fc = _raf.getChannel();
}
return _raf;
}
public boolean close() throws DbException {
if(_opened) {
this._opened = false;
// close resources
try {
_raf.close();
_fc.close();
} catch (IOException e) {
throw new DbException(e);
}
reset();
return true;
} else {
return false;
}
}
protected final void checkOpened() throws DbException {
if(!_opened) {
throw new DbException("Not opened");
}
}
private final void reset() {
this._raf = null;
this._fc = null;
}
public boolean drop() throws DbException {
close();
if(exists()) {
return getFile().delete();
} else {
return true;
}
}
public final boolean exists() {
return _file.exists();
}
public void flush() throws DbException {
try {
if(_fileHeader._fhDirty) {
_fileHeader.write();
}
_fc.force(true);
} catch (IOException e) {
throw new DbException(e);
}
}
/**
* createFileHeader must be implemented by a Paged implementation
* in order to create an appropriate subclass instance of a FileHeader.
*
* @return a new FileHeader
*/
protected abstract FileHeader createFileHeader(int pageSize);
/**
* createPageHeader must be implemented by a Paged implementation
* in order to create an appropriate subclass instance of a PageHeader.
*/
protected abstract PageHeader createPageHeader();
/**
* getPage returns the page specified by pageNum.
*/
protected final Page getPage(long pageNum) throws DbException {
Page p = null;
// if not check if it's already loaded in the page cache
Reference<Page> ref = _pages.get(pageNum); // Check if required page is in the volatile cache
if(ref != null) {
p = ref.get();
}
if(p == null) {
// if still not found we need to create it and add it to the page cache.
p = new Page(pageNum);
try {
p.read(); // Load the page from disk if necessary
} catch (IOException e) {
throw new DbException(e);
}
_pages.put(pageNum, new WeakReference<Page>(p));
}
return p;
}
/**
* getFreePage returns the first free Page from secondary storage.
* If no Pages are available, the file is grown as appropriate.
*/
protected final Page getFreePage() throws DbException {
Page p = null;
// Synchronize read and write to the fileHeader.firstFreePage
if(_fileHeader._firstFreePage != NO_PAGE) {
// Steal a deleted page
p = getPage(_fileHeader._firstFreePage);
_fileHeader.setFirstFreePage(p._pageHeader._nextPage);
if(_fileHeader._firstFreePage == NO_PAGE) {
_fileHeader.setLastFreePage(NO_PAGE);
}
}
if(p == null) { // No deleted pages, grow the file
p = getPage(_fileHeader.incrTotalPageCount());
}
p.initPage(); // Initialize The Page Header (Cleanly)
return p;
}
/**
* unlinkPages unlinks a set of pages starting at the specified
* page number.
*/
protected final void unlinkPages(long pageNum) throws DbException {
unlinkPages(getPage(pageNum));
}
/**
* unlinkPages unlinks a set of pages starting at the specified Page.
*/
protected final void unlinkPages(Page page) throws DbException {
Page nextPage = page;
if(nextPage != null) {
// Walk the chain and add it to the unused list
long firstPage = nextPage._pageNum;
long nextPageNum = nextPage.getPageHeader().getNextPage();
while(nextPageNum != NO_PAGE) {
nextPage = getPage(nextPageNum);
nextPageNum = nextPage.getPageHeader().getNextPage();
}
long lastPage = nextPage.getPageNum();
// Free the chain
if(_fileHeader._lastFreePage != NO_PAGE) {
Page p = getPage(_fileHeader._lastFreePage);
p._pageHeader.setNextPage(firstPage);
p.write();
}
if(_fileHeader._firstFreePage == NO_PAGE) {
_fileHeader.setFirstFreePage(firstPage);
}
_fileHeader.setLastFreePage(lastPage);
}
}
/**
* writeValue writes the multi-Paged Value starting at the specified
* Page.
*
* @param page The starting Page
* @param value The Value to write
*/
public final void writeValue(Page page, Value value) throws DbException {
InputStream is = value.getInputStream();
// Write as much as we can onto the primary page.
PageHeader hdr = page.getPageHeader();
hdr.setRecordLength(value.getLength());
try {
page.readData(is);
} catch (IOException e) {
throw new DbException(e);
}
// Write out the rest of the value onto any needed overflow pages
Page lastPage = page;
while(true) {
final int available;
try {
available = is.available();
} catch (IOException e) {
throw new DbException(e);
}
if(available == 0) {
break;
}
LOG.debug("page overflowed");
Page lpage = lastPage;
PageHeader lhdr = hdr;
// Find an overflow page to use
long np = lhdr.getNextPage();
if(np != NO_PAGE) {
// Use an existing page
lastPage = getPage(np);
} else {
// Create a new overflow page
lastPage = getFreePage();
lhdr.setNextPage(lastPage.getPageNum());
}
// Mark the page as an overflow page
hdr = lastPage.getPageHeader();
hdr.setStatus(OVERFLOW);
// Write some more of the value to the overflow page
try {
lastPage.readData(is);
} catch (IOException e) {
throw new DbException(e);
}
lpage.write();
}
// Cleanup any unused overflow pages. i.e. the value is smaller then the
// last time it was written.
long np = hdr.getNextPage();
if(np != NO_PAGE) {
unlinkPages(np);
}
hdr.setNextPage(NO_PAGE);
lastPage.write();
}
/**
* writeValue writes the multi-Paged Value starting at the specified
* page number.
*
* @param page The starting page number
* @param value The Value to write
*/
public final void writeValue(long page, Value value) throws DbException {
writeValue(getPage(page), value);
}
@Deprecated
public final long writeValue(Value value) throws DbException {
Page p = getFreePage();
writeValue(p, value);
return p.getPageNum();
}
/**
* readValue reads the multi-Paged Value starting at the specified
* Page.
*
* @param page The starting Page
* @return The Value
*/
public final Value readValue(Page page) throws DbException {
PageHeader sph = page.getPageHeader();
FastMultiByteArrayOutputStream bos = new FastMultiByteArrayOutputStream(sph.getRecordLength());
// Loop until we've read all the pages into memory
Page p = page;
while(true) {
// Add the contents of the page onto the stream
try {
p.writeData(bos);
} catch (IOException e) {
throw new DbException(e);
}
// Continue following the list of pages until we get to the end
PageHeader ph = p.getPageHeader();
long nextPage = ph.getNextPage();
if(nextPage == NO_PAGE) {
break;
}
p = getPage(nextPage);
}
// Return a Value with the collected contents of all pages
return new Value(bos.toByteArray());
}
/**
* readValue reads the multi-Paged Value starting at the specified
* page number.
*
* @param page The starting page number
* @return The Value
*/
@Deprecated
public final Value readValue(long page) throws DbException {
return readValue(getPage(page));
}
protected FileHeader getFileHeader() {
return _fileHeader;
}
public abstract class FileHeader {
private boolean _fhDirty = true;
private int _workSize;
//--------------------------------------------
// persistant entry
/** The size of the <code>FileHeader</code>. Usually 1 OS page (4096 byte). */
private short _fhSize;
/** The size of a <code>Page</code> */
private int _pageSize;
/** The number of total pages in the file */
private long _totalPageCount;
/** The first free page in unused secondary space */
private long _firstFreePage = NO_PAGE;
/** The last free page in unused secondary space */
private long _lastFreePage = NO_PAGE;
/** The size of the <code>PageHeader</code> */
private byte _pageHeaderSize = PageHeader.DEFAULT_PAGE_HEADER_SIZE;
//--------------------------------------------
public FileHeader(int pageSize) {
this._pageSize = pageSize;
this._fhSize = (short) 4096;
this._workSize = calculateWorkSize();
}
public final void write() throws IOException {
if(!_fhDirty) {
return;
}
_raf.seek(0);
write(_raf);
if(LOG.isDebugEnabled()) {
LOG.debug("wrote file header");
}
this._fhDirty = false;
}
protected void write(RandomAccessFile raf) throws IOException {
raf.writeShort(_fhSize);
raf.writeInt(_pageSize);
raf.writeLong(_totalPageCount);
raf.writeLong(_firstFreePage);
raf.writeLong(_lastFreePage);
raf.writeByte(_pageHeaderSize);
}
public final void read() throws IOException {
_raf.seek(0);
read(_raf);
this._workSize = calculateWorkSize();
}
protected void read(RandomAccessFile raf) throws IOException {
this._fhSize = raf.readShort();
this._pageSize = raf.readInt();
this._totalPageCount = raf.readLong();
this._firstFreePage = raf.readLong();
this._lastFreePage = raf.readLong();
this._pageHeaderSize = raf.readByte();
}
//--------------------------------------------
public final void setFirstFreePage(long page) {
this._firstFreePage = page;
this._fhDirty = true;
}
public long getFirstFreePage() {
return _firstFreePage;
}
public final void setLastFreePage(long page) {
this._lastFreePage = page;
this._fhDirty = true;
}
public long getLastFreePage() {
return _lastFreePage;
}
public final long incrTotalPageCount() {
this._fhDirty = true;
return _totalPageCount++;
}
public final void setDirty(boolean dirty) {
this._fhDirty = dirty;
}
public void setTotalPageCount(long pageCount) {
this._fhDirty = true;
this._totalPageCount = pageCount;
}
public final long getTotalPageCount() {
return _totalPageCount;
}
public final int getPageSize() {
return _pageSize;
}
public final int getWorkSize() {
return _workSize;
}
private final int calculateWorkSize() {
return _pageSize - _pageHeaderSize;
}
}
public static abstract class PageHeader {
public static final int DEFAULT_PAGE_HEADER_SIZE = 127;
//--------------------------------------------
// persistent entry
private byte _status = UNUSED;
private int _dataLen;
private int _recordLen;
private long _nextPage = NO_PAGE;
//--------------------------------------------
public PageHeader() {}
public PageHeader(ByteBuffer buf) {
read(buf);
}
public void read(ByteBuffer buf) {
this._status = buf.get();
if(_status == UNUSED) {
return;
}
this._dataLen = buf.getInt();
this._recordLen = buf.getInt();
this._nextPage = buf.getLong();
}
public void write(ByteBuffer buf) {
buf.put(_status);
buf.putInt(_dataLen);
buf.putInt(_recordLen);
buf.putLong(_nextPage);
}
//--------------------------------------------
// getter, setter
/** The status of this page (UNUSED, RECORD, DELETED, etc...) */
public final void setStatus(byte status) {
this._status = status;
}
/** The status of this page (UNUSED, RECORD, DELETED, etc...) */
public final byte getStatus() {
return _status;
}
/** The next page for this Record (if overflowed) */
public final void setNextPage(long nextPage) {
this._nextPage = nextPage;
}
/** The next page for this Record (if overflowed) */
public final long getNextPage() {
return _nextPage;
}
/** The length of the Data */
public final void setDataLength(int dataLen) {
this._dataLen = dataLen;
}
public final int getDataLength() {
return _dataLen;
}
public final void setRecordLength(int length) {
this._recordLen = length;
}
public final int getRecordLength() {
return _recordLen;
}
}
public final class Page implements Comparable<Page> {
private final long _pageNum;
private final PageHeader _pageHeader;
/** The offset into the file that this page starts */
private final long _pageOffset;
/** The data for this page */
private ByteBuffer _pageData = null;
/** The position (relative) of the Data in the data array */
private int _dataPos;
public Page(long pageNum) {
this._pageNum = pageNum;
this._pageHeader = createPageHeader();
this._pageOffset = _fileHeader._fhSize + (pageNum * _fileHeader._pageSize);
}
public synchronized void read() throws IOException {
if(_pageData == null) {
if(LOG.isDebugEnabled()) {
LOG.debug("read in page#" + _pageNum + " from page offset " + _pageOffset);
}
byte[] buf = new byte[_fileHeader._pageSize];
_raf.seek(_pageOffset);
_raf.read(buf);
this._pageData = ByteBuffer.wrap(buf);
_pageHeader.read(_pageData);
this._dataPos = _fileHeader._pageHeaderSize;
}
}
public synchronized void write() throws DbException {
_pageData.rewind();
_pageHeader.write(_pageData);
try {
_raf.seek(_pageOffset);
_raf.write(_pageData.array());
} catch (IOException e) {
throw new DbException(e);
}
}
/**
* Flushes content of the dirty page into the file
*/
public void flush() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("write out page#" + _pageNum + " to page offset " + _pageOffset);
}
_raf.seek(_pageOffset);
_raf.write(_pageData.array());
}
public void writeData(OutputStream os) throws IOException {
if(_pageHeader._dataLen > 0) {
byte[] b = new byte[_pageHeader._dataLen];
_pageData.position(_dataPos);
_pageData.get(b);
os.write(b);
}
}
public void readData(InputStream is) throws IOException {
// set data length in the page header
int avail = is.available();
int datalen = _fileHeader._workSize;
if(avail < datalen) {
datalen = avail;
}
_pageHeader.setDataLength(datalen);
// read data from stream
if(datalen > 0) {
byte[] b = new byte[datalen];
is.read(b);
_pageData.position(getDataPos());
_pageData.put(b);
}
}
public PageHeader getPageHeader() {
return _pageHeader;
}
public long getPageNum() {
return _pageNum;
}
private int getDataPos() {
return _dataPos;
}
protected void initPage() {
_pageHeader.setNextPage(NO_PAGE);
_pageHeader.setStatus(UNUSED);
}
public int compareTo(Page other) {
return (int) (_pageNum - other._pageNum);
}
@Override
public String toString() {
return "page#" + _pageNum;
}
}
}