package dovetaildb.bagindex;
import java.io.File;
import java.util.Collection;
import dovetaildb.fileaccessor.OffsetValueFilePair;
import dovetaildb.fileaccessor.PagedFile;
import dovetaildb.scan.Scanner;
public class BdbBagIndex extends BagIndex implements java.io.Serializable {
PagedFile terms;
PagedFile postings;
OffsetValueFilePair docData;
File termsFile, postingsFile, docOffsetFile, docValueFile;
long maxDocId;
public BdbBagIndex() {}
String homeDir;
public void setHomedir(String homeDir) {
this.homeDir = homeDir;
open();
}
public String getHomedir() { return homeDir; }
private void open() {
terms = new PagedFile(termsFile=new File(homeDir + File.separator + "terms"));
postings = new PagedFile(postingsFile=new File(homeDir + File.separator + "postings"));
docData = new OffsetValueFilePair(
docOffsetFile=new File(homeDir + File.separator + "docOffset"),
docValueFile=new File(homeDir + File.separator + "docValue")
);
}
public void close() {
terms.close();
postings.close();
docData.close();
}
@Override
public long commitNewRev(String txnId, long[] deletions, Collection<byte[][]> inserts) {
// TODO Auto-generated method stub
return 0;
}
@Override
public Scanner fetchDeletions(long revNum) {
// TODO Auto-generated method stub
return null;
}
@Override
public BagIndexDoc fetchDoc(long docId) {
// TODO Auto-generated method stub
return null;
}
@Override
public Scanner fetchRange(byte[] term1, byte[] term2, boolean isExclusive1, boolean isExclusive2, long revNum) {
// TODO Auto-generated method stub
return null;
}
@Override
public Scanner fetchTd(byte[] term, long revNum) {
// TODO Auto-generated method stub
return null;
}
@Override
public BagIndexDoc refetchDoc(BagIndexDoc doc, long docId) {
// TODO Auto-generated method stub
}
/*
SparsePagedFile terms;
SparsePagedFile postings;
OffsetValueFilePair docData;
long maxDocId;
static final boolean TRACKER_ENABLED = false;
public BdbBagIndex() {}
String homeDir, bdbDir;
public void setHomedir(String homeDir) {
this.homeDir = homeDir;
open();
}
public String getHomedir() { return homeDir; }
private void open() {
BitFieldFile termsFree = new BitFieldFile(new File(homeDir + File.separator + "terms.free"));
PagedFile termsPage = new PagedFile(new File(homeDir + File.separator + "terms.page"));
BitFieldFile postingsFree = new BitFieldFile(new File(homeDir + File.separator + "postings.free"));
PagedFile postingsPage = new PagedFile(new File(homeDir + File.separator + "postings.page"));
terms = new SparsePagedFile(termsPage, termsFree);
postings = new SparsePagedFile(postingsPage, postingsFree);
}
public void close() {
terms.close();
postings.close();
terms = null;
postings = null;
}
public String toString() {
try {
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
try {
DatabaseEntry ke = new DatabaseEntry();
DatabaseEntry ve = new DatabaseEntry();
StringBuffer buf = new StringBuffer("BdbBagIndex[");
if (cur.getFirst(ke, ve, null) == OperationStatus.SUCCESS) {
do {
buf.append(Util.bytesAsString(ke.getData()));
buf.append(": ");
buf.append(Util.bytesAsString(ve.getData()));
buf.append("\n");
} while(cur.getNext(ke, ve, null) == OperationStatus.SUCCESS);
}
buf.append(']');
return buf.toString();
} finally {
openStuff.close(cur);
cur.close();
}
} catch(DatabaseException e) { throw new RuntimeException(e); }
}
public long getMaxDocId() { return this.maxDocId; }
public long fetchMaxDocId() {
try {
Cursor cur = docToData.openCursor(null, cursorConfig);
openStuff.open(cur);
try {
DatabaseEntry ke = new DatabaseEntry();
DatabaseEntry ve = new DatabaseEntry();
if (cur.getLast(ke, ve, null) == OperationStatus.NOTFOUND) return 0;
byte[] data = ke.getData();
return Util.leBytesToUInt(data, data.length-4) + 1;
} finally {
openStuff.close(cur);
cur.close();
}
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
private static final CursorConfig cursorConfig = new CursorConfig();
static { cursorConfig.setReadUncommitted(true); }
public long docIdForId(String id, long revNum) {
// !! id is assumed to be already s-encoded
try {
byte[] idTerm = new byte[3+id.length()];
idTerm[0]=2;
idTerm[1]=(byte)'i';
idTerm[2]=(byte)'d';
byte[] data = id.getBytes("utf-8");
System.arraycopy(data, 0, idTerm, 3, data.length);
DatabaseEntry ke = new DatabaseEntry(idTerm);
DatabaseEntry ve = new DatabaseEntry();
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
try {
if (cur.getSearchKey(ke, ve, null) == OperationStatus.NOTFOUND) return -1;
CursorDeletionTermDocs curDels = this.getDeletions(revNum);
try {
CursorTermDocs curById = new CursorTermDocs(cur, idTerm, ve.getData(), revNum);
int docId = -1;
while(true) {
if (!curById.next()) {docId = -1; break; }
docId = curById.doc();
if (curDels.doc() < docId) {
if (!curDels.skipTo(docId)) break;
}
if (curDels.doc() != docId) break;
}
return docId;
} finally {
curDels.close();
}
} finally {
openStuff.close(cur);
cur.close();
}
} catch(UnsupportedEncodingException e) {
throw new RuntimeException(e);
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
public CursorDeletionTermDocs getDeletions(long revNum) {
try {
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
try {
DatabaseEntry ke = new DatabaseEntry();
byte[] nullKey = new byte[]{};
ke.setData(nullKey);
DatabaseEntry ve = new DatabaseEntry();
cur.getSearchKey(ke, ve, null);
if (cur.getSearchKey(ke, ve, null) != OperationStatus.SUCCESS) {
ve.setData(new byte[]{});
cur.put(ke, ve);
}
CursorDeletionTermDocs td = new CursorDeletionTermDocs(cur, nullKey, ve.getData(), revNum);
cur = null;
return td;
} finally {
if (cur != null) {
openStuff.close(cur);
cur.close();
}
}
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
// byte[] sufficies for a term - it's <key len : 1 byte><key><value>
public TermDocs getTd(byte[] term, long revNum) {
try {
DatabaseEntry ke = new DatabaseEntry(term);
DatabaseEntry ve = new DatabaseEntry();
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
if (cur.getSearchKeyRange(ke, ve, null) == OperationStatus.NOTFOUND) {
openStuff.close(cur);
cur.close();
return TermDocsEmpty.EMPTY_TERM_DOCS;
}
return new CursorTermDocs(cur, term, ve.getData(), revNum);
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
// may return results outside the range, postfiltering is required
public TermDocs getRange(byte[] term1, byte[] term2, boolean isExclusive1, boolean isExclusive2, long revNum) {
try {
DatabaseEntry ke = new DatabaseEntry(term1);
DatabaseEntry ve = new DatabaseEntry();
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
ArrayList<TermDocs> tds = new ArrayList<TermDocs>();
try {
OperationStatus status = cur.getSearchKeyRange(ke, ve, null);
if (isExclusive1) {
boolean hitTerm = (status == OperationStatus.SUCCESS) && Arrays.equals(term1, ke.getData());
if (hitTerm) {
cur.getNextNoDup(ke, ve, null);
}
}
while(true) {
Cursor dup = cur.dup(true);
openStuff.open(dup);
tds.add(new CursorTermDocs(dup, ke.getData(), ve.getData(), revNum));
cur.getNext(ke, ve, null);
int cmp = Util.compareBytes(term2, ke.getData());
if ((cmp < 0) ||
(cmp == 0 && isExclusive2)) break;
}
} finally {
openStuff.close(cur);
cur.close();
}
return TermDocsDisjunction.make(tds);
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
// returns the rev number
public synchronized long commitNewRev(String txnId, long[] deletions, Collection<ResultMap> inserts) {
try {
Arrays.sort(deletions);
long oldMaxDocId = maxDocId;
long newMaxDocId = maxDocId + inserts.size();
if (inserts.isEmpty()) {
// commits with only deletions occupy a docId space, so schedule a deletion for it
long[] newDeletions = new long[deletions.length+1];
System.arraycopy(deletions, 0, newDeletions, 0, deletions.length);
newDeletions[deletions.length] = newMaxDocId;
deletions = newDeletions;
inserts.add(new LiteralResultMap());
newMaxDocId++;
}
if (deletions.length > 0) {
CursorTermDocs scanner = getDeletions(Long.MAX_VALUE);
try {
int deletionsIdx = 0;
byte[] insertBuf = new byte[8];
Util.leIntToBytes((int)newMaxDocId, insertBuf, 4);
while(scanner.next()) {
if (scanner.doc() >= deletions[deletionsIdx]) {
if (scanner.doc() == deletions[deletionsIdx]) throw new ApiException("UpdateConflict","Another update comitted prior to your change; transaction aborted.");
Util.leIntToBytes((int)deletions[deletionsIdx], insertBuf, 0);
scanner.insert(insertBuf);
deletionsIdx++;
if (deletionsIdx >= deletions.length) break;
}
}
while(deletionsIdx < deletions.length) {
long deletion = deletions[deletionsIdx];
if (deletion != -1) {
Util.leIntToBytes((int)deletion, insertBuf, 0);
scanner.insert(insertBuf);
}
deletionsIdx++;
}
} finally {
scanner.close();
}
}
DatabaseEntry ke = new DatabaseEntry();
DatabaseEntry ve = new DatabaseEntry();
Cursor cur = termToPostings.openCursor(null, cursorConfig);
openStuff.open(cur);
try {
for(ResultMap map : inserts) {
byte[][] allFieldKeys = new byte[map.size()][];
byte[][] allFieldVals = new byte[map.size()][];
int allFieldsIdx = 0;
int totBytes = 0;
for(String key : map.keySet()) {
String val = map.getEncoded(key);
byte[] keyBytes = key.getBytes("utf-8");
byte[] valBytes = val.getBytes("utf-8");
// temporary
if (valBytes.length > 65535) valBytes=Util.sencode(null).getBytes("utf-8");
allFieldKeys[allFieldsIdx] = keyBytes;
allFieldVals[allFieldsIdx] = valBytes;
allFieldsIdx++;
totBytes += keyBytes.length + valBytes.length;
byte[] termBytes = this.getTermFromEncoded(key, val);
ke.setData(termBytes);
if (cur.getSearchKey(ke, ve, null) != OperationStatus.SUCCESS) {
byte[] newDocs = new byte[4];
Util.leIntToBytes((int)maxDocId, newDocs, 0);
ve.setData(newDocs);
cur.put(ke, ve);
} else {
if (cur.getNextNoDup(ke, ve, null) == OperationStatus.SUCCESS) {
cur.getPrev(ke,ve,null);
} else {
// we're at the end already
}
byte[] docs = ve.getData();
byte[] newDocs;
if (docs.length+4 >= CursorTermDocs.MAX_BLOCK_SIZE) {
newDocs = new byte[4];
Util.leIntToBytes((int)maxDocId, newDocs, 0);
} else {
cur.delete();
newDocs = new byte[docs.length+4];
System.arraycopy(docs, 0, newDocs, 0, docs.length);
Util.leIntToBytes((int)maxDocId, newDocs, docs.length);
}
ve.setData(newDocs);
cur.put(ke, ve);
}
}
// two bytes for num fields, four bytes per field (two offsets into data), then data
int headerSize = 2 + allFieldKeys.length*4;
byte[] docRec = new byte[headerSize + totBytes];
if (docRec.length > 65535) throw new ApiException("InvalidFieldSize","Object size too large: "+map);
Util.leShortToBytes(allFieldKeys.length, docRec, 0);
int pos = 0;
for(int i=0; i<allFieldKeys.length; i++) {
int len = allFieldKeys[i].length;
System.arraycopy(allFieldKeys[i], 0, docRec, headerSize+pos, len);
pos += len;
Util.leShortToBytes(pos, docRec, i*4+2);
len = allFieldVals[i].length;
System.arraycopy(allFieldVals[i], 0, docRec, headerSize+pos, len);
pos += len;
Util.leShortToBytes(pos, docRec, i*4+4);
}
byte[] keyBytes = new byte[4];
Util.leIntToBytes((int)maxDocId, keyBytes, 0);
ke.setData(keyBytes);
ve.setData(docRec);
this.docToData.put(null, ke, ve);
maxDocId++;
// System.out.println(" ----- MAX DOC INCR TO "+maxDocId);
}
} finally {
openStuff.close(cur);
cur.close();
}
ke.setData(txnId.getBytes("utf-8"));
byte[] revNumBytes = new byte[4];
Util.leIntToBytes((int)oldMaxDocId, revNumBytes, 0);
ve.setData(revNumBytes);
txnToRev.put(null, ke, ve);
maxDocId = newMaxDocId;
return newMaxDocId;
} catch(DatabaseException e) {
throw new RuntimeException(e);
} catch(UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public long revForTxn(String txnId) {
try {
DatabaseEntry ke = new DatabaseEntry();
ke.setData(txnId.getBytes("utf-8"));
DatabaseEntry ve = new DatabaseEntry();
if (txnToRev.get(null, ke, ve, null) == OperationStatus.SUCCESS) {
return Util.leBytesToUInt(ve.getData(), 0);
} else {
throw new RuntimeException("Unknown transaction ID: "+txnId);
}
} catch(DatabaseException e) {
throw new RuntimeException(e);
} catch(UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public void getDoc(long docId, BagIndexResultMap map) {
try {
DatabaseEntry ke = new DatabaseEntry();
DatabaseEntry ve = new DatabaseEntry();
byte[] idBytes = new byte[4];
Util.leIntToBytes((int)docId, idBytes, 0);
docToData.get(null, ke, ve, null);
map.init(ve.getData());
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
public BagIndexResultMap getDoc(long docId) {
try {
DatabaseEntry ke = new DatabaseEntry();
DatabaseEntry ve = new DatabaseEntry();
byte[] idBytes = new byte[4];
Util.leIntToBytes((int)docId, idBytes, 0);
ke.setData(idBytes);
if (docToData.get(null, ke, ve, null) == OperationStatus.SUCCESS)
return new BagIndexResultMap(ve.getData());
else
throw new RuntimeException("Requested nonexistent document ID "+docId);
} catch(DatabaseException e) {
throw new RuntimeException(e);
}
}
*/
}