/*
* Citrusleaf Aerospike Java Library
*
* Copyright 2009-2010 by Citrusleaf, Inc. All rights reserved.
*
* Availability of this source code to partners and customers includes
* redistribution rights covered by individual contract. Please check
* your contract for exact rights and responsibilities.
*/
package net.citrusleaf;
import gnu.crypto.hash.*;
// import org.bouncycastle.crypto.digests.GeneralDigest;
// import org.bouncycastle.crypto.digests.RIPEMD160Digest;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.*;
import net.citrusleaf.CitrusleafClient.*;
public class CLBuffer {
// NB - it's possible, and cooler, to use enums for things like this,
// but they're really the constant numbers used in the wire protocol.
// Java enums are annoying to map to numbers.
// This is easier and more readable.
public final static int FIELD_TYPE_NAMESPACE = 0;
public final static int FIELD_TYPE_TABLE = 1;
public final static int FIELD_TYPE_KEY = 2;
public final static int FIELD_TYPE_BIN = 3;
public final static int FIELD_TYPE_DIGEST_RIPE = 4;
public final static int FIELD_TYPE_GU_TID = 5;
public final static int FIELD_TYPE_DIGEST_RIPE_ARRAY = 6;
public final static int FIELD_TYPE_TRID = 7; // user supplied transaction id, which is simply passed back
public final static int FIELD_TYPE_SCAN_OPTIONS = 8;
public final static int DIGEST_SIZE = 20;
private final static int OP_READ = 1;
private final static int OP_WRITE = 2;
@SuppressWarnings("unused") private final static int OP_WRITE_UNIQUE = 3;
@SuppressWarnings("unused") private final static int OP_WRITE_NOW = 4;
private final static int OP_ADD = 5;
private final static int OP_APPEND_SEGMENT = 6;
private final static int OP_APPEND_SEGMENT_EXT = 7;
private final static int OP_APPEND_SEGMENT_QUERY = 8;
private final static int OP_APPEND = 9;
private final static int OP_PREPEND = 10;
private final static int OP_TOUCH = 11;
@SuppressWarnings("unused") private final static int OP_MC_INCR = 129;
@SuppressWarnings("unused") private final static int OP_MC_APPEND = 130;
@SuppressWarnings("unused") private final static int OP_MC_PREPEND = 131;
@SuppressWarnings("unused") private final static int OP_MC_TOUCH = 132;
// byte - 1 byte
// size - 7 bytes
// op - 1 byte
// particle type - 1 byte
// version - 1 byte
// name_sz - 1 byte
// name N
// per-particle data
//
// Adds the output bytes to the collection
// return value is the new offset
// Throws an array out of bounds if buffer is too small
private int
makeOp(int op, String name, Object o, byte[] buf, int offset, byte[] java_serialized_bytes) throws SerializeException
{
// copy name into buffer, get length of the name
int nameLen = stringToUtf8(name, buf, offset + 8);
// only a true write needs the write data from object
// the number 4 is the size *after* the size field
int field_sz = 4 + nameLen;
ParticleBytesReturn pbr = null;
if (op == OP_WRITE || op == OP_PREPEND || op == OP_APPEND || op == OP_PREPEND || op == OP_ADD || op == OP_APPEND_SEGMENT || op == OP_APPEND_SEGMENT_EXT || op == OP_APPEND_SEGMENT_QUERY) {
pbr = objectToParticleBytes(o, buf, 8 + nameLen + offset, java_serialized_bytes);
field_sz += pbr.size;
}
set_htonl(field_sz, buf, offset);
buf[offset+4] = (byte) op;
buf[offset+5] = (byte) (pbr == null ? PARTICLE_TYPE_NULL : pbr.type);
buf[offset+6] = (byte) 0; // version
buf[offset+7] = (byte) nameLen;
return(field_sz + 4);
}
private int makeOpEstimate(Object o) {
int estimate = 8 + 32;
if (o == null) {
return(estimate);
}
if (o instanceof byte[]) {
byte[] b = (byte[]) o;
estimate += b.length;
}
else if (o instanceof String) {
String s = (String) o;
estimate += s.length();
} else if (o instanceof Integer) {
estimate += 9;
}
else if (o instanceof Long) {
estimate += 9;
}
else {
// regrettably, there's not an easy way to get an estimate out of the JBLOB system. return 0 so caller
// can do the serialization once to find the length
estimate = 0;
}
return(estimate);
}
//
// Sets the 5 bytes of the field header
//
static private int set_field_header(int type, long sz, byte[] buf, int offset)
{
// CitrusleafClient.info("set_field_header: sz "+sz+" offset "+offset);
set_htonl((int)(sz+1),buf, offset);
offset+=4;
buf[offset] = (byte) type;
offset++;
// CitrusleafClient.info("bytes: "+buf[offset-1]+" "+buf[offset]+" "+buf[offset+1]+" "+buf[offset+2]);
return(5);
}
static public void set_htonll(long v, byte[] buf, int offset)
{
for (int i=7;i>=0;i--) {
buf[offset+i] = (byte) (v & 0xff);
v >>>= 8;
}
}
static public void set_htonl(int v, byte[] buf, int offset)
{
for (int i=3;i>=0;i--) {
buf[offset+i] = (byte) (v & 0xff);
v >>>= 8;
}
}
static public void set_htons(int v, byte[] buf, int offset)
{
buf[offset] = (byte) ((v >> 8) & 0xff);
buf[offset+1] = (byte) (v & 0xff);
}
static private int get_unsigned(byte b) {
int r = b;
if (r < 0) {
r = r & 0x7f;
r = r | 0x80;
}
return(r);
}
static public long get_ntohll(byte[] buf, int offset) {
long a = 0;
for (int i=0;i<8;i++) {
a <<= 8;
a |= get_unsigned(buf[offset+i]);
}
return(a);
}
static public int get_ntohl(byte[] buf, int offset) {
return (
((buf[offset] & 0xFF) << 24) |
((buf[offset+1] & 0xFF) << 16) |
((buf[offset+2] & 0xFF) << 8) |
(buf[offset+3] & 0xFF)
);
}
static public int get_ntohl_intel(byte[] buf, int offset) {
return (
((buf[offset+3] & 0xFF) << 24) |
((buf[offset+2] & 0xFF) << 16) |
((buf[offset+1] & 0xFF) << 8) |
(buf[offset] & 0xFF)
);
}
static public int get_ntohs(byte[] buf, int offset) {
return ( ((buf[offset] & 0xFF) << 8) + (buf[offset+1] & 0xFF) );
}
public byte[] getDigest(String set, Object key) {
int set_len, key_len, set_offset, key_offset;
byte[] digest;
IMessageDigest md = HashFactory.getInstance("RIPEMD-160");
byte[] buf = new byte[256]; //for holding set and key
set_offset = 0;
set_len = stringToUtf8(set, buf, set_offset);
key_offset = set_len+1;
key_len = keyToBytes(key, buf, key_offset);
md.update(buf, set_offset, set_len);
md.update(buf, key_offset, key_len);
digest = md.digest();
return digest;
}
public static String byteArrayToHexString(byte[] b) {
StringBuffer sb = new StringBuffer(b.length * 2);
for (int i = 0; i < b.length; i++) {
int v = b[i] & 0xff;
if (v < 16) {
sb.append('0');
}
sb.append(Integer.toHexString(v));
}
return sb.toString();
}
public int keyToBytes(Object key, byte[] buf, int key_offset) {
try {
ParticleBytesReturn pbr = objectToParticleBytes(key, buf, key_offset+1, null);
buf[key_offset] = (byte)pbr.type;
return (pbr.size + 1);
} catch (Exception e) {
CitrusleafClient.info("CLBuffer getDigest: exception "+e.toString());
return 0;
}
}
//
// All requests have namespace, table, key, so make a byte array that includes
// all those fields, I think that's the fastest way
// Makes three headers, really
// 1 type = namespace
// 3 len
// [namespace bytes]
// 1 type = table
// 7 len
// [table bytes]
// 1 type = key
// 7 len
// 1 particle type
// key particle bytes
private int
makeFields(byte[] buf, int offset)
{
int initial_offset = offset, set_offset = 0, key_offset = 0, key_len = 0, set_len = 0, digest_offset =0, digest_len = 0;
if (this.ns != null) {
int ns_len = stringToUtf8(this.ns, buf, offset+5);
offset += set_field_header(FIELD_TYPE_NAMESPACE, ns_len, buf, offset) + ns_len;
}
// If digest exists, get the offset for the digest header and length
if (this.digest != null) {
System.arraycopy(this.digest, 0, buf, offset+5, this.digest.length);
offset += set_field_header(FIELD_TYPE_DIGEST_RIPE, this.digest.length, buf, offset);
offset += this.digest.length;
} else {
if (this.set != null){
set_offset = offset + 5;
set_len = stringToUtf8(this.set, buf, offset+5);
offset += set_field_header(FIELD_TYPE_TABLE, set_len, buf, offset) + set_len;
}
if (this.key != null) {
ParticleBytesReturn pbr;
try {
pbr = objectToParticleBytes(this.key, buf, offset + 1 + 5, null);
} catch (Exception e) {
CitrusleafClient.info("Asbuffer makeFields prepare: exception "+e.toString());
return(0);
}
offset += set_field_header(FIELD_TYPE_KEY, pbr.size+1, buf, offset);
key_offset = offset;
buf[offset] = (byte)pbr.type;
offset++;
offset += pbr.size;
key_len = pbr.size + 1;
}
}
if (this.scan_options != null) {
System.arraycopy(this.scan_options, 0, buf, offset+5, 2); // scan_option is 2 bytes
offset += set_field_header(FIELD_TYPE_SCAN_OPTIONS, 2, buf, offset);
offset += 2;
}
// THIS VERSION IS GNU.DRYPTO, which is faster than some
if (partition_id == -1) {
if (this.digest != null) {
partition_id = get_ntohl_intel(this.digest, 0);
} else {
IMessageDigest md = HashFactory.getInstance("RIPEMD-160");
md.update(buf, set_offset, set_len);
md.update(buf, key_offset, key_len);
byte[] dig = md.digest();
partition_id = get_ntohl_intel(dig, 0);
}
// CAN'T USE MOD directly - mod will give negative numbers. First AND
// makes positive and negative correctly, then mod.
partition_id = (partition_id & 0xFFFF) % CLConnectionManager.n_server_partitions;
}
// BOUNCYCASTLE - seems quite a bit slower than gnu.crypto
// if (partition_id == -1) {
// GeneralDigest md = new RIPEMD160Digest();
// md.update(buf, set_offset, set_len);
// md.update(buf, key_offset, key_len);
// byte[] dig = new byte[20];
// md.doFinal(dig,0);
// partition_id = get_ntohl_intel(dig, 0);
// partition_id = partition_id & 0xFFF;
// }
return(offset-initial_offset);
}
private int makeBatchFields(byte[] buf, int offset, ArrayList<byte[]> alldigests) {
int initial_offset = offset;
//Prepare the namespace field
int ns_len = stringToUtf8(this.ns, buf, offset+5);
offset += set_field_header(FIELD_TYPE_NAMESPACE, ns_len, buf, offset) + ns_len;
//Prepare the digests field metadata
int digest_len = alldigests.size() * DIGEST_SIZE;
offset += set_field_header(FIELD_TYPE_DIGEST_RIPE_ARRAY, digest_len, buf, offset);
//Write out all the digests as a field
for (Iterator<byte[]> it=alldigests.iterator() ; it.hasNext() ; )
{
byte[] digest = it.next();
for (int i=0 ; i<digest.length ; i++)
{
buf[offset+i] = digest[i];
}
offset += digest.length;
}
return(offset-initial_offset);
}
//
// All requests have namespace, table, key, so make a byte array that includes
// all those fields, I think that's the fastest way
// Makes three headers, really
// 1 type = namespace
// 3 len
// [namespace bytes]
// 1 type = table
// 7 len
// [table bytes]
// 1 type = key
// 7 len
// 1 particle type
// key particle bytes
private int
makeFieldsEstimate() throws SerializeException
{
int offset = 0, op_estimate_sz;
byte[] value = null;
int ns_len = 0;
if (ns != null) {
try {
value = ns.getBytes("UTF8");
ns_len = value.length;
} catch (UnsupportedEncodingException e) {
// If you can't encode a UTF8 string you're in sad, sad shape
throw new SerializeException();
}
offset += 5 + ns_len;
}
if (digest != null) {
offset += 5 + digest.length;
}
int set_len = 0;
if (set != null) {
try {
value = set.getBytes("UTF8");
set_len = value.length;
} catch (UnsupportedEncodingException e) {
// If you can't encode a UTF8 string you're in sad, sad shape
throw new SerializeException();
}
offset += 5 + set_len;
}
try {
if (key != null) {
op_estimate_sz = makeOpEstimate(key);
if (op_estimate_sz == 0) {
byte[] jblob = objectToByteArray(key);
CitrusleafClient.info("Warning: Key is a serialized object\n");
if (jblob == null)
CitrusleafClient.info("jblob is NULL!!!\n");
op_estimate_sz = 8 + 32 + jblob.length;
}
offset += op_estimate_sz;
offset++;
}
} catch (Exception e) {
CitrusleafClient.info("Asbuffer makeFieldsEstimate: exception "+e.toString());
return(0);
}
if (this.scan_options!=null) {
offset += 2;
}
offset += 5;
return(offset);
}
private final int INFO1_READ = (1 << 0); // contains a read operation
private final int INFO1_GET_ALL = (1 << 1); // get all bins, period
@SuppressWarnings("unused")
private final int INFO1_GET_ALL_NODATA = (1 << 2); // get all bins WITHOUT data (currently unimplemented)
@SuppressWarnings("unused")
private final int INFO1_VERIFY = (1 << 3); // verify is a GET transaction that includes data, and assert if the data aint right
private final int INFO1_XDS = (1 << 4); // Operation is being performed by XDS
private final int INFO1_NOBINDATA = (1 << 5); // Do not read the bins
private final int INFO2_WRITE = (1 << 0); // contains a write semantic
private final int INFO2_DELETE = (1 << 1); // fling a record into the belly of Moloch
private final int INFO2_GENERATION = (1 << 2); // pay attention to the generation
private final int INFO2_GENERATION_GT = (1 << 3); // apply write if new generation >= old, good for restore
private final int INFO2_GENERATION_DUP = (1 << 4); // if a generation collision, create a duplicate
private final int INFO2_WRITE_UNIQUE = (1 << 5); // write only if it doesn't exist
@SuppressWarnings("unused")
private final int INFO2_WRITE_BINUNIQUE = (1 << 6);
@SuppressWarnings("unused")
private final int INFO3_LAST = (1 << 0); // this is the last of a multi-part message
@SuppressWarnings("unused")
private final int INFO3_TRACE = (1 << 1); // apply server trace logging for this transaction
@SuppressWarnings("unused")
private final int INFO3_TOMBSTONE = (1 << 2); // if set on response, a version was a delete tombstone
private final long CL_MSG_VERSION = 2L;
private final long AS_MSG_TYPE = 3L;
private final int CL_MSG_HEADER_LEN = 8;
private final int AS_MSG_HEADER_LEN = 22;
public String ns;
public String set;
public Object key;
public byte [] digest;
public byte [] scan_options; // 2 byte scan option
public long partition_id;
public CLConnectionManager.PartitionType partition_type;
private ClWriteOptions cl_wp;
private Collection<String> readBins;
private Collection<ClBin> writeBins;
private Collection<ClBin> appendBins;
private Collection<ClBin> prependBins;
private Collection<ClBin> addBins;
private Collection<ClBin> appendSegmentBins;
private Collection<ClBinAppendSegmentWriteOptions> appendSegmentExtWriteBins;
private Collection<ClBinAppendSegmentQueryOptions> appendSegmentQueryBins;
private boolean touch;
private boolean readAll;
private boolean readIterate;
private boolean readExists;
private boolean readNoBindata;
private ScanCallback callback;
private Object scan_user_data;
int generation;
int expiration;
boolean delete;
//parameters for batch operations
private boolean batchread = false;
private boolean gotlastmsg = false;
private ArrayList<Object> allkeys = null;
private ArrayList<byte[]> alldigests = null;
private byte msg_header_buf[]; // A 30 byte buffer used for reading just the response header
public byte msg_write_buf[];
public long msg_write_len;
public byte msg_read_buf[]; // used to write the request, and read in the response
public CLBuffer()
{
ns = null;
set = null;
key = null;
digest = null;
scan_options = null;
cl_wp = null;
partition_id = -1;
partition_type = CLConnectionManager.PartitionType.READ; // write is more generic
readBins = null;
writeBins = null;
prependBins = null;
appendBins = null;
addBins = null;
appendSegmentBins = null;
appendSegmentExtWriteBins = null;
appendSegmentQueryBins = null;
readAll = false;
touch = false;
readIterate = false;
readExists = false;
readNoBindata = false;
generation = 0;
delete = false;
batchread = false;
allkeys = null;
alldigests = null;
msg_header_buf = new byte[CL_MSG_HEADER_LEN + AS_MSG_HEADER_LEN];
msg_write_buf = null;
msg_read_buf = null;
msg_write_len = 0;
}
void reset() {
ns = null;
set = null;
key = null;
digest = null;
scan_options = null;
cl_wp = null;
partition_id = -1;
partition_type = CLConnectionManager.PartitionType.READ; // write is more generic
readBins = null;
writeBins = null;
prependBins = null;
appendBins = null;
addBins = null;
appendSegmentBins = null;
appendSegmentExtWriteBins = null;
appendSegmentQueryBins = null;
readAll = false;
touch = false;
readIterate = false;
readExists = false;
readNoBindata = false;
generation = 0;
delete = false;
batchread = false;
allkeys = null;
alldigests = null;
// really just trying to save the allocation of these two arrays. Worth it???
msg_header_buf = new byte[CL_MSG_HEADER_LEN + AS_MSG_HEADER_LEN];
msg_write_len = 0;
msg_write_buf = null;
msg_read_buf = null;
}
// Concurrent linked queues are slow to find the size, and
static java.util.concurrent.atomic.AtomicInteger CLBufferQueueSize =
new java.util.concurrent.atomic.AtomicInteger(0);
static java.util.concurrent.ConcurrentLinkedQueue<CLBuffer> CLBufferQueue =
new java.util.concurrent.ConcurrentLinkedQueue<CLBuffer> ();
public static CLBuffer create()
{
CLBuffer b = CLBufferQueue.poll();
if (b == null) {
b = new CLBuffer();
}
else {
CLBufferQueueSize.decrementAndGet();
}
return(b);
}
public void destroy()
{
if (CLBufferQueueSize.get() < 50) {
this.reset();
CLBufferQueue.add(this);
CLBufferQueueSize.decrementAndGet();
}
}
//
// Sets singleton initial values
// Function must be expanded for secondary references
public void set_required(String namespace, String set, Object key, ClWriteOptions cl_wp)
{
this.ns = namespace;
this.set = set;
this.key = key;
this.cl_wp = cl_wp;
}
//
// Set digest values
public void set_required(String namespace, byte[] digest , ClWriteOptions cl_wp)
{
this.ns = namespace;
this.digest = digest;
this.cl_wp = cl_wp;
}
// left-to-right bits
// 0-3 priority hint = ClScanningPriority
// 4 = failOnClusterChange
// 8-15 = scan_pct (1 byte)
public void scan_options(ClScanningOptions scan_opt,int scan_pct)
{
scan_options = new byte[2];
scan_options[0] = (byte)scan_opt.getPriority().ordinal();
scan_options[0] <<= 4;
if (scan_opt.isFailOnClusterChange()) {
scan_options[0] |= 0x08;
}
scan_options[1] = (byte) scan_pct;
}
// on a read request - read this bin too
public void addRead(String bin)
{
if (bin == null) {
this.readAll();
} else {
if (readBins == null) {
readBins = new ArrayList<String>(5);
}
readBins.add(bin);
}
}
// NB: it could be interesting to do a clone. The documentation
// should state what happens to the collection. It would be even faster
// to simply hand off the collection to our interface, if they're not using
// it or know it won't change (like, it's a final), so we could have an api
// for that
public void addRead(Collection<String> bins)
{
if (bins == null) {
this.readAll();
} else {
if (readBins == null)
readBins = bins;
else
readBins.addAll(bins);
}
}
public void readAll()
{
// if we previously had explict bins, throw them away
if (readBins != null)
readBins = null;
readAll = true;
}
public void readIterate(ScanCallback sc, Object user_data)
{
if (readBins != null)
readBins = null;
readIterate = true;
callback = sc;
scan_user_data = user_data;
}
public void readExists()
{
readExists = true;
}
public void readNoBindata()
{
readNoBindata = true;
}
public boolean isReadIterate()
{
return(readIterate);
}
// on a write request - write this bin too
public void addWrite(Collection<ClBin> bins)
{
if (writeBins == null)
writeBins = bins;
else
writeBins.addAll(bins);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a write request - write these bins too
public void addWrite(Map<String, Object> bins)
{
if (writeBins == null)
writeBins = new ArrayList<ClBin>(bins.size());
for (String key : bins.keySet()) {
ClBin asc = new ClBin();
asc.name = key;
asc.value = bins.get(key);
writeBins.add(asc);
}
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addWrite(String bin, Object o)
{
if (writeBins == null)
writeBins = new ArrayList<ClBin>(5);
ClBin asc = new ClBin();
asc.name = bin;
asc.value = o;
writeBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a prepend request - prepend this bin too
public void addPrepend(Collection<ClBin> bins)
{
if (prependBins == null)
prependBins = bins;
else
prependBins.addAll(bins);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a prepend request - prepend these bins too
public void addPrepend(Map<String, Object> bins)
{
if (prependBins == null)
prependBins = new ArrayList<ClBin>(bins.size());
for (String key : bins.keySet()) {
ClBin asc = new ClBin();
asc.name = key;
asc.value = bins.get(key);
prependBins.add(asc);
}
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addPrepend(String bin, Object o)
{
if (prependBins == null)
prependBins = new ArrayList<ClBin>(5);
ClBin asc = new ClBin();
asc.name = bin;
asc.value = o;
prependBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a append request - aappend this bin too
public void addAppend(Collection<ClBin> bins)
{
if (appendBins == null)
appendBins = bins;
else
appendBins.addAll(bins);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a append request - append these bins too
public void addAppend(Map<String, Object> bins)
{
if (appendBins == null)
appendBins = new ArrayList<ClBin>(bins.size());
for (String key : bins.keySet()) {
ClBin asc = new ClBin();
asc.name = key;
asc.value = bins.get(key);
appendBins.add(asc);
}
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addAppend(String bin, Object o)
{
if (appendBins == null)
appendBins = new ArrayList<ClBin>(5);
ClBin asc = new ClBin();
asc.name = bin;
asc.value = o;
appendBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on an add request - add this bin too
public void addAdd(Collection<ClBin> bins)
{
if (addBins == null)
addBins = bins;
else
addBins.addAll(bins);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on an add request - add these bins too
public void addAdd(Map<String, Object> bins)
{
if (addBins == null)
addBins = new ArrayList<ClBin>(bins.size());
for (String key : bins.keySet()) {
ClBin asc = new ClBin();
asc.name = key;
asc.value = bins.get(key);
addBins.add(asc);
}
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addAdd(String bin, Object o)
{
if (addBins == null)
addBins = new ArrayList<ClBin>(5);
ClBin asc = new ClBin();
asc.name = bin;
asc.value = o;
addBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a write request - write this bin too
public void addAppendSegment(Collection<ClBin> bins)
{
if (appendSegmentBins == null)
appendSegmentBins = bins;
else
appendSegmentBins.addAll(bins);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
// on a write request - write these bins too
public void addAppendSegment(Map<String, Object> bins)
{
if (appendSegmentBins == null)
appendSegmentBins = new ArrayList<ClBin>(bins.size());
for (String key : bins.keySet()) {
ClBin asc = new ClBin();
asc.name = key;
asc.value = bins.get(key);
appendSegmentBins.add(asc);
}
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addAppendSegment(String bin, Object o)
{
if (appendSegmentBins == null)
appendSegmentBins = new ArrayList<ClBin>(5);
ClBin asc = new ClBin();
asc.name = bin;
asc.value = o;
appendSegmentBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addAppendSegmentQuery(String bin, Collection <ClAppendSegmentQueryFilter> filters,
int max_results)
{
if (appendSegmentQueryBins == null)
appendSegmentQueryBins = new ArrayList<ClBinAppendSegmentQueryOptions>(5);
ClBinAppendSegmentQueryOptions asc = new ClBinAppendSegmentQueryOptions(bin, null, filters, max_results);
appendSegmentQueryBins.add(asc);
}
public void addAppendSegmentExt(String bin, Object o, ClAppendSegmentWriteOptions append_segment_opts)
{
if (appendSegmentExtWriteBins == null)
appendSegmentExtWriteBins = new ArrayList<ClBinAppendSegmentWriteOptions>(5);
ClBinAppendSegmentWriteOptions asc = new ClBinAppendSegmentWriteOptions(bin, o, append_segment_opts);
appendSegmentExtWriteBins.add(asc);
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void addTouch() {
touch = true;
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void setDelete() {
this.delete = true;
partition_type = CLConnectionManager.PartitionType.WRITE;
}
public void setBatchRead(ArrayList<Object> keys, ArrayList<byte[]> alldigests) {
this.batchread = true;
this.allkeys = keys;
this.alldigests = alldigests;
}
public boolean isBatchRead() {
return this.batchread;
}
//
// Do all the buffer preparation that makes it ready to read or write
// (Should this be internal? almost doesn't matter, since the entire interface
// is internal)
public void prepare() throws SerializeException {
byte[][] jblobs = null;
int sz_estimate = 30;
if (readBins != null)
sz_estimate += readBins.size() * 50;
if (writeBins != null) {
jblobs = new byte[writeBins.size()][];
for (int i = 0; i < writeBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBin col : writeBins) {
int op_estimate_sz = makeOpEstimate(col.value);
if (op_estimate_sz == 0) {
jblobs[i] = objectToByteArray(col.value);
op_estimate_sz = 8 + 32 + jblobs[i].length;
}
sz_estimate += op_estimate_sz;
i++;
}
}
if (appendBins != null) {
jblobs = new byte[appendBins.size()][];
for (int i = 0; i < appendBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBin col : appendBins) {
int op_estimate_sz = makeOpEstimate(col.value);
if (op_estimate_sz == 0) {
jblobs[i] = objectToByteArray(col.value);
op_estimate_sz = 8 + 32 + jblobs[i].length;
}
sz_estimate += op_estimate_sz;
i++;
}
}
if (prependBins != null) {
jblobs = new byte[prependBins.size()][];
for (int i = 0; i < prependBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBin col : prependBins) {
int op_estimate_sz = makeOpEstimate(col.value);
if (op_estimate_sz == 0) {
jblobs[i] = objectToByteArray(col.value);
op_estimate_sz = 8 + 32 + jblobs[i].length;
}
sz_estimate += op_estimate_sz;
i++;
}
}
if (addBins != null) {
jblobs = new byte[addBins.size()][];
for (int i = 0; i < addBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBin col : addBins) {
int op_estimate_sz = makeOpEstimate(col.value);
if (op_estimate_sz == 0) {
jblobs[i] = objectToByteArray(col.value);
op_estimate_sz = 8 + 32 + jblobs[i].length;
}
sz_estimate += op_estimate_sz;
i++;
}
}
if (appendSegmentBins != null) {
jblobs = new byte[appendSegmentBins.size()][];
for (int i = 0; i < appendSegmentBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBin col : appendSegmentBins) {
int op_estimate_sz = makeOpEstimate(col.value);
if (op_estimate_sz == 0) {
jblobs[i] = objectToByteArray(col.value);
op_estimate_sz = 8 + 32 + jblobs[i].length;
}
sz_estimate += op_estimate_sz;
i++;
}
}
if (appendSegmentExtWriteBins != null) {
jblobs = new byte[appendSegmentExtWriteBins.size()][];
for (int i = 0; i < appendSegmentExtWriteBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBinAppendSegmentWriteOptions col : appendSegmentExtWriteBins) {
// now serialize with protobuf...
ClProtobuf.AppendSegmentWriteOption.Builder write_opts_builder =
ClProtobuf.AppendSegmentWriteOption.newBuilder();
ClAppendSegmentWriteOptions write_opts = col.write_opts;
ParticleBytesReturn pbr = objectToParticleBytes(col.value, null, 0, null);
byte[] byteArray = new byte[pbr.size];
objectToParticleBytes(col.value, byteArray, 0, null);
write_opts_builder.setValue(com.google.protobuf.ByteString.copyFrom(byteArray));
if( write_opts.tag != null )
write_opts_builder.setTag(write_opts.tag);
if( write_opts.expiration != 0 )
write_opts_builder.setTimeExpires(write_opts.expiration/1000);
if( write_opts.creation_time != null )
write_opts_builder.setCreationTime(new Long(write_opts.creation_time.getTime()/1000).intValue());
jblobs[i] = write_opts_builder.build().toByteArray();
sz_estimate += 8 + 32 + jblobs[i].length;
i++;
}
}
if (appendSegmentQueryBins != null) {
jblobs = new byte[appendSegmentQueryBins.size()][];
for (int i = 0; i < appendSegmentQueryBins.size(); i++)
jblobs[i] = null;
int i = 0;
for (ClBinAppendSegmentQueryOptions col : appendSegmentQueryBins) {
// now serialize with protobuf...
ClProtobuf.AppendSegmentQuery.Builder query_builder =
ClProtobuf.AppendSegmentQuery.newBuilder();
// set max_results - simple
query_builder.setMaxResponses(col.max_results);
// query options - a little harder. Need to iterate through list of filters
if( col.query_filters != null ){
for (ClAppendSegmentQueryFilter filter : col.query_filters) {
// create filter
ClProtobuf.AppendSegmentQuery.AppendSegmentQueryFilter.Builder query_filter_builder =
ClProtobuf.AppendSegmentQuery.AppendSegmentQueryFilter.newBuilder();
if( filter.start_time != null )
query_filter_builder.setStartTime(new Long(filter.start_time.getTime()/1000).intValue());
if( filter.end_time != null )
query_filter_builder.setEndTime(new Long(filter.end_time.getTime()/1000).intValue());
if( filter.tag != null )
query_filter_builder.setTag(filter.tag);
query_builder.addFilter(query_filter_builder);
}
}
jblobs[i] = query_builder.build().toByteArray();
sz_estimate += 8 + 32 + jblobs[i].length; // XXX why is this the size?
i++;
}
}
int fields_sz = makeFieldsEstimate();
sz_estimate += fields_sz;
if (sz_estimate < 2 * 1024) sz_estimate = 2 * 1024;
// CitrusleafClient.info("size estimate final "+sz_estimate);
if ((msg_write_buf == null) ||
(msg_write_buf.length < sz_estimate)) {
msg_write_buf = new byte[sz_estimate];
}
int offset = 30;
offset += makeFields(msg_write_buf, offset);
byte info1 = 0;
byte info2 = 0;
byte info3 = 0;
if (readBins != null) {
for (String s : readBins) {
offset += makeOp(OP_READ, s, null, msg_write_buf, offset, null);
}
info1 |= INFO1_READ;
}
else if (readAll) {
info1 |= INFO1_GET_ALL;
info1 |= INFO1_READ;
} else if (readIterate || readExists) {
info1 |= INFO1_READ;
}
if (readNoBindata) {
info1 |= INFO1_NOBINDATA;
}
if (writeBins != null) {
int i = 0;
for (ClBin col : writeBins) {
offset += makeOp(OP_WRITE, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (appendBins != null) {
int i = 0;
for (ClBin col : appendBins) {
offset += makeOp(OP_APPEND, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (prependBins != null) {
int i = 0;
for (ClBin col : prependBins) {
offset += makeOp(OP_PREPEND, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (touch) {
offset += makeOp(OP_TOUCH, "", "", msg_write_buf, offset, null);
info2 |= INFO2_WRITE;
}
if (addBins != null) {
int i = 0;
for (ClBin col : addBins) {
offset += makeOp(OP_ADD, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (appendSegmentBins != null) {
int i = 0;
for (ClBin col : appendSegmentBins) {
offset += makeOp(OP_APPEND_SEGMENT, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (appendSegmentExtWriteBins != null) {
int i = 0;
for( ClBin col : appendSegmentExtWriteBins ){
offset += makeOp(OP_APPEND_SEGMENT_EXT, col.name, col.value, msg_write_buf, offset, jblobs[i]);
i++;
}
info2 |= INFO2_WRITE;
}
if (appendSegmentQueryBins != null ){
int i = 0;
for( ClBin col : appendSegmentQueryBins ){
offset += makeOp(OP_APPEND_SEGMENT_QUERY, col.name, "foobar", msg_write_buf, offset, jblobs[i]);
i++;
}
info1 |= INFO1_READ;
}
msg_write_len = offset;
int generation = 0;
if (cl_wp != null) {
if (cl_wp.mUseGeneration) {
info2 |= INFO2_GENERATION;
generation = cl_wp.mGeneration;
}
else if (cl_wp.mUseGenerationGt) {
info2 |= INFO2_GENERATION_GT;
generation = cl_wp.mGeneration;
}
else if (cl_wp.mUseGenerationDup) {
info2 |= INFO2_GENERATION_DUP;
generation = cl_wp.mGeneration;
}
else if (cl_wp.unique) {
info2 |= INFO2_WRITE_UNIQUE;
}
}
if (this.delete) {
info2 |= INFO2_WRITE | INFO2_DELETE;
}
// Make the actual header
long sz = (msg_write_len - 8) | (CL_MSG_VERSION << 56) | (AS_MSG_TYPE << 48);
set_htonll(sz, msg_write_buf, 0);
msg_write_buf[8] = AS_MSG_HEADER_LEN; // 22!
msg_write_buf[9] = info1;
msg_write_buf[10] = info2;
msg_write_buf[11] = info3;
msg_write_buf[12] = 0; // unused
msg_write_buf[13] = 0; // clear the result code
set_htonl(generation, msg_write_buf, 14);
// record ttl
if (cl_wp != null) {
set_htonl(cl_wp.expiration, msg_write_buf, 18);
}
else {
msg_write_buf[18] = msg_write_buf[19] = msg_write_buf[20] = msg_write_buf[21] = 0;
}
// transaction ttl - init to reasonable value
msg_write_buf[22] = msg_write_buf[23] = msg_write_buf[24] = msg_write_buf[25] = 0;
// number of fields: if digest n_fileds=2 (for namespace and digest)
// else one each for namespace, set and key
int n_fields = 0;
if (this.digest != null) {
n_fields = 2;
} else {
n_fields = ((this.key != null)?1:0) + ((this.set != null)?1:0) + ((this.ns != null)?1:0)+ ((this.scan_options !=null)?1:0);
}
set_htons(n_fields, msg_write_buf, 26);
int n_ops = 0;
if (readBins != null) n_ops += readBins.size();
if (writeBins != null) n_ops += writeBins.size();
if (prependBins != null) n_ops += prependBins.size();
if (appendBins != null) n_ops += appendBins.size();
if (addBins != null) n_ops += addBins.size();
if (appendSegmentBins != null) n_ops += appendSegmentBins.size();
if (appendSegmentExtWriteBins != null) n_ops += appendSegmentExtWriteBins.size();
if (appendSegmentQueryBins != null) n_ops += appendSegmentQueryBins.size();
set_htons(n_ops,msg_write_buf,28);
// DEBUG
// dump_buf("write msg ",msg_write_buf,(int)msg_write_len);
}
// Updates the timeout field..
// uses the 'start time' that was captured when the buffer was created
// and the policies in the ClOptions
public void setTimeout(int ms_remaining) {
set_htonl(ms_remaining, msg_write_buf, 22);
}
// In order to profile, pick this apart into multiple methods
private ClResult parseResult(byte[] msg, int len, int n_ops, int n_fields, int resultCode, int generation) throws SerializeException
{
ClResult r = new ClResult();
r.generation = generation; // yo! sup with this???
switch (resultCode) {
case 0:
r.resultCode = ClResultCode.OK;
break;
case 1:
default:
r.resultCode = ClResultCode.SERVER_ERROR;
break;
case 2:
r.resultCode = ClResultCode.KEY_NOT_FOUND_ERROR;
break;
case 3:
r.resultCode = ClResultCode.GENERATION_ERROR;
break;
case 4:
r.resultCode = ClResultCode.PARAMETER_ERROR;
break;
case 5:
r.resultCode = ClResultCode.KEY_EXISTS_ERROR;
break;
case 6:
r.resultCode = ClResultCode.BIN_EXISTS_ERROR;
break;
}
// if there are headers, we'll have to skip over them,
// but for now, how about just alerting if there are any?
if (n_fields != 0) {
CitrusleafClient.info("recv message: there are fields set, unexpected, failure");
return(null);
}
int offset = 0;
for (int i=0 ; i<n_ops ; i++) {
int op_sz = get_ntohl(msg, offset);
byte particle_type = msg[offset+5];
byte version = msg[offset+6];
byte name_sz = msg[offset+7];
String name = utf8ToString(msg, offset+8, name_sz);
offset += 4 + 4 + name_sz;
// System.out.printf("op size %d op %d part-type %d name_sz %d name %s\n",
// op_sz, op, particle_type, name_sz, name);
int particleBytes_sz = (int) (op_sz - (4 + name_sz));
Object value = null;
try {
value = bytesToParticle(particle_type, msg, offset, particleBytes_sz);
offset += particleBytes_sz;
}
catch (SerializeException e) {
if (key != null) {
CitrusleafClient.info(" de-serialize exception: namespace "+ns+" set "+set+" key "+key.toString() );
}
else {
CitrusleafClient.info(" de-serialize exception: namespace "+ns+" set "+set);
}
throw e;
}
// place this nvp somewhere cool
// I detest ArrayList. I should be able to set() and have it auto-grow.
Map<String,Object> vmap = null;
if (version > 0 || r.results_dup != null) {
// CitrusleafClient.info(" version greater than 0? srsly? "+version);
if (r.results_dup == null) {
r.results_dup = new ArrayList<Map<String,Object>>(4);
r.results_dup.add(r.results);
r.results = null;
for (int j=0;j<version;j++) r.results_dup.add(null);
} else {
for (int j=r.results_dup.size(); j<version+1;j++) r.results_dup.add(null);
}
vmap = r.results_dup.get(version);
if (vmap == null) {
vmap = new HashMap<String,Object>();
r.results_dup.set(version,vmap);
}
}
else {
if (r.results == null){
r.results = new HashMap<String,Object>();
}
vmap = r.results;
}
vmap.put(name, value);
}
// just in case there were holes in the versin number space
if (r.results_dup != null)
while ( r.results_dup.remove(null) );
return(r);
}
// Method to parse multiple results in case of scan API
private ClResult parseMultiResult(byte[] msg, int n_bytes) throws SerializeException
{
ClResult r = null;
byte [] digest = null;
String nsname = null;
String setname = null;
int offset = 0;
do
{
r = new ClResult(ClResultCode.OK);
byte h_len = msg[offset];
byte info1 = msg[offset+1];
byte info2 = msg[offset+2];
byte info3 = msg[offset+3];
byte unused = msg[offset+4];
byte resultCode = msg[offset+5];
int generation = get_ntohl(msg, offset + 6);
int record_ttl = get_ntohl(msg, offset + 10);
int transaction_ttl = get_ntohl(msg, offset + 14);
int n_fields = get_ntohs(msg, offset + 18);
int n_ops = get_ntohs(msg, offset + 20);
offset += 22;
if (h_len != 22)
{
r.resultCode = ClResultCode.SERVER_ERROR;
return (r);
}
// check for last message
if ((info3 & INFO3_LAST) == INFO3_LAST)
{
gotlastmsg = true;
return (r);
}
r.generation = generation;
setResultCode(r, resultCode);
if (r.resultCode != ClResultCode.OK)
{
return (r);
}
// Parse the fields
if (n_fields > 0) {
// grab digest, namespace might be in there too
for (int i = 0; i < n_fields; i++) {
int field_sz = (int)get_ntohl(msg, offset);
byte type = msg[offset + 4];
if (type == FIELD_TYPE_DIGEST_RIPE) {
digest = new byte[field_sz - 1];
System.arraycopy(msg, offset + 5, digest, 0, field_sz - 1);
} else if (type == FIELD_TYPE_NAMESPACE) {
nsname = new String(msg, offset + 5, field_sz - 1);
} else if (type == FIELD_TYPE_TABLE) {
setname = new String(msg, offset + 5, field_sz - 1);
}
// do something with the data? there's a namespace here.
offset += 4 + field_sz;
}
}
// Parse ops
try {
// Pass continueOnError as true when calling from scan
r.corruptedData = false; //Assume that there is no corruption to start with
offset += parseOps(r, msg, offset, n_ops, true);
}
catch (SerializeException e) {
CitrusleafClient.info("de-serialize exception during scan. Digest="+byteArrayToHexString(digest));
throw e;
}
// If data is corrupted, print error and continue scan
if (r.corruptedData) {
CitrusleafClient.info("de-serialize exception during scan. Digest="+byteArrayToHexString(digest));
continue;
}
// Call the callback function with the values
callback.scanCallback(nsname, setname, digest, r.results, generation, record_ttl, scan_user_data);
} while (offset < n_bytes);
return(r);
}
// Special prepare function to read batch of digests
public void batchPrepare() throws SerializeException {
int sz_estimate = 30;
if (readBins != null) {
sz_estimate += readBins.size() * 50;
}
int fields_sz = makeFieldsEstimate();
sz_estimate += fields_sz;
//we should also consider the size of all the digests for the fields
sz_estimate += alldigests.size() * DIGEST_SIZE;
if (sz_estimate < 2 * 1024) {
sz_estimate = 2 * 1024;
}
if ((msg_write_buf == null) || (msg_write_buf.length < sz_estimate)) {
msg_write_buf = new byte[sz_estimate];
}
int offset = 30;
offset += makeBatchFields(msg_write_buf, offset, alldigests);
byte info1 = 0;
byte info2 = 0;
byte info3 = 0;
if (readBins != null) {
for (String s : readBins) {
offset += makeOp(OP_READ, s, null, msg_write_buf, offset, null);
}
info1 |= INFO1_READ;
} else if (readAll) {
info1 |= INFO1_GET_ALL;
info1 |= INFO1_READ;
} else if (readExists && readNoBindata) {
info1 |= INFO1_NOBINDATA;
info1 |= INFO1_READ;
}
msg_write_len = offset;
int generation = 0;
// We do not currently support writeparameters
assert (cl_wp == null) : cl_wp;
// Make the actual header
long sz = (msg_write_len - 8) | (CL_MSG_VERSION << 56) | (AS_MSG_TYPE << 48);
set_htonll(sz, msg_write_buf, 0);
msg_write_buf[8] = AS_MSG_HEADER_LEN; // 22!
msg_write_buf[9] = info1;
msg_write_buf[10] = info2;
msg_write_buf[11] = info3;
msg_write_buf[12] = 0; // unused
msg_write_buf[13] = 0; // clear the result code
set_htonl(generation, msg_write_buf, 14);
// record ttl
msg_write_buf[18] = msg_write_buf[19] = msg_write_buf[20] = msg_write_buf[21] = 0;
// transaction ttl
msg_write_buf[22] = msg_write_buf[23] = msg_write_buf[24] = msg_write_buf[25] = 0;
// number of fields=2: Namespace and digests
set_htons(2, msg_write_buf, 26);
int n_ops = 0;
if (readBins != null) {
n_ops += readBins.size();
}
set_htons(n_ops,msg_write_buf,28);
}
//Sets the corresponding result code in the result object
private void setResultCode(ClResult r, int resultCode)
{
switch (resultCode) {
case 0:
r.resultCode = ClResultCode.OK;
break;
case 1:
default:
r.resultCode = ClResultCode.SERVER_ERROR;
break;
case 2:
r.resultCode = ClResultCode.KEY_NOT_FOUND_ERROR;
break;
case 3:
r.resultCode = ClResultCode.GENERATION_ERROR;
break;
case 4:
r.resultCode = ClResultCode.PARAMETER_ERROR;
break;
case 5:
r.resultCode = ClResultCode.KEY_EXISTS_ERROR;
break;
case 6:
r.resultCode = ClResultCode.BIN_EXISTS_ERROR;
break;
case 7:
r.resultCode = ClResultCode.CLUSTER_KEY_MISMATCH;
break;
case 8:
r.resultCode = ClResultCode.SERVER_MEM_ERROR;
break;
// Note - server timeouts are conflated with client timeouts.
case 9:
r.resultCode = ClResultCode.TIMEOUT;
break;
case 10:
r.resultCode = ClResultCode.NO_XDS;
break;
case 11:
r.resultCode = ClResultCode.SERVER_NOT_AVAILABLE;
break;
case 12:
r.resultCode = ClResultCode.BIN_TYPE_ERROR;
break;
case 13:
r.resultCode = ClResultCode.RECORD_TOO_BIG;
break;
case 14:
r.resultCode = ClResultCode.KEY_BUSY;
break;
}
}
//Parses the given byte buffer and populate the result object
//Retruns the number of bytes that were parsed from the given buffer
private int parseOps(ClResult r, byte[] buf, int opsoffset, int n_ops, boolean continueOnError) throws SerializeException
{
int initialoffset = opsoffset;
boolean skipbin=false;
for (int i=0 ; i<n_ops ; i++) {
int op_sz = get_ntohl(buf, opsoffset);
byte op = buf[opsoffset + 4];
byte op_type = buf[opsoffset + 5];
byte version = buf[opsoffset + 6];
byte name_sz = buf[opsoffset + 7];
String name = utf8ToString(buf, opsoffset + 8, name_sz);
opsoffset += 4 + 4 + name_sz;
//Currently, the batch command returns all the bins even if a subset of
//the bins are requested. So, we have to filter it on the client side.
//We have to filter if the client does not want to read all bins
skipbin = false;
if (!readAll && (readBins != null) && !readBins.contains(name)) {
//System.out.println("skipping the bin "+name);
skipbin = true;;
}
int particleBytes_sz = (int) (op_sz - (4 + name_sz));
Object value = null;
try {
value = bytesToParticle(op_type, buf, opsoffset, particleBytes_sz);
}
catch (SerializeException e) {
// In case of call from scan, continueOnError is set, so avoid throwing exception and set that data is corrupted
if (continueOnError) {
r.corruptedData = true;
} else {
//Key may not be set in some cases like the scan
if (key != null) {
CitrusleafClient.info(" de-serialize exception: namespace "+ns+" set "+set+" key "+key.toString() );
}
else {
CitrusleafClient.info(" de-serialize exception: namespace "+ns+" set "+set);
}
throw e;
}
}
opsoffset += particleBytes_sz;
// If data is corrupted, continue without processing the result
if (r.corruptedData) {
continue;
}
if (!skipbin) {
// place this nvp somewhere cool
// I detest ArrayList. I should be able to set() and have it auto-grow.
Map<String,Object> vmap = null;
if (version > 0 || r.results_dup != null) {
// CitrusleafClient.info(" version greater than 0? srsly? "+version);
if (r.results_dup == null) {
r.results_dup = new ArrayList<Map<String,Object>>(4);
r.results_dup.add(r.results);
r.results = null;
for (int j=0;j<version;j++) r.results_dup.add(null);
} else {
for (int j=r.results_dup.size(); j<version+1;j++) r.results_dup.add(null);
}
vmap = r.results_dup.get(version);
if (vmap == null) {
vmap = new HashMap<String,Object>();
r.results_dup.set(version,vmap);
}
}
else {
if (r.results == null)
r.results = new HashMap<String,Object>();
vmap = r.results;
}
vmap.put(name, value);
}
}
// just in case there were holes in the versin number space
if (r.results_dup != null)
{
while ( r.results_dup.remove(null) );
}
return (opsoffset - initialoffset);
}
// Given a contigious byte array of all the respose messages this function
// will parse each message and create a ClResult object out of it and store
// them in an array which is passed by the caller. The result will be placed
// in the position corresponding to the position of the key i.e The index
// of the key in the registed arraylist and that of its corresponding result
// in the clresult array that is returned will be the same. If there is no
// data corresponding to a key in the response from the server, its clresult
// will be set to null. The array of clresult objects will be a field in the
// ClResult object that is returned. The caller has to use the array.
private void parseBatchResult(ClResult r_main, byte[] buf, int buflen) throws SerializeException
{
int msgoffset, fieldoffset, opsoffset, keyindex;
byte[] digest = new byte[DIGEST_SIZE];
String ns = null;
ClResult r = null;
//Parse each message response and add it to th clresult array
msgoffset = 0;
while (msgoffset < buflen) {
keyindex = -2; //will be used to know if we got a digest in this msg or not
r = null;
byte h_len = buf[msgoffset + 0];
byte info1 = buf[msgoffset + 1];
byte info2 = buf[msgoffset + 2];
byte info3 = buf[msgoffset + 3];
byte unused = buf[msgoffset + 4];
int resultCode = buf[msgoffset + 5];
int generation = get_ntohl(buf, msgoffset + 6);
int record_ttl = get_ntohl(buf, msgoffset + 10);
int txn_ttl = get_ntohl(buf, msgoffset + 14);
int n_fields = get_ntohs(buf, msgoffset + 18);
int n_ops = get_ntohs(buf, msgoffset + 20);
if (h_len != AS_MSG_HEADER_LEN) {
CitrusleafClient.info("read header batch: unexpected header size");
}
// If this is the end marker of the response, do not proceed further
if ((info3 & INFO3_LAST) == INFO3_LAST) {
gotlastmsg = true;
return;
}
//Response msg for batch will have fields.
fieldoffset = msgoffset + AS_MSG_HEADER_LEN;
for (int i=0; i<n_fields; i++) {
int fieldlen = get_ntohl(buf, fieldoffset);
int fieldtype = buf[fieldoffset + 4];
if (fieldtype == FIELD_TYPE_KEY) {
CitrusleafClient.info("read message batch: unexpectedly found key");
}
else if (fieldtype == FIELD_TYPE_DIGEST_RIPE) {
for (int j=0; j<DIGEST_SIZE; j++) {
digest[j] = buf[fieldoffset + 5 + j];
}
// Find the index of the corresponding key for the digest
// indexOf() cannot be used for byte arrays. So, we do it ourselves
int k;
Iterator<byte[]> it;
for (it=alldigests.iterator(), k=0 ; it.hasNext(); k++ ) {
byte[] querydigest = it.next();
if (Arrays.equals(digest, querydigest))
{
keyindex = k;
break;
}
}
}
else if (fieldtype == FIELD_TYPE_NAMESPACE) {
//Remember, one byte is used for type out of field
ns = new String(buf, fieldoffset + 5, fieldlen-1);
}
//Next field is at (4bytes of filedlen value + field length)
fieldoffset += 4 + fieldlen;
}
if (keyindex == -2) {
// This avoids throwing an ugly exception on batch requests
// using non-existent namespaces. [PSI]
if (ClResultCode.NOT_SET == ClResultCode.values()[resultCode]) {
r_main.resultCode = ClResultCode.PARAMETER_ERROR;
gotlastmsg = true;
}
CitrusleafClient.info("read message batch: Did not get any digest in this msg, aborting midway");
return;
}
else if (keyindex == -1) {
CitrusleafClient.info("read message batch: Got an unexpected digest, aborting midway");
return;
}
else
{
r_main.resultCode = ClResultCode.OK;
r_main.resultArray[keyindex] = new ClResult();
r = r_main.resultArray[keyindex];
}
r.generation = generation;
setResultCode(r, resultCode);
//Now read the ops
opsoffset = fieldoffset;
// Pass continueOnError as false when calling from batch
opsoffset += parseOps(r, buf, opsoffset, n_ops, false);
//Move on to the next msg
msgoffset = opsoffset;
}
return;
}
public ClResult get_and_parse_batch(InputStream is) throws IOException, SerializeException
{
ClResult r_main = new ClResult();
int numkeys = allkeys.size();
r_main.resultArray = new ClResult[numkeys];
for (int i=0; i<numkeys ; i++) {
r_main.resultArray[i] = null;
}
gotlastmsg = false;
do {
int rlen = 0;
while (rlen < CL_MSG_HEADER_LEN) {
int read_rv = is.read(msg_header_buf, rlen, CL_MSG_HEADER_LEN - rlen);
if (read_rv < 0) {
throw new java.net.SocketException("socket closed");
}
rlen += read_rv;
}
long sz = get_ntohll(msg_header_buf, 0);
byte cl_version = (byte) (((int)(sz >> 56)) & 0xff);
byte cl_type = (byte) (((int)(sz >> 48)) & 0xff);
// In case of batch result, we will get multiple msgs as response
// Get the size of all the msgs together. Each msg will have its
// own header and body.
int allmsgs_bytes = ((int) (sz & 0xFFFFFFFFFFFFL));
if (cl_version != CL_MSG_VERSION) {
CitrusleafClient.info("read header batch: incorrect version");
}
if (cl_type != AS_MSG_TYPE) {
CitrusleafClient.info("read header batch: incorrect message type");
}
//Read all the messages along with their headers and store in a buffer
int alloc_bytes = 2 * 1024;
if (alloc_bytes < allmsgs_bytes) {
alloc_bytes = allmsgs_bytes;
}
if ((msg_read_buf == null) || (msg_read_buf.length < alloc_bytes)) {
msg_read_buf = new byte[alloc_bytes];
}
rlen = 0;
while (rlen < allmsgs_bytes) {
int read_rv = is.read(msg_read_buf, rlen, allmsgs_bytes - rlen);
if (read_rv < 0) {
throw new java.net.SocketException("socket closed");
}
rlen += read_rv;
}
//dump_buf("batch response header", msg_header_buf, CL_MSG_HEADER_LEN);
//dump_buf("batch response msgs", msg_read_buf, allmsgs_bytes);
//gotlastmsg is a class variable and will be modified by this function.
parseBatchResult(r_main, msg_read_buf, allmsgs_bytes);
} while (!gotlastmsg);
return r_main;
}
// Get and parse method for scan API
public ClResult multi_get_and_parse(InputStream is) throws IOException, SerializeException
{
ClResult r = new ClResult(ClResultCode.OK);
gotlastmsg = false;
do {
int rlen = 0;
while (rlen < CL_MSG_HEADER_LEN) {
int read_rv = is.read(msg_header_buf, rlen, CL_MSG_HEADER_LEN-rlen);
if (read_rv < 0) {
CitrusleafClient.info("scan: header connection from server lost." );
throw new java.net.SocketException("socket closed on scan header read");
}
else if (read_rv == 0) {
CitrusleafClient.info("scan: Got zero bytes." );
}
rlen += read_rv;
}
long sz = get_ntohll(msg_header_buf, 0);
byte cl_version = (byte) (((int)(sz >> 56)) & 0xff);
byte cl_type = (byte) (((int)(sz >> 48)) & 0xff);
int n_bytes = (int)(sz & 0xFFFFFFFFFFFFL);
if (cl_version != CL_MSG_VERSION) {
CitrusleafClient.info("read header: incorrect version, aborting");
}
if (cl_type != AS_MSG_TYPE) {
CitrusleafClient.info("read header: incorrect message type, aborting receive");
}
if (n_bytes != 0) {
if (msg_read_buf == null) {
msg_read_buf = new byte[n_bytes < (2 * 1024) ? (2 * 1024) : n_bytes];
}
if (msg_read_buf.length < n_bytes) {
msg_read_buf = new byte[n_bytes];
}
// readit
rlen = 0;
while (rlen < n_bytes) {
int read_rv = is.read(msg_read_buf, rlen, n_bytes - rlen);
if (read_rv < 0) {
CitrusleafClient.info("scan: read connection from server lost." );
throw new java.net.SocketException("socket closed on scan content read");
}
else if (read_rv == 0) {
CitrusleafClient.info("scan: Got zero bytes." );
}
rlen += read_rv;
}
}
//CitrusleafClient.debug("scan: processing "+n_bytes +" bytes from CL");
r = parseMultiResult(msg_read_buf, n_bytes);
if (r.resultCode != ClResultCode.OK) {
// Special case - if we scan a set name that doesn't exist on a
// node, it will return "not found" - we unify this with the
// case where OK is returned and no callbacks were made. [AKG]
if (r.resultCode == ClResultCode.KEY_NOT_FOUND_ERROR) {
r.resultCode = ClResultCode.OK;
}
return (r);
}
} while (!gotlastmsg);
return (r);
}
public ClResult get_and_parse(InputStream is) throws IOException, SerializeException
{
int rlen = 0;
while (rlen < CL_MSG_HEADER_LEN + AS_MSG_HEADER_LEN) {
int read_rv = is.read(msg_header_buf);
if (read_rv < 0) {
throw new java.net.SocketException("socket closed");
}
rlen += read_rv;
}
// Debug
// dump_buf("read header", msg_header_buf, 0);
// A number of these are commented out because we just don't care enough to read
// that section of the header. If we do care, uncomment and check!
long sz = get_ntohll(msg_header_buf, 0);
byte cl_version = (byte) (((int)(sz >> 56)) & 0xff);
byte cl_type = (byte) (((int)(sz >> 48)) & 0xff);
byte h_len = msg_header_buf[8];
// byte info1 = msg_header_buf[9];
// byte info2 = msg_header_buf[10];
// byte info3 = msg_header_buf[11];
// byte unused = msg_header_buf[12];
int resultCode = msg_header_buf[13];
int generation = get_ntohl(msg_header_buf,14);
// int record_ttl = get_ntohl(msg_header_buf,18);
// int transaction_ttl = get_ntohl(msg_header_buf, 22);
int n_fields = get_ntohs(msg_header_buf,26); // almost certainly 0
int n_ops = get_ntohs(msg_header_buf,28);
int n_bytes = ((int) (sz & 0xFFFFFFFFFFFFL)) - h_len;
if (cl_version != CL_MSG_VERSION) {
CitrusleafClient.info("read header: incorrect version, aborting");
}
if (cl_type != AS_MSG_TYPE) {
CitrusleafClient.info("read header: incorrect message type, aborting receive");
}
if (h_len != AS_MSG_HEADER_LEN) {
CitrusleafClient.info("read header: unexpected header size, aborting");
}
// System.out.printf("read header: version %d hlen %d result %d nops %d nbytes %d\n",
// cl_version, h_len, resultCode, n_ops, n_bytes);
// read! more - allocate a byte buffer for the whole steenking thing
if (n_bytes != 0) {
if (msg_read_buf == null) {
msg_read_buf = new byte[n_bytes < (2 * 1024) ? (2 * 1024) : n_bytes];
}
if (msg_read_buf.length < n_bytes) {
msg_read_buf = new byte[n_bytes];
}
// readit
rlen = 0;
while (rlen < n_bytes) {
int read_rv = is.read(msg_read_buf, rlen, n_bytes - rlen);
if (read_rv < 0) throw new java.net.SocketException("socket closed");
rlen += read_rv;
}
// dump_buf("read msg body", msg_read_buf, n_bytes);
}
// parseit
ClResult r = parseResult(msg_read_buf, n_bytes, n_ops, n_fields, resultCode, generation);
return(r);
}
final static int PARTICLE_TYPE_NULL = 0;
final static int PARTICLE_TYPE_INTEGER = 1;
final static int PARTICLE_TYPE_BIGNUM = 2;
final static int PARTICLE_TYPE_STRING = 3;
final static int PARTICLE_TYPE_BLOB = 4;
final static int PARTICLE_TYPE_TIMESTAMP = 5;
final static int PARTICLE_TYPE_DIGEST = 6;
final static int PARTICLE_TYPE_JBLOB = 7;
final static int PARTICLE_TYPE_CSHARP_BLOB = 8;
final static int PARTICLE_TYPE_PYTHON_BLOB = 9;
final static int PARTICLE_TYPE_RUBY_BLOB = 10;
final static int PARTICLE_TYPE_PHP_BLOB = 11;
final static int PARTICLE_TYPE_ERLANG_BLOB = 12;
final static int PARTICLE_TYPE_SEGMENT_POINTER = 13;
final static int PARTICLE_TYPE_LIST = 14;
final static int PARTICLE_TYPE_DICT = 15;
private static class ParticleBytesReturn {
int type;
int size; // bytes copied
};
// Copy object into buffer, return the type and the new offset
// IOException is thrown when the class can't be serialized
// Parameter: 'java_serialized_bytes' is passed in when we've already figured out the object is requires java
// serialization, and
@SuppressWarnings("unchecked")
private ParticleBytesReturn objectToParticleBytes(Object o, byte[] buf, int offset, byte[] java_serialized_bytes) throws SerializeException
{
ParticleBytesReturn r = new ParticleBytesReturn();
if (o == null) {
r.type = PARTICLE_TYPE_NULL;
r.size = 0;
return(r);
}
// shortcut: if java_serialized bytes are in, just use them
if (java_serialized_bytes != null) {
r.type = PARTICLE_TYPE_JBLOB;
if( buf != null )
System.arraycopy(java_serialized_bytes, 0, buf, offset, java_serialized_bytes.length);
r.size = java_serialized_bytes.length;
return(r);
}
// people who use byte arrays care more about speed...
if (o instanceof byte[]) {
// I'd like to support byte[] as blob, not quite sure how to do it yet
r.type = PARTICLE_TYPE_BLOB;
byte[] b = (byte[]) o;
if( buf != null )
System.arraycopy(b, 0, buf, offset, b.length);
r.size = b.length;
}
else if (o instanceof String) {
r.type = PARTICLE_TYPE_STRING;
String s = (String) o;
byte[] value;
try {
value = s.getBytes("UTF8");
} catch (UnsupportedEncodingException e) {
// If you can't encode a UTF8 string you're in sad, sad shape
throw new SerializeException();
}
if (buf != null)
System.arraycopy(value, 0, buf, offset, value.length);
r.size = value.length;
}
else if ((o instanceof Integer) || (o instanceof Long)) {
/*
* If it is of integer type(4bytes) or long type(8 bytes),
* when sending to the server always send in singned 64bit
* format (2's complement method for -ve numbers). This will
* make it easy to store and retrive. Single byte +ve values
* are an exception to this rule which use single byte.
*/
r.type = PARTICLE_TYPE_INTEGER;
// Both Interger & Long class has longValue() method.
// Typecast the object to Number which is a superclass.
// Polymorphism will take care of the rest.
long i = ((Number)o).longValue();
// Note: commenting out the if clause fixes the digest difference
// between C & Java for the 0 <= long <= 127 range. [PSI]
// if ((i >= 0) && (i < 128)) {
// if (buf != null)
// buf[offset] = (byte) (i & 0xff);
// r.size = 1;
// }
// else
{
if (buf != null)
set_htonll(i, buf, offset);
r.size = 8;
}
}
// everything else gets serialized with the java serializer
// although we expect to not his this path often
else {
r.type = PARTICLE_TYPE_JBLOB;
if (buf != null)
r.size = objectToByteArray(o, buf, offset); // use serialization to make a blob
else
r.size = objectToByteArray(o).length;
}
return(r);
}
private static Object parseList(byte[] buf, int offset, int len) throws SerializeException {
int limit = offset + len;
int n_items = (int) get_ntohl(buf, offset);
offset += 4;
ArrayList<Object> r = new ArrayList<Object>(n_items);
while (offset < limit)
{
int sz = (int) get_ntohl(buf, offset);
offset += 4;
int type = buf[offset];
offset++;
try {
r.add( bytesToParticle(type, buf, offset, sz));
offset += sz;
} catch (SerializeException e) {
CitrusleafClient.info(" de-serialize exception in parseList");
throw e;
}
}
return(r);
}
private static Object parseDict(byte[] buf, int offset, int len) throws SerializeException {
Object key;
Object value;
int limit = offset + len;
int n_items = (int) get_ntohl(buf, offset);
offset += 4;
HashMap<Object, Object> d = new HashMap<Object, Object>(n_items);
while (offset < limit)
{
// read out the key
int sz = (int) get_ntohl(buf, offset);
offset += 4;
int type = buf[offset];
offset++;
try {
key = bytesToParticle(type, buf, offset, len);
offset += sz;
} catch (SerializeException e) {
CitrusleafClient.info(" de-serialize exception in parseDict");
throw e;
}
// read out the value
sz = (int) get_ntohl(buf, offset);
offset += 4;
type = buf[offset];
offset++;
try {
value = bytesToParticle(type, buf, offset, len);
offset += sz;
} catch (SerializeException e) {
CitrusleafClient.info(" de-serialize exception in parseDict: key "+key.toString() );
throw e;
}
d.put(key, value);
}
return (d);
}
private static Object bytesToParticle(int type, byte[] buf, int offset, int len) throws SerializeException
{
Object r;
if (type == PARTICLE_TYPE_STRING) {
r = utf8ToString(buf, offset, len);
}
else if (type == PARTICLE_TYPE_INTEGER) {
r = bytesToInteger(buf, offset, len);
}
else if (type == PARTICLE_TYPE_BLOB) {
r = Arrays.copyOfRange(buf, offset, offset+len);
}
else if (type == PARTICLE_TYPE_JBLOB) {
r = byteArrayToObject(buf, offset, len);
}
else if (type == PARTICLE_TYPE_LIST) {
r = parseList(buf, offset, len);
}
else if (type == PARTICLE_TYPE_DICT) {
r = parseDict(buf, offset, len);
}
else {
//CitrusleafClient.info("bytesToParticle: Unsupported type "+type);
r = null;
}
return(r);
}
// Need a nice private function to serialize objects into byte buffers
// This is the hideously general purpose way; very slow; probably should use
// some kind of special purpose for strings. There's lots of stuff out there for
// serializing types
public static Object byteBufferToObject(ByteBuffer b) throws Exception
{
ObjectInputStream oistream;
ByteArrayInputStream bastream;
// if we can get the backing store, use it directly
if (b.hasArray()) {
byte[] barray = b.array();
int boffset = b.arrayOffset();
bastream = new ByteArrayInputStream(barray,boffset + b.position() , boffset+b.remaining() );
}
// otherwise copy
else {
int len = b.remaining();
byte[] barray = new byte[len];
b.get(barray);
bastream = new ByteArrayInputStream(barray);
}
oistream = new ObjectInputStream(bastream);
Object o = oistream.readObject();
// CitrusleafClient.info("byteBufferToObject: converted class type "+o.getClass().getName() );
return(o);
}
private static byte[] objectToByteArray(Object o) throws SerializeException
{
ObjectOutputStream ostream;
ByteArrayOutputStream bstream;
byte[] barray;
try {
bstream = new ByteArrayOutputStream();
ostream = new ObjectOutputStream(bstream );
ostream.writeObject(o);
ostream.close();
barray = bstream.toByteArray();
// System.arraycopy(barray, 0, buf, offset, barray.length);
} catch (Exception e) {
CitrusleafClient.info("objectToByteArray: exception serializing object "+o.getClass().getName()+" "+ e.getMessage() );
throw new SerializeException();
}
return(barray);
}
private static int objectToByteArray(Object o, byte[] buf, int offset) throws SerializeException
{
ObjectOutputStream ostream;
ByteArrayOutputStream bstream;
byte[] barray;
try {
bstream = new ByteArrayOutputStream();
ostream = new ObjectOutputStream(bstream );
ostream.writeObject(o);
ostream.close();
barray = bstream.toByteArray();
System.arraycopy(barray, 0, buf, offset, barray.length);
} catch (Exception e) {
CitrusleafClient.info("objectToByteArray: exception serializing object "+o.getClass().getName()+" "+ e.getMessage() );
throw new SerializeException();
}
return(barray.length);
}
public static Object byteArrayToObject(byte[] b, int offset, int length) throws SerializeException
{
try {
ByteArrayInputStream bastream = new ByteArrayInputStream(b, offset, length);
ObjectInputStream oistream = new ObjectInputStream(bastream);
Object o = oistream.readObject();
// CitrusleafClient.info("byteBufferToObject: converted class type "+o.getClass().getName() );
return(o);
} catch (Exception e) {
// this could be a simple application problem, or something far more pernicious
CitrusleafClient.info("byteBufferToObject: exception deserializing object "+e.getMessage()+" len "+length);
// e.printStackTrace(System.err);
dump_buf("serializer error ", b, offset, length);
if (e.getCause() != null)
CitrusleafClient.info("byteBufferToObject: exception deserializing object - cause "+e.getCause().toString() );
throw new SerializeException();
}
}
// Converts input string to UTF-8, copies into buffer (at given offset).
// Returns number of bytes in the string.
//
// Java's internal UTF8 conversion is very, very slow.
// This is, rather amazingly, 8x faster than the to-string method
// returns the number of bytes this transalated into
public static int stringToUtf8(String s, byte[] buf, int offset)
{
int l = s.length();
int start_offset = offset;
for (int i=0;i<l;i++) {
char c = s.charAt(i);
if (c < 0x7f) {
buf[offset] = (byte) c;
offset++;
}
else if (c < 0x07FF) {
buf[offset] = (byte) (0xC0 | (c >> 6));
buf[offset+1] = (byte) (0x80 | (c & 0x5f));
offset += 2;
}
else {
//Encountered a different encoding other than 2-byte UTF8. Let java handle it.
try {
byte[] value = s.getBytes("UTF8");
System.arraycopy(value, 0, buf, start_offset, value.length);
return value.length;
}
catch (UnsupportedEncodingException e) {
//This should never happen as we are using standard encoding
CitrusleafClient.info("Encountered illegally encoded string");
return 0;
}
}
}
return(offset - start_offset);
}
public static String utf8ToString(byte[] buf, int offset, int length) throws SerializeException
{
StringBuilder sb = new StringBuilder(length);
int limit = offset+length;
int origoffset = offset;
while (offset < limit ) {
if ((buf[offset] & 0x80) == 0) { // 1 byte
char c = (char) buf[offset];
sb.append(c);
offset++;
}
else if ((buf[offset] & 0xE0) == 0xC0) { // 2 bytes
char c = (char) (((buf[offset] & 0x1f) << 6) | (buf[offset+1] & 0x3f));
sb.append(c);
offset += 2;
}
else {
//Encountered an UTF encoding which uses more than 2bytes.
//Use a native function to do the conversion.
try {
String s = new String(buf, origoffset, length, "UTF8");
return s;
}
catch(Exception e) {
CitrusleafClient.info("Encountered illegally encoded string");
throw new SerializeException();
}
}
}
return( sb.toString() );
}
public static Object bytesToBigInteger(byte[] buf, int offset, int len) {
boolean is_neg;
if ((buf[offset] & 0x80) != 0) {
is_neg = true;
buf[offset] &= 0x7f;
} else {
is_neg = false;
}
byte[] ba = new byte[len];
System.arraycopy(buf, offset, ba, 0, len);
BigInteger a = new BigInteger(ba);
if (is_neg) a = a.negate();
return(a);
}
public static Object bytesToInteger(byte[] buf, int offset, int len) {
if (len == 0) return(new Integer(0));
if (len > 8) return(bytesToBigInteger(buf, offset, len));
// This will work for negative integers too which
// will be represented in 2's complemenet representation
long a=0;
for (int i=0; i<len; i++) {
a <<= 8;
a |= buf[offset+i] & 0xFF;
}
if (a <= Integer.MAX_VALUE && a >= Integer.MIN_VALUE)
return(new Integer((int) a));
return(new Long(a));
}
public static void dump_buf(String info, byte[] buf, int limit) {
CitrusleafClient.info("dump buffer: "+info+" size "+buf.length+" limit "+limit);
if (limit == 0) limit = buf.length;
for (int i=0;i<limit;i++) {
System.err.format(" %02x", buf[i]);
if (i % 16 == 7)
System.err.print(":");
if (i % 16 == 15)
System.err.println();
}
System.err.println();
}
// Todo: pass in the StreamWriter instead of using System.err
public static void dump_buf(String info, byte[] buf, int offset, int len) {
CitrusleafClient.info("dump buffer: "+info+" bufsize "+buf.length+" offset "+offset+" len "+len);
if (len == 0) len = buf.length - offset; // do to end
for (int i=offset,pos=0;i<offset+len;i++,pos++) {
System.err.format(" %02x", buf[i]);
if (pos % 16 == 7)
System.err.print(":");
if (pos % 16 == 15)
System.err.println();
}
System.err.println();
}
}