package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import com.cloudhopper.mq.transcoder.Transcoder;
import com.codahale.metrics.Meter;
import java.util.LinkedList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author garth
*/
public abstract class AbstractQueue<E> extends BaseQueue<E> implements QueueMXBean, InitializingQueue<E> {
private static final Logger logger = LoggerFactory.getLogger(AbstractQueue.class);
// in-memory priority queue
protected final ReentrantLock lock;
protected final Condition notEmpty;
// queue counters
protected final AtomicLong size;
protected final AtomicLong errorCount;
protected final AtomicLong putCount;
protected final AtomicLong takeCount;
// for creating composite keys in the data store
protected CompositeKeyUtil keyUtil;
// data store for long-term
protected DataStore ds;
protected Class<E> elementType;
protected Transcoder<E> transcoder;
// for calculating median age
protected AverageQueueSize averageQueueSize;
protected Meter putRateMeter;
protected Meter takeRateMeter;
/**
*/
protected AbstractQueue() {
// for locking, in-memory store
this.lock = new ReentrantLock(true);
this.notEmpty = lock.newCondition();
// initialize the counters
this.size = new AtomicLong();
this.errorCount = new AtomicLong();
this.putCount = new AtomicLong();
this.takeCount = new AtomicLong();
this.averageQueueSize = AverageQueueSize.get();
this.averageQueueSize.add(this);
this.putRateMeter = new Meter();
this.takeRateMeter = new Meter();
}
private boolean activated = false;
@Override
public void setQueueManager(QueueManager queueManager) {
if (activated) throw new IllegalStateException("QueueManager cannot be set after activation");
super.setQueueManager(queueManager);
}
@Override
public void setQueueId(int queueId) {
if (activated) throw new IllegalStateException("QueueId cannot be set after activation");
super.setQueueId(queueId);
}
@Override
public void setQueueName(String queueName) {
if (activated) throw new IllegalStateException("QueueName cannot be set after activation");
super.setQueueName(queueName);
}
@Override
public void setLocalOnly(boolean isLocalOnly) {
if (activated) throw new IllegalStateException("LocalOnly cannot be set after activation");
super.setLocalOnly(isLocalOnly);
}
@Override
public void setDataStore(DataStore ds) {
if (activated) throw new IllegalStateException("DataStore cannot be set after activation");
this.ds = ds;
}
@Override
public void setCompositeKeyUtil(CompositeKeyUtil keyUtil) {
if (activated) throw new IllegalStateException("CompositeKeyUtil cannot be set after activation");
this.keyUtil = keyUtil;
}
@Override
public void setTranscoder(Transcoder transcoder) {
if (activated) throw new IllegalStateException("Transcoder cannot be set after activation");
this.transcoder = (Transcoder<E>)transcoder;
}
@Override
public void setElementType(Class<E> elementType) {
if (activated) throw new IllegalStateException("ElementType cannot be set after activation");
this.elementType = elementType;
}
@Override
public void setProperty(String name, Object value) {
//do nothing. subclass must override
}
/**
* Subclasses that need to "activate" after properties set must implement this.
*/
protected abstract void doActivate() throws Exception;
@Override
public void activate() throws Exception {
if (activated) throw new IllegalStateException("This queue is already activated");
doActivate();
activated = true;
}
@Override
public long getErrorCount() {
return this.errorCount.get();
}
@Override
public long getPutCount() {
return this.putCount.get();
}
@Override
public long getTakeCount() {
return this.takeCount.get();
}
@Override
public long getSize() {
return this.size.get();
}
public Transcoder<E> getTranscoder() {
return this.transcoder;
}
public Class<?> getTranscoderType() {
return this.transcoder.getClass();
}
public Class<?> getElementType() {
return this.elementType;
}
public String getTranscoderTypeName() {
return this.transcoder.getClass().getCanonicalName();
}
public String getElementTypeName() {
return this.elementType.getCanonicalName();
}
public void resetCounters() {
// grab an exclusive lock
lock.lock();
try {
this.putCount.set(0);
this.takeCount.set(0);
this.errorCount.set(0);
} finally {
lock.unlock();
}
}
public long purge() throws QueueInvalidStateException, QueueFatalException, DataStoreFatalException, InterruptedException {
checkIfShutdown();
// grab an exclusive lock on take/puts
lock.lockInterruptibly();
// number of items we need to purge
long purgeSize = this.getSize();
long purgeCount = 0;
try {
for (long i = 0; i < purgeSize; i++) {
Object item = this.take(0);
if (item == null) {
return purgeCount;
}
purgeCount++;
}
} catch (QueueTimeoutException e) {
logger.error("Impossible timeout exception while purging", e);
} finally {
lock.unlock();
}
return purgeCount;
}
@Override
public double getPutRate() {
return putRateMeter.getFiveMinuteRate();
}
@Override
public double getTakeRate() {
return takeRateMeter.getFiveMinuteRate();
}
@Override
public double getAverageAge() {
double avgSize = averageQueueSize.getMean(this);
double putRate = putRateMeter.getFiveMinuteRate();
// double age = avgSize / putRate;
// logger.debug("avgSize:{} / putRate:{} = age:{}", avgSize, putRate, age);
// return age;
if (putRate == 0) return 0; // check NaN
else return avgSize / putRate;
}
@Override
public double getAverageSize() {
return averageQueueSize.getMean(this);
}
@Override
public E getMedianEntry(long timeout) throws QueueTimeoutException, QueueInvalidStateException, InterruptedException {
throw new UnsupportedOperationException("getMedianEntry not supported for this queue type");
}
@Override
public String getMedianEntryString(long timeout) throws QueueInvalidStateException, InterruptedException {
try {
E item = getMedianEntry(timeout);
if (item == null) return "n/a";
return item.toString();
} catch (QueueTimeoutException qte) {
return "timeout";
} catch (RuntimeException re) {
return re.getMessage();
}
}
/**
* Allows implementations to provide empty/nonempty information without necessarily invoking size.get()
*/
protected abstract boolean hasElements();
protected abstract byte[] encodeItem(E item); //maybe encode the item before locking
protected abstract boolean doStore(E item, byte[] encoded) throws QueueInvalidStateException, QueueFatalException, QueueIsFullException, QueueTimeoutException, DataStoreFatalException; //store this one
protected abstract void afterStore(E item, byte[] encoded); //after it's stored
protected abstract E doTake() throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException; //take the next one
protected abstract void afterTake(E item); //after the take is complete
protected abstract E decodeItem(byte[] encoded); //maybe decode the [] after unlocking
public boolean put(E item, long timeout) throws QueueInvalidStateException, QueueFatalException, QueueIsFullException, QueueTimeoutException, DataStoreFatalException, InterruptedException {
checkIfShutdown();
// check that the item isn't null
if (item == null) {
throw new NullPointerException();
}
// before locking, we can transcode the item into a byte[]
byte[] encoded = encodeItem(item);
if (timeout == 0) {
if (!lock.tryLock()) {
return false;
}
} else if (timeout > 0) {
if (!lock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new QueueTimeoutException("Timeout while waiting for lock [method=add(), queue=" + getName() + "]");
}
} else {
// wait forever until we get the lock
lock.lockInterruptibly();
}
long ns = 0l;
try {
boolean stored = doStore(item, encoded);
ns = size.incrementAndGet();
putCount.incrementAndGet();
afterStore(item, encoded);
// mark the put rate meter
putRateMeter.mark();
// signal any waiting threads that an item is ready
notEmpty.signal();
} finally {
lock.unlock();
}
// notify the QueueListener
if (ns == 1) notifyListenerNotEmpty();
return true;
}
public E take(long timeout) throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException, InterruptedException {
checkIfShutdown();
if (timeout == 0) {
if (!lock.tryLock()) {
return null;
}
} else if (timeout > 0) {
if (!lock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new QueueTimeoutException("Timeout while waiting for lock [method=take(), queue=" + getName() + "]");
}
} else {
// wait forever until we get the lock
lock.lockInterruptibly();
}
try {
try {
// continue waiting until an object is in the queue
//while (queue.size() == 0) {
while (!hasElements()) {
// if timeout is zero, then we're not supposed to wait
if (timeout == 0) {
return null;
} else if (timeout < 0) {
// wait indefinitely
notEmpty.await();
} else {
// FIXME: what about a "spurious wakeup" where the full timeout??
// await for a period of time (returns true if no timeout, false
// if there was a timeout
boolean waitTimeout = !notEmpty.await(timeout, TimeUnit.MILLISECONDS);
if (waitTimeout) {
throw new QueueTimeoutException("Timeout while waiting for item [method=take(), queue=" + getName() + "]");
}
}
}
} catch (InterruptedException ie) {
notEmpty.signal(); // propagate to non-interrupted thread
throw ie;
}
E item = doTake();
// TODO: should this happen after afterTake(item)
size.decrementAndGet();
takeCount.incrementAndGet();
afterTake(item);
// mark the take rate meter
takeRateMeter.mark();
return item;
} finally {
lock.unlock();
}
}
@Override
public String toString() {
return new StringBuilder(100)
.append("[name=")
.append(getName())
.append(", id=")
.append(getId())
.append(", size=")
.append(getSize())
.append(", putCount=")
.append(getPutCount())
.append(", takeCount=")
.append(getTakeCount())
.append("]")
.toString();
}
}