package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.mq.message.PriorityMQMessage;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.mq.util.PriorityCompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import com.google.common.collect.EvictingQueue;
import com.google.common.collect.MinMaxPriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Priority queue that buffers nothing in memory, relying only on ordering of persisitent
* storage.
*
* @author garth
*/
public class DirectPriorityQueue<E> extends DefaultPriorityQueue<E> {
private static final Logger logger = LoggerFactory.getLogger(DirectPriorityQueue.class);
private static final boolean DEBUG = false;
public DirectPriorityQueue() {
super();
this.trxLog = EvictingQueue.create(100);
}
private long initialSize = 0L;
private final EvictingQueue<String> trxLog;
@Override
protected void doActivate() throws Exception {
this.size.set(initialSize);
this.putCount.set(initialSize);
}
@Override
public void preload(DataStoreIterator iterator) throws DataStoreFatalException, QueueFatalException {
int itemsLoaded = 0;
while (iterator.next()) {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = priorityKeyUtil.decode(record.getKey());
PriorityMQMessage element = null;
try {
element = priorityTranscoder.decode(record.getValue());
} catch (Throwable t) {
throw new QueueFatalException("Unable to decode element with transcoder for queueId " + getId() + ". Perhaps incorrect transcoder?", t);
}
initialSize++;
}
}
@Override
protected boolean doStore(PriorityMQMessage<E> item, byte[] encoded) throws QueueInvalidStateException, QueueFatalException, QueueIsFullException, QueueTimeoutException, DataStoreFatalException {
long itemId = item.key();
byte[] key = priorityKeyUtil.encode(getId(), itemId);
try {
if (DEBUG) {
PriorityMQMessage.Key pkey = new PriorityMQMessage.Key(itemId);
logger.trace("doStore[{}]: {}", getId(), pkey);
trxLog.offer(pkey.toCompactString("STORE:"));
}
ds.setRecord(key, encoded);
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently store item [" + getId() + "]: " + new PriorityMQMessage.Key(itemId), e);
this.errorCount.incrementAndGet();
}
return true;
}
@Override
protected PriorityMQMessage<E> doTake() throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException {
DataStoreIterator iterator = ds.getAscendingIterator();
boolean jumped = iterator.jump(priorityKeyUtil.encode(getId(), 0L));
if (!jumped) logger.warn("DataStoreIterator failed to jump to queue's first record {} {}", getId(), 0L);
// if (!jumped) something is wrong, we should zero the size and exit
try {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = priorityKeyUtil.decode(record.getKey());
PriorityMQMessage.Key pkey = new PriorityMQMessage.Key(key.getItemId());
if (key.getQueueId() != getId()) {
logger.error("{} != {}: {} for queue {}", key.getQueueId(), getId(), pkey, this);
throw new DataStoreFatalException("The next item wasn't for this queueId");
}
if (DEBUG) {
logger.trace("doTake[{}]: {}", getId(), pkey);
trxLog.offer(pkey.toCompactString("TAKE:"));
}
return priorityTranscoder.decode(record.getValue());
} catch (DataStoreFatalException e) {
logger.error("Unable to get record from datastore for {}", this);
dumpTrxLog();
if (size.get() != 0) {
logger.warn("This is dangerous, but we're going to zero the queue size.");
size.set(0);
}
this.errorCount.incrementAndGet();
throw e;
} finally {
iterator.close();
}
}
@Override
protected void afterTake(PriorityMQMessage<E> item) {
long itemId = item.key();
try {
byte[] key = priorityKeyUtil.encode(getId(), itemId);
if (DEBUG) {
PriorityMQMessage.Key pkey = new PriorityMQMessage.Key(itemId);
logger.trace("afterTake[{}]: {}", getId(), pkey);
trxLog.offer(pkey.toCompactString("REMOVE:"));
}
ds.deleteRecord(key);
} catch (RecordNotFoundException e) {
logger.error("Key not found in DataStore [" + getId() + "]: " + new PriorityMQMessage.Key(itemId), e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently delete key [" + getId() + "]" + new PriorityMQMessage.Key(itemId), e);
this.errorCount.incrementAndGet();
}
}
@Override
public PriorityMQMessage<E> take(long timeout) throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException, InterruptedException {
return super.take(timeout);
}
// I'm assuming you have the lock
private void dumpTrxLog() {
if (DEBUG) {
int id = getId();
for (int i = 0;i < trxLog.size();i++) {
String s = trxLog.poll();
logger.warn("TRXDUMP [{}] {}", id, s);
}
}
}
}