/**
*
* Copyright 2004 Hiram Chirino
* Copyright 2004 Protique Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
**/
package org.activemq.store.journal;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import javax.jms.JMSException;
import javax.transaction.xa.XAException;
import org.activeio.adapter.PacketByteArrayOutputStream;
import org.activeio.adapter.PacketInputStream;
import org.activeio.journal.InvalidRecordLocationException;
import org.activeio.journal.Journal;
import org.activeio.journal.JournalEventListener;
import org.activeio.journal.RecordLocation;
import org.activeio.journal.active.JournalImpl;
import org.activeio.journal.howl.HowlJournal;
import org.activemq.io.WireFormat;
import org.activemq.io.impl.StatelessDefaultWireFormat;
import org.activemq.message.ActiveMQMessage;
import org.activemq.message.ActiveMQXid;
import org.activemq.message.MessageAck;
import org.activemq.message.Packet;
import org.activemq.service.MessageIdentity;
import org.activemq.store.MessageStore;
import org.activemq.store.PersistenceAdapter;
import org.activemq.store.TopicMessageStore;
import org.activemq.store.TransactionStore;
import org.activemq.store.jdbc.JDBCPersistenceAdapter;
import org.activemq.store.journal.JournalTransactionStore.Tx;
import org.activemq.store.journal.JournalTransactionStore.TxOperation;
import org.activemq.util.JMSExceptionHelper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.objectweb.howl.log.Configuration;
import EDU.oswego.cs.dl.util.concurrent.Channel;
import EDU.oswego.cs.dl.util.concurrent.ClockDaemon;
import EDU.oswego.cs.dl.util.concurrent.ConcurrentHashMap;
import EDU.oswego.cs.dl.util.concurrent.Latch;
import EDU.oswego.cs.dl.util.concurrent.LinkedQueue;
import EDU.oswego.cs.dl.util.concurrent.QueuedExecutor;
import EDU.oswego.cs.dl.util.concurrent.ThreadFactory;
/**
* An implementation of {@link PersistenceAdapter} designed for
* use with a {@link Journal} and then checkpointing asynchronously
* on a timeout with some other long term persistent storage.
*
* @version $Revision: 1.1 $
*/
public class JournalPersistenceAdapter implements PersistenceAdapter, JournalEventListener {
private static final Log log = LogFactory.getLog(JournalPersistenceAdapter.class);
public static final String DEFAULT_JOURNAL_TYPE = "default";
public static final String HOWL_JOURNAL_TYPE = "howl";
private Journal journal;
private String journalType = DEFAULT_JOURNAL_TYPE;
private PersistenceAdapter longTermPersistence;
private File directory = new File("logs");
private final StatelessDefaultWireFormat wireFormat = new StatelessDefaultWireFormat();
private final ConcurrentHashMap messageStores = new ConcurrentHashMap();
private final ConcurrentHashMap topicMessageStores = new ConcurrentHashMap();
private static final int PACKET_RECORD_TYPE = 0;
private static final int COMMAND_RECORD_TYPE = 1;
private static final int TX_COMMAND_RECORD_TYPE = 2;
private static final int ACK_RECORD_TYPE = 3;
private Channel checkpointRequests = new LinkedQueue();
private QueuedExecutor checkpointExecutor;
ClockDaemon clockDaemon;
private Object clockTicket;
private JournalTransactionStore transactionStore = new JournalTransactionStore(this);
private int logFileSize=1024*1024*20;
private int logFileCount=2;
private long checkpointInterval = 1000 * 60 * 5;
public JournalPersistenceAdapter() {
checkpointExecutor = new QueuedExecutor(new LinkedQueue());
checkpointExecutor.setThreadFactory(new ThreadFactory() {
public Thread newThread(Runnable runnable) {
Thread answer = new Thread(runnable, "Checkpoint Worker");
answer.setDaemon(true);
answer.setPriority(Thread.MAX_PRIORITY);
return answer;
}
});
}
public JournalPersistenceAdapter(File directory, PersistenceAdapter longTermPersistence) throws IOException {
this();
this.directory = directory;
this.longTermPersistence = longTermPersistence;
}
public Map getInitialDestinations() {
return longTermPersistence.getInitialDestinations();
}
private MessageStore createMessageStore(String destination, boolean isQueue) throws JMSException {
if(isQueue) {
return createQueueMessageStore(destination);
} else {
return createTopicMessageStore(destination);
}
}
public MessageStore createQueueMessageStore(String destinationName) throws JMSException {
JournalMessageStore store = (JournalMessageStore) messageStores.get(destinationName);
if( store == null ) {
MessageStore checkpointStore = longTermPersistence.createQueueMessageStore(destinationName);
store = new JournalMessageStore(this, checkpointStore, destinationName);
messageStores.put(destinationName, store);
}
return store;
}
public TopicMessageStore createTopicMessageStore(String destinationName) throws JMSException {
JournalTopicMessageStore store = (JournalTopicMessageStore) topicMessageStores.get(destinationName);
if( store == null ) {
TopicMessageStore checkpointStore = longTermPersistence.createTopicMessageStore(destinationName);
store = new JournalTopicMessageStore(this, checkpointStore, destinationName);
topicMessageStores.put(destinationName, store);
}
return store;
}
public TransactionStore createTransactionStore() throws JMSException {
return transactionStore;
}
public void beginTransaction() throws JMSException {
longTermPersistence.beginTransaction();
}
public void commitTransaction() throws JMSException {
longTermPersistence.commitTransaction();
}
public void rollbackTransaction() {
longTermPersistence.rollbackTransaction();
}
public synchronized void start() throws JMSException {
if( longTermPersistence instanceof JDBCPersistenceAdapter ) {
// Disabled periodic clean up as it deadlocks with the checkpoint operations.
((JDBCPersistenceAdapter)longTermPersistence).setCleanupPeriod(0);
}
longTermPersistence.start();
createTransactionStore();
if (journal == null) {
try {
log.info("Opening journal.");
journal = createJournal();
log.info("Opened journal: " + journal);
journal.setJournalEventListener(this);
}
catch (Exception e) {
throw JMSExceptionHelper.newJMSException("Failed to open transaction journal: " + e, e);
}
try {
recover();
}
catch (Exception e) {
throw JMSExceptionHelper.newJMSException("Failed to recover transactions from journal: " + e, e);
}
}
// Do a checkpoint periodically.
clockTicket = getClockDaemon().executePeriodically(checkpointInterval, new Runnable() {
public void run() {
checkpoint(false);
}
}, false);
}
public synchronized void stop() throws JMSException {
if (clockTicket != null) {
// Stop the periodical checkpoint.
ClockDaemon.cancel(clockTicket);
clockTicket=null;
clockDaemon.shutDown();
}
// Take one final checkpoint and stop checkpoint processing.
checkpoint(true);
checkpointExecutor.shutdownAfterProcessingCurrentlyQueuedTasks();
JMSException firstException = null;
if (journal != null) {
try {
journal.close();
journal = null;
}
catch (Exception e) {
firstException = JMSExceptionHelper.newJMSException("Failed to close journals: " + e, e);
}
}
longTermPersistence.stop();
if (firstException != null) {
throw firstException;
}
}
// Properties
//-------------------------------------------------------------------------
public PersistenceAdapter getLongTermPersistence() {
return longTermPersistence;
}
public void setLongTermPersistence(PersistenceAdapter longTermPersistence) {
this.longTermPersistence = longTermPersistence;
}
/**
* @return Returns the directory.
*/
public File getDirectory() {
return directory;
}
/**
* @param directory The directory to set.
*/
public void setDirectory(File directory) {
this.directory = directory;
}
/**
* @return Returns the wireFormat.
*/
public WireFormat getWireFormat() {
return wireFormat;
}
public String getJournalType() {
return journalType;
}
public void setJournalType(String journalType) {
this.journalType = journalType;
}
protected Journal createJournal() throws IOException {
if( DEFAULT_JOURNAL_TYPE.equals(journalType) ) {
return new JournalImpl(directory,logFileCount,logFileSize);
}
if( HOWL_JOURNAL_TYPE.equals(journalType) ) {
try {
Configuration config = new Configuration();
config.setLogFileDir(directory.getCanonicalPath());
return new HowlJournal(config);
} catch (IOException e) {
throw e;
} catch (Exception e) {
throw (IOException)new IOException("Could not open HOWL journal: "+e.getMessage()).initCause(e);
}
}
throw new IllegalStateException("Unsupported valued for journalType attribute: "+journalType);
}
// Implementation methods
//-------------------------------------------------------------------------
/**
* The Journal give us a call back so that we can move old data out of the journal.
* Taking a checkpoint does this for us.
*
* @see org.activemq.journal.JournalEventListener#overflowNotification(org.activemq.journal.RecordLocation)
*/
public void overflowNotification(RecordLocation safeLocation) {
checkpoint(false);
}
/**
* When we checkpoint we move all the journaled data to long term storage.
* @param b
*/
public void checkpoint(boolean sync) {
try {
if( journal == null )
throw new IllegalStateException("Journal is closed.");
// Do the checkpoint asynchronously?
Latch latch=null;
if( sync ) {
latch = new Latch();
checkpointRequests.put(latch);
} else {
checkpointRequests.put(Boolean.TRUE);
}
checkpointExecutor.execute(new Runnable() {
public void run() {
ArrayList listners = new ArrayList();
try {
// Avoid running a checkpoint too many times in a row.
// Consume any queued up checkpoint requests.
try {
boolean requested = false;
Object t;
while ((t=checkpointRequests.poll(0)) != null) {
if( t.getClass()==Latch.class )
listners.add(t);
requested = true;
}
if (!requested) {
return;
}
}
catch (InterruptedException e1) {
return;
}
log.debug("Checkpoint started.");
RecordLocation newMark = null;
Iterator iterator = messageStores.values().iterator();
while (iterator.hasNext()) {
try {
JournalMessageStore ms = (JournalMessageStore) iterator.next();
RecordLocation mark = ms.checkpoint();
if (mark != null && (newMark == null || newMark.compareTo(mark) < 0)) {
newMark = mark;
}
}
catch (Exception e) {
log.error("Failed to checkpoint a message store: " + e, e);
}
}
iterator = topicMessageStores.values().iterator();
while (iterator.hasNext()) {
try {
JournalTopicMessageStore ms = (JournalTopicMessageStore) iterator.next();
RecordLocation mark = ms.checkpoint();
if (mark != null && (newMark == null || newMark.compareTo(mark) < 0)) {
newMark = mark;
}
}
catch (Exception e) {
log.error("Failed to checkpoint a message store: " + e, e);
}
}
try {
if (newMark != null) {
if( log.isDebugEnabled() )
log.debug("Marking journal: "+newMark);
journal.setMark(newMark, true);
}
}
catch (Exception e) {
log.error("Failed to mark the Journal: " + e, e);
}
// Clean up the DB if it's a JDBC store.
if( longTermPersistence instanceof JDBCPersistenceAdapter ) {
// Disabled periodic clean up as it deadlocks with the checkpoint operations.
((JDBCPersistenceAdapter)longTermPersistence).cleanup();
}
log.debug("Checkpoint done.");
} finally {
for (Iterator iter = listners.iterator(); iter.hasNext();) {
Latch latch = (Latch) iter.next();
latch.release();
}
}
}
});
if( sync ) {
latch.acquire();
}
}
catch (InterruptedException e) {
log.warn("Request to start checkpoint failed: " + e, e);
}
}
/**
* @param destinationName
* @param message
* @param sync
* @throws JMSException
*/
public RecordLocation writePacket(String destination, Packet packet, boolean sync) throws JMSException {
try {
PacketByteArrayOutputStream pos = new PacketByteArrayOutputStream();
DataOutputStream os = new DataOutputStream(pos);
os.writeByte(PACKET_RECORD_TYPE);
os.writeUTF(destination);
os.close();
org.activeio.Packet p = wireFormat.writePacket(packet, pos);
return journal.write(p, sync);
}
catch (IOException e) {
throw createWriteException(packet, e);
}
}
/**
* @param destinationName
* @param message
* @param sync
* @throws JMSException
*/
public RecordLocation writeCommand(String command, boolean sync) throws JMSException {
try {
PacketByteArrayOutputStream pos = new PacketByteArrayOutputStream();
DataOutputStream os = new DataOutputStream(pos);
os.writeByte(COMMAND_RECORD_TYPE);
os.writeUTF(command);
os.close();
return journal.write(pos.getPacket(), sync);
}
catch (IOException e) {
throw createWriteException(command, e);
}
}
/**
* @param location
* @return
* @throws JMSException
*/
public Packet readPacket(RecordLocation location) throws JMSException {
try {
org.activeio.Packet data = journal.read(location);
DataInputStream is = new DataInputStream(new PacketInputStream(data));
byte type = is.readByte();
if (type != PACKET_RECORD_TYPE) {
throw new IOException("Record is not a packet type.");
}
String destination = is.readUTF();
Packet packet = wireFormat.readPacket(data);
is.close();
return packet;
}
catch (InvalidRecordLocationException e) {
throw createReadException(location, e);
}
catch (IOException e) {
throw createReadException(location, e);
}
}
/**
* Move all the messages that were in the journal into long term storeage. We just replay and do a checkpoint.
*
* @throws JMSException
* @throws IOException
* @throws InvalidRecordLocationException
* @throws IllegalStateException
*/
private void recover() throws IllegalStateException, InvalidRecordLocationException, IOException, JMSException {
RecordLocation pos = null;
int transactionCounter = 0;
log.info("Journal Recovery Started.");
// While we have records in the journal.
while ((pos = journal.getNextRecordLocation(pos)) != null) {
org.activeio.Packet data = journal.read(pos);
DataInputStream is = new DataInputStream(new PacketInputStream(data));
// Read the destination and packate from the record.
String destination = null;
Packet packet = null;
try {
byte type = is.readByte();
switch (type) {
case PACKET_RECORD_TYPE:
// Is the current packet part of the destination?
destination = is.readUTF();
packet = wireFormat.readPacket(data);
// Try to replay the packet.
if (packet instanceof ActiveMQMessage) {
ActiveMQMessage msg = (ActiveMQMessage) packet;
JournalMessageStore store = (JournalMessageStore) createMessageStore(destination, msg.getJMSActiveMQDestination().isQueue());
if( msg.getTransactionId()!=null ) {
transactionStore.addMessage(store, msg, pos);
} else {
store.replayAddMessage(msg);
transactionCounter++;
}
}
else if (packet instanceof MessageAck) {
MessageAck ack = (MessageAck) packet;
JournalMessageStore store = (JournalMessageStore) createMessageStore(destination, ack.getDestination().isQueue());
if( ack.getTransactionId()!=null ) {
transactionStore.removeMessage(store, ack, pos);
} else {
store.replayRemoveMessage(ack);
transactionCounter++;
}
}
else {
log.error("Unknown type of packet in transaction log which will be discarded: " + packet);
}
break;
case TX_COMMAND_RECORD_TYPE:
TxCommand command = new TxCommand();
command.setType(is.readByte());
command.setWasPrepared(is.readBoolean());
switch(command.getType()) {
case TxCommand.LOCAL_COMMIT:
case TxCommand.LOCAL_ROLLBACK:
command.setTransactionId(is.readUTF());
break;
default:
command.setTransactionId(ActiveMQXid.read(is));
break;
}
// Try to replay the packet.
switch(command.getType()) {
case TxCommand.XA_PREPARE:
transactionStore.replayPrepare(command.getTransactionId());
break;
case TxCommand.XA_COMMIT:
case TxCommand.LOCAL_COMMIT:
Tx tx = transactionStore.replayCommit(command.getTransactionId(), command.getWasPrepared());
// Replay the committed operations.
if( tx!=null) {
tx.getOperations();
for (Iterator iter = tx.getOperations().iterator(); iter.hasNext();) {
TxOperation op = (TxOperation) iter.next();
if( op.operationType == TxOperation.ADD_OPERATION_TYPE ) {
op.store.replayAddMessage((ActiveMQMessage) op.data);
}
if( op.operationType == TxOperation.REMOVE_OPERATION_TYPE) {
op.store.replayRemoveMessage((MessageAck) op.data);
}
if( op.operationType == TxOperation.ACK_OPERATION_TYPE) {
JournalAck ack = (JournalAck) op.data;
((JournalTopicMessageStore)op.store).replayAcknowledge(ack.getSubscription(), new MessageIdentity(ack.getMessageId()));
}
}
transactionCounter++;
}
break;
case TxCommand.LOCAL_ROLLBACK:
case TxCommand.XA_ROLLBACK:
transactionStore.replayRollback(command.getTransactionId());
break;
}
break;
case ACK_RECORD_TYPE:
destination = is.readUTF();
String subscription = is.readUTF();
String messageId = is.readUTF();
Object transactionId=null;
JournalTopicMessageStore store = (JournalTopicMessageStore) createMessageStore(destination, false);
if( transactionId!=null ) {
JournalAck ack = new JournalAck(destination, subscription, messageId, transactionId);
transactionStore.acknowledge(store, ack, pos);
} else {
store.replayAcknowledge(subscription, new MessageIdentity(messageId));
transactionCounter++;
}
case COMMAND_RECORD_TYPE:
break;
default:
log.error("Unknown type of record in transaction log which will be discarded: " + type);
break;
}
}
finally {
is.close();
}
}
RecordLocation location = writeCommand("RECOVERED", true);
journal.setMark(location, true);
log.info("Journal Recovered: " + transactionCounter + " message(s) in transactions recovered.");
}
private JMSException createReadException(RecordLocation location, Exception e) {
return JMSExceptionHelper.newJMSException("Failed to read to journal for: " + location + ". Reason: " + e, e);
}
protected JMSException createWriteException(Packet packet, Exception e) {
return JMSExceptionHelper.newJMSException("Failed to write to journal for: " + packet + ". Reason: " + e, e);
}
private XAException createWriteException(TxCommand command, Exception e) {
return (XAException)new XAException("Failed to write to journal for: " + command + ". Reason: " + e).initCause(e);
}
protected JMSException createWriteException(String command, Exception e) {
return JMSExceptionHelper.newJMSException("Failed to write to journal for command: " + command + ". Reason: " + e, e);
}
protected JMSException createRecoveryFailedException(Exception e) {
return JMSExceptionHelper.newJMSException("Failed to recover from journal. Reason: " + e, e);
}
public ClockDaemon getClockDaemon() {
if (clockDaemon == null) {
clockDaemon = new ClockDaemon();
clockDaemon.setThreadFactory(new ThreadFactory() {
public Thread newThread(Runnable runnable) {
Thread thread = new Thread(runnable, "Checkpoint Timmer");
thread.setDaemon(true);
return thread;
}
});
}
return clockDaemon;
}
public void setClockDaemon(ClockDaemon clockDaemon) {
this.clockDaemon = clockDaemon;
}
/**
* @param xid
* @return
*/
public RecordLocation writeTxCommand(TxCommand command, boolean sync) throws XAException {
try {
PacketByteArrayOutputStream pos = new PacketByteArrayOutputStream();
DataOutputStream os = new DataOutputStream(pos);
os.writeByte(TX_COMMAND_RECORD_TYPE);
os.writeByte(command.getType());
os.writeBoolean(command.getWasPrepared());
switch(command.getType()) {
case TxCommand.LOCAL_COMMIT:
case TxCommand.LOCAL_ROLLBACK:
os.writeUTF( (String) command.getTransactionId() );
break;
default:
ActiveMQXid xid = (ActiveMQXid) command.getTransactionId();
xid.write(os);
break;
}
os.close();
return journal.write(pos.getPacket(), sync);
}
catch (IOException e) {
throw createWriteException(command, e);
}
}
/**
* @param destinationName
* @param persistentKey
* @param messageIdentity
* @param b
* @return
*/
public RecordLocation writePacket(String destinationName, String subscription, MessageIdentity messageIdentity, boolean sync) throws JMSException{
try {
PacketByteArrayOutputStream pos = new PacketByteArrayOutputStream();
DataOutputStream os = new DataOutputStream(pos);
os.writeByte(ACK_RECORD_TYPE);
os.writeUTF(destinationName);
os.writeUTF(subscription);
os.writeUTF(messageIdentity.getMessageID());
os.close();
return journal.write(pos.getPacket(), sync);
}
catch (IOException e) {
throw createWriteException("Ack for message: "+messageIdentity, e);
}
}
public JournalTransactionStore getTransactionStore() {
return transactionStore;
}
public int getLogFileCount() {
return logFileCount;
}
public void setLogFileCount(int logFileCount) {
this.logFileCount = logFileCount;
}
public int getLogFileSize() {
return logFileSize;
}
public void setLogFileSize(int logFileSize) {
this.logFileSize = logFileSize;
}
public long getCheckpointInterval() {
return checkpointInterval;
}
public void setCheckpointInterval(long checkpointInterval) {
this.checkpointInterval = checkpointInterval;
}
}