/*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.linkedin.databus.client.bootstrap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.avro.Schema;
import org.apache.log4j.Logger;
import com.linkedin.databus.client.DatabusHttpClientImpl;
import com.linkedin.databus.client.DatabusHttpClientImpl.CheckpointPersistenceStaticConfig.ProviderType;
import com.linkedin.databus.client.DatabusSourcesConnection;
import com.linkedin.databus.client.generic.ConsumerPauseRequestProcessor;
import com.linkedin.databus.client.generic.DatabusConsumerPauseInterface;
import com.linkedin.databus.client.pub.ConsumerCallbackResult;
import com.linkedin.databus.client.pub.DatabusBootstrapConsumer;
import com.linkedin.databus.client.pub.DatabusClientException;
import com.linkedin.databus.client.pub.DatabusStreamConsumer;
import com.linkedin.databus.client.pub.DbusEventDecoder;
import com.linkedin.databus.client.pub.mbean.UnifiedClientStats;
import com.linkedin.databus.client.pub.SCN;
import com.linkedin.databus.client.pub.ServerInfo.ServerInfoBuilder;
import com.linkedin.databus.client.SingleSourceSCN;
import com.linkedin.databus.core.DbusEvent;
import com.linkedin.databus.core.DbusEventBuffer;
import com.linkedin.databus.core.FileBasedEventTrackingCallback;
import com.linkedin.databus.core.monitoring.mbean.StatsCollectors;
import com.linkedin.databus.core.util.InvalidConfigException;
import com.linkedin.databus2.core.DatabusException;
import com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException;
/**
*
*/
public class IntegratedDummyDatabusConsumer extends DatabusBootstrapDummyConsumer
implements DatabusStreamConsumer, DatabusConsumerPauseInterface
{
public final static String MODULE = IntegratedDummyDatabusConsumer.class.getName();
public final static Logger LOG = Logger.getLogger(MODULE);
private static final int FIFTEEN_HUNDRED_KILOBYTES_IN_BYTES = 1500000;
private static final int TEN_MEGABYTES_IN_BYTES = 10000000;
private DatabusHttpClientImpl _dbusClient;
private FileBasedEventTrackingCallback _fileBasedCallback;
private long _maxEventBufferSize;
private int _maxReadBufferSize;
private long _maxBootstrapWindownScn;
private long _maxRelayWindowScn;
private boolean _isPaused;
private boolean _useConsumerTimeout;
private int _numUserSpecifiedErrorResults;
public IntegratedDummyDatabusConsumer(String outputFilename, long maxEventBufferSize, int maxReadBufferSize,
boolean useConsumerTimeout)
{
// use the default log4j configuration
//System.out.println("level set to debug");
_fileBasedCallback = new FileBasedEventTrackingCallback(outputFilename, false);
_maxEventBufferSize = maxEventBufferSize;
_maxReadBufferSize = maxReadBufferSize;
_useConsumerTimeout = useConsumerTimeout;
}
public IntegratedDummyDatabusConsumer(String outputFilename, long maxEventBufferSize, int maxReadBufferSize)
{
this(outputFilename, maxEventBufferSize, maxReadBufferSize, true);
}
public IntegratedDummyDatabusConsumer(String outputFilename)
{
this(outputFilename, TEN_MEGABYTES_IN_BYTES, FIFTEEN_HUNDRED_KILOBYTES_IN_BYTES);
}
@Override
public ConsumerCallbackResult onBootstrapEvent(DbusEvent e, DbusEventDecoder eventDecoder)
{
waitIfPaused();
// this is for the final comparison only because events from bootstrap
// have different scn. But the window scn shall be of the same.
_fileBasedCallback.onEvent(e);
printBootstrapEventInfo(BootstrapStage.OnBootstrapEvent, e.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onEndBootstrapSequence(SCN endScn)
{
waitIfPaused();
_maxBootstrapWindownScn = ((SingleSourceSCN)endScn).getSequence();
printBootstrapEventInfo(BootstrapStage.EndBootstrapSequence, endScn.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onCheckpoint(SCN checkpointScn)
{
waitIfPaused();
printStreamEventInfo(StreamStage.OnCheckpointEvent, checkpointScn.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder)
{
waitIfPaused();
_fileBasedCallback.onEvent(e);
printStreamEventInfo(StreamStage.OnStreamEvent, e.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onEndDataEventSequence(SCN endScn)
{
waitIfPaused();
_maxRelayWindowScn = ((SingleSourceSCN)endScn).getSequence();
printStreamEventInfo(StreamStage.EndDataEventSequence, endScn.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onEndSource(String source, Schema sourceSchema)
{
printStreamEventInfo(StreamStage.EndStreamSource,
" source=" + source +
" schema=" + ((null == sourceSchema) ? "null" : sourceSchema.getFullName()));
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onRollback(SCN startScn)
{
printStreamEventInfo(StreamStage.InvalidStage, "rollback not implemented");
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn)
{
printStreamEventInfo(StreamStage.StartDataEventSequence, startScn.toString());
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onStartSource(String source, Schema sourceSchema)
{
printStreamEventInfo(StreamStage.StartStreamSource,
" source=" + source +
" schema=" + ((null == sourceSchema) ? "null" : sourceSchema.getFullName()));
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onStartConsumption()
{
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onStopConsumption()
{
return getUserSpecifiedCallbackResult();
}
@Override
public ConsumerCallbackResult onError(Throwable err)
{
return getUserSpecifiedCallbackResult();
}
public void initConn(List<String> sources) throws IOException, InvalidConfigException,
DatabusClientException, DatabusException
{
StringBuilder sourcesString = new StringBuilder();
boolean firstSrc = true;
for (String source: sources)
{
if (! firstSrc) sourcesString.append(",");
firstSrc = false;
sourcesString.append(source);
}
_fileBasedCallback.init();
ArrayList<DatabusBootstrapConsumer> bootstrapCallbacks = new ArrayList<DatabusBootstrapConsumer>();
bootstrapCallbacks.add(this);
ArrayList<DatabusStreamConsumer> streamCallbacks = new ArrayList<DatabusStreamConsumer>();
streamCallbacks.add(this);
DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
clientConfigBuilder.getContainer().getJmx().setJmxServicePort(5555);
clientConfigBuilder.getContainer().setId(545454);
clientConfigBuilder.getContainer().setHttpPort(8082);
clientConfigBuilder.getCheckpointPersistence().setType(ProviderType.FILE_SYSTEM.toString());
clientConfigBuilder.getCheckpointPersistence().getFileSystem().setRootDirectory("./integratedconsumer-checkpoints");
clientConfigBuilder.getCheckpointPersistence().setClearBeforeUse(true);
clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
DatabusSourcesConnection.Config srcDefaultConfig= new DatabusSourcesConnection.Config();
srcDefaultConfig.setFreeBufferThreshold((int) (_maxEventBufferSize*0.05));
srcDefaultConfig.setCheckpointThresholdPct(80);
srcDefaultConfig.setConsumerTimeBudgetMs(_useConsumerTimeout? 60000 : 0); // 60 sec before retries (unless disabled)
srcDefaultConfig.getDispatcherRetries().setMaxRetryNum(3); // max of 3 retries
clientConfigBuilder.setConnectionDefaults(srcDefaultConfig);
DbusEventBuffer.Config eventBufferConfig = clientConfigBuilder.getConnectionDefaults().getEventBuffer();
eventBufferConfig.setMaxSize(_maxEventBufferSize);
eventBufferConfig.setAverageEventSize(_maxReadBufferSize);
// TODO: the following shall be used once we can set eventbuffer for bootstrap through the config builder (DDSDBUS-82)
// For now, bootstrap buffer will use the same config as relay buffer.
//clientConfigBuilder.getConnectionDefaults().
DatabusHttpClientImpl.StaticConfig clientConfig = clientConfigBuilder.build();
ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
relayBuilder.setName("DefaultRelay");
relayBuilder.setHost("localhost");
relayBuilder.setPort(9000);
relayBuilder.setSources(sourcesString.toString());
ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
bootstrapBuilder.setName("DefaultBootstrapServices");
bootstrapBuilder.setHost("localhost");
bootstrapBuilder.setPort(6060);
bootstrapBuilder.setSources(sourcesString.toString());
_dbusClient = new DatabusHttpClientImpl(clientConfig);
_dbusClient.registerDatabusStreamListener(this, sources, null);
_dbusClient.registerDatabusBootstrapListener(this, sources, null);
// add pause processor
try
{
_dbusClient.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME,
new ConsumerPauseRequestProcessor(null, this));
}
catch (ProcessorRegistrationConflictException e)
{
LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
}
}
protected void printStreamEventInfo(StreamStage stage, String info)
{
LOG.debug(stage + ": " + info);
}
public synchronized void start() throws Exception
{
_isPaused = false;
_dbusClient.start();
LOG.info(MODULE + " started!");
}
public synchronized void shutdown()
{
_isPaused = false;
_dbusClient.shutdown();
}
public long getMaxBootstrapWindowScn()
{
return _maxBootstrapWindownScn;
}
public long getMaxRelayWindowScn()
{
return _maxRelayWindowScn;
}
public StatsCollectors<UnifiedClientStats> getUnifiedClientStatsCollectors()
{
return _dbusClient.getUnifiedClientStatsCollectors();
}
public List<DatabusSourcesConnection> getRelayConnections()
{
return _dbusClient.getRelayConnections();
}
@Override
public synchronized void pause()
{
_isPaused = true;
LOG.info("Consumer is set to pause!");
}
@Override
public synchronized void resume()
{
_isPaused = false;
notifyAll();
LOG.info("Consumer is set to resume!");
}
@Override
public synchronized void waitIfPaused()
{
while (_isPaused)
{
LOG.info("Consumer is paused!");
try
{
wait();
}
catch (InterruptedException e)
{
// resume waiting, nothing to do
}
}
}
public synchronized void setNumUserSpecifiedErrorResults(int numErrorResults)
{
LOG.info("Will return " + numErrorResults + " ConsumerCallbackResult.ERROR as requested.");
_numUserSpecifiedErrorResults = numErrorResults;
}
protected synchronized ConsumerCallbackResult getUserSpecifiedCallbackResult()
{
if (_numUserSpecifiedErrorResults > 0)
{
--_numUserSpecifiedErrorResults;
if (_numUserSpecifiedErrorResults > 0)
{
LOG.info("Returning ConsumerCallbackResult.ERROR as requested; still have " +
_numUserSpecifiedErrorResults + " more to go.");
}
else
{
LOG.info("Returning ConsumerCallbackResult.ERROR as requested (last one).");
}
return ConsumerCallbackResult.ERROR;
}
return ConsumerCallbackResult.SUCCESS;
}
enum StreamStage
{
StartDataEventSequence,
EndDataEventSequence,
OnStreamEvent,
OnCheckpointEvent,
StartStreamSource,
EndStreamSource,
InvalidStage
}
public static void main(String args[]) throws Exception
{
//BasicConfigurator.configure();
//Logger.getRootLogger().setLevel(Level.INFO);
IntegratedDummyDatabusConsumer consumer = new IntegratedDummyDatabusConsumer("IntegratedDummyDatabusConsumerMain");
ArrayList<String> sources = new ArrayList<String>();
sources.add("source1");
consumer.initConn(sources);
consumer.start();
consumer.shutdown();
}
}