*

* Copyright 2013 LinkedIn Corp. All rights reserved

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*

*/

    public static final String MODULE = TestGenericDispatcher.class.getName();

    public static final Logger LOG = Logger.getLogger(MODULE);

    private DbusEventBuffer.Config _generic100KBufferConfig;

    private DbusEventBuffer.StaticConfig _generic100KBufferStaticConfig;

    private DatabusSourcesConnection.Config _genericRelayConnConfig;

    private DatabusSourcesConnection.StaticConfig _genericRelayConnStaticConfig;

    private static final String SOURCE1_SCHEMA_STR = "{\"name\":\"source1\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";

    private static final String SOURCE2_SCHEMA_STR = "{\"name\":\"source2\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";;

    private static final String SOURCE3_SCHEMA_STR = "{\"name\":\"source3\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";;

    private static final String META1_SCHEMA_STR = "{\"name\":\"meta-source\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";

    private static final String META2_SCHEMA_STR = "{\"name\":\"meta-source\",\"type\":\"record\",\"fields\":[{\"name\":\"sNew\",\"type\":\"string\"}]}";;

    private void initBufferWithEvents(DbusEventBuffer eventsBuf,

        long keyBase,

        int numEvents,

        short srcId,

        Hashtable<Long, AtomicInteger> keyCounts,

        Hashtable<Short, AtomicInteger> srcidCounts)

    {

      String schemaStr=null;

      switch (srcId)

      {

        case 1:

          schemaStr=SOURCE1_SCHEMA_STR;

          break;

        case 2:

          schemaStr=SOURCE2_SCHEMA_STR;

          break;

        case 3:

          schemaStr=SOURCE3_SCHEMA_STR;

          break;

        default:

          break;

      }

      SchemaId schemaId = schemaStr != null ? SchemaId.createWithMd5(schemaStr) : null;

      byte[] schemaBytes = schemaId != null? schemaId.getByteArray(): new byte[16];

      initBufferWithEvents(eventsBuf, keyBase, numEvents, srcId,schemaBytes, keyCounts, srcidCounts);

    }

    private void initBufferWithEvents(DbusEventBuffer eventsBuf,

            long keyBase,

            int numEvents,

            short srcId,

            byte[] schemaId,

            Hashtable<Long, AtomicInteger> keyCounts,

            Hashtable<Short, AtomicInteger> srcidCounts)

    {

        if (null != srcidCounts) srcidCounts.put(srcId, new AtomicInteger(0));

        for (long i = 0; i < numEvents; ++i)

        {

            String value = "{\"s\":\"value" + i + "\"}";

            try {

                eventsBuf.appendEvent(new DbusEventKey(keyBase + i), (short)0, (short)1, (short)0, srcId,

                        schemaId, value.getBytes("UTF-8"), false);

            } catch (UnsupportedEncodingException e) {

                //ignore

            }

            if (null != keyCounts) keyCounts.put(keyBase + i, new AtomicInteger(0));

        }

    }

    @BeforeClass

    public void beforeClass() throws Exception

    {

      TestUtil.setupLoggingWithTimestampedFile(true, "/tmp/TestGenericDispatcher_", ".log", Level.INFO);

        _generic100KBufferConfig = new DbusEventBuffer.Config();

        _generic100KBufferConfig.setAllocationPolicy(AllocationPolicy.HEAP_MEMORY.toString());

        _generic100KBufferConfig.setMaxSize(100000);

        _generic100KBufferConfig.setQueuePolicy(QueuePolicy.BLOCK_ON_WRITE.toString());

        _generic100KBufferConfig.setScnIndexSize(10000);

        _generic100KBufferConfig.setAverageEventSize(100000);

        _generic100KBufferStaticConfig = _generic100KBufferConfig.build();

        _genericRelayConnConfig = new DatabusSourcesConnection.Config();

        _genericRelayConnConfig.setConsumerParallelism(1);

        _genericRelayConnConfig.setEventBuffer(_generic100KBufferConfig);

        _genericRelayConnConfig.setConsumerTimeBudgetMs(1000);

        _genericRelayConnStaticConfig = _genericRelayConnConfig.build();

    }

    @Test(groups = {"small", "functional"})

    public void testOneWindowHappyPath()

    {

        final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowHappyPath");

        //log.setLevel(Level.DEBUG);

        log.info("start");

        int source1EventsNum = 2;

        int source2EventsNum = 2;

        Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();

        Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();

        final TestGenericDispatcherEventBuffer eventsBuf =

            new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);

        eventsBuf.start(0);

        final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);

        //svsConsumer.getLog().setLevel(Level.DEBUG);

        DatabusStreamConsumer mockConsumer =

                new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);

        SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);

        List<String> sources = new ArrayList<String>();

        Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();

        for (int i = 1; i <= 3; ++i)

        {

            IdNamePair sourcePair = new IdNamePair((long)i, "source" + i);

            sources.add(sourcePair.getName());

            sourcesMap.put(sourcePair.getId(), sourcePair);

        }

        DatabusV2ConsumerRegistration consumerReg =

                new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);

        List<DatabusV2ConsumerRegistration> allRegistrations =

                Arrays.asList(consumerReg);

        MultiConsumerCallback callback =

                new MultiConsumerCallback(

                        allRegistrations,

                        Executors.newSingleThreadExecutor(),

                        1000,

                        new StreamConsumerCallbackFactory(null, null),

                        null,

                        null,

                        null,

                        null);

        callback.setSourceMap(sourcesMap);

        List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);

        RelayDispatcher dispatcher =

                new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs,

                        new InMemoryPersistenceProvider(),

                        eventsBuf, callback, null,null,null, null, null);

        dispatcher.setSchemaIdCheck(false);

        Thread dispatcherThread = new Thread(dispatcher, "testOneWindowHappyPath-dispatcher");

        //dispatcherThread.setDaemon(true);

        dispatcherThread.start();

        HashMap<Long, List<RegisterResponseEntry>> schemaMap =

                new HashMap<Long, List<RegisterResponseEntry>>();

        List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();

        List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();

        List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();

        l1.add(new RegisterResponseEntry(1L, (short)1,SOURCE1_SCHEMA_STR));

        l2.add(new RegisterResponseEntry(2L, (short)1,SOURCE2_SCHEMA_STR));

        l3.add(new RegisterResponseEntry(3L, (short)1,SOURCE3_SCHEMA_STR));

        schemaMap.put(1L, l1);

        schemaMap.put(2L, l2);

        schemaMap.put(3L, l3);

        dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));

        dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));

        eventsBuf.startEvents();

        initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short)1, keyCounts, srcidCounts);

        initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short)2, keyCounts, srcidCounts);

        eventsBuf.endEvents(100L,null);

        try

        {

            Thread.sleep(2000);

        }

        catch (InterruptedException ie) {}

        dispatcher.shutdown();

        Checkpoint cp = dispatcher.getDispatcherState().getLastSuccessfulCheckpoint();

        Assert.assertEquals(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET, cp.getWindowOffset());

        Assert.assertEquals(cp.getWindowScn(), cp.getPrevScn());

        for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i)

        {

            assertEquals("correct amount of callbacks for key " + i,

                    1, keyCounts.get(i).intValue());

        }

        assertEquals("correct amount of callbacks for srcid 1", source1EventsNum,

                     srcidCounts.get((short)1).intValue());

        assertEquals("correct amount of callbacks for srcid 2", source2EventsNum,

                     srcidCounts.get((short)2).intValue());

        verifyNoLocks(log, eventsBuf);

        log.info("end\n");

    }

    /**

     * @param log

     * @param eventsBuf

     */

    private void verifyNoLocks(final Logger log,

                               final TestGenericDispatcherEventBuffer eventsBuf)

    {

      TestUtil.assertWithBackoff(new ConditionCheck()

      {

        @Override

        public boolean check()

        {

          if (null != log)

              log.debug("rwlocks: " + eventsBuf.getRangeLocksProvider());

          return 0 == eventsBuf.getRangeLocksProvider().getNumReaders();

        }

      }, "no locks remaining on buffer", 5000, log);

    }

    @Test(groups = {"small", "functional"})

    /**

     * Test a relatively big window which forces a checkpoint which fails. Then trigger

     * a rollback. The dispatcher should shutdown. */

    public void testRollbackFailure() throws InvalidConfigException

    {

        final Logger log = Logger.getLogger("TestGenericDispatcher.testRollbackFailure");

        //log.setLevel(Level.DEBUG);

        log.info("start");