*

* Copyright 2013 LinkedIn Corp. All rights reserved

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*

*/

  public static Logger LOG = Logger.getLogger(TestGoldenGateEventProducer.class);

  private final String avroSchema =

    "{" +

    "\"type\" : \"record\"," +

    "\"name\" : \"user\"," +

    "\"namespace\" : \"my.example\"," +

    "\"fields\" : [{\"name\" : \"name\", \"type\" : \"string\"}]" +

    "}";

  private final String avroSchema2 =

      "{" +

      "\"type\" : \"record\"," +

      "\"name\" : \"user\"," +

      "\"namespace\" : \"my.example\"," +

      "\"fields\" : [{\"name\" : \"name1\", \"type\" : \"string\"}," +

      "              {\"name\" : \"name2\", \"type\" : \"string\"}]" +

      "}";

  private final String sourceAvroSchema = "{"

  + "  \"name\" : \"Shortlinks_V3\",  "

  + "   \"doc\" : \"Auto-generated Avro schema for SHORTLINKS. Generated at May 21, 2013 12:05:14 PM PDT\","

  + " \"type\" : \"record\", "

  + "   \"meta\" : \"dbFieldName=SHORTLINKS;dbFieldType=SHORTLINKS;pk=linkCode;\","

  + "   \"fields\" : [ {"

  + "     \"name\" : \"linkCode\", "

  + "     \"type\" : [ \"string\", \"null\" ],"

  + "     \"meta\" : \"dbFieldName=LINK_CODE;dbFieldPosition=0;dbFieldType=VARCHAR2;\""

  + "   }, {"

  + "     \"name\" : \"ggModiTs\","

  + "     \"type\" : [ \"long\", \"null\" ],"

  + "     \"meta\" : \"dbFieldName=GG_MODI_TS;dbFieldPosition=8;dbFieldType=TIMESTAMP;\""

  + "   }, {"

  + "     \"name\" : \"ggStatus\","

  + "     \"type\" : [ \"string\", \"null\" ],"

  + "     \"meta\" : \"dbFieldName=GG_STATUS;dbFieldPosition=9;dbFieldType=VARCHAR2;\""

  + "   } ],"

  + "   \"namespace\" : \"com.linkedin.events.sourc\""

  + " }";

  private final String SCNPATTERN = "SCNPATTERN";

  private final String _transactionPattern = ""

  + "<transaction timestamp=\"2013-07-29:13:26:15.000000\">\n"

  + "<dbupdate table=\"part1.source1\" type=\"insert\">\n"

  + "    <columns>\n"

  + "      <column name=\"LINK_CODE\" key=\"true\">100</column>\n"

  + "      <column name=\"GG_MODI_TS\">2013-07-28:13:26:15.208130000</column>\n"

  + "      <column name=\"GG_STATUS\">o</column>\n"

  + "    </columns>\n"

  + "    <tokens>\n"

  + "      <token name=\"TK-XID\">4.24.94067</token>\n"

  + "      <token name=\"TK-CSN\">" + SCNPATTERN + "</token>\n"

  + "      <token name=\"TK-UNAME\">SOURCE1</token>\n"

  + "    </tokens>\n"

  + "  </dbupdate>\n"

  + "  <dbupdate table=\"part1.source2\" type=\"insert\">\n"

  + "    <columns>\n"

  + "      <column name=\"LINK_CODE\" key=\"true\">101</column>\n"

  + "      <column name=\"GG_MODI_TS\">2013-07-29:13:26:15.212166000</column>\n"

  + "      <column name=\"GG_STATUS\">o</column>\n"

  + "    </columns>\n"

  + "    <tokens>\n"

  + "      <token name=\"TK-XID\">4.24.94067</token>\n"

  + "      <token name=\"TK-CSN\">" + SCNPATTERN + "</token>\n"

  + "      <token name=\"TK-UNAME\">SOURCE2</token>\n"

  + "    </tokens>\n"

  + "  </dbupdate>\n"

  + "</transaction>";

  private final int _transactionPatternSize = _transactionPattern.length() - 7*2; // -7 to compensate diff in length betwee "SCNPATTERN" and length of actuall scn (length("200"))

  static {

    TestUtil.setupLogging(Level.INFO);

  }

  @BeforeClass

  public void setUpClass() throws InvalidConfigException

  {

    String rootDirPath = System.getProperty(TestGoldenGateEventProducer.class

        .getName() + ".rootDir");

    // set up logging

    TestUtil.setupLoggingWithTimestampedFile(true,

        "/tmp/TestGoldenGateEventProducer_", ".log", Level.INFO);

    InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory());

  }

  @Test

  public void testRegexParsing() throws Exception

  {

    //Set SCN

    Pattern rexpr = Pattern.compile("(<token\\s+name=\"TK-CSN\"\\s*>\\s*([0-9]+)\\s*</token>)");

    String source1 = "<token name=\"TK-CSN\">1234</token><token name=\"TK-CSN\">1234</token>";

    Matcher result1 = rexpr.matcher(source1);

    while (result1.find())

    {

        String m = result1.group(2);

        long newScn =  Long.parseLong(m);

        Assert.assertEquals(newScn, 1234);

    }

    String source2 = "<token   name=\"TK-CSN\"   >  1234  </token><token   name=\"TK-CSN\" >1234 </token>";

    Matcher result2 = rexpr.matcher(source2);

    while (result2.find())

    {

        String m = result2.group(2);

        long newScn =  Long.parseLong(m);

        Assert.assertEquals(newScn, 1234);

    }

  }

  @Test

  public void testDbUpdatesMergeUtils()

    throws Exception

  {

    short[] sourceIds = new short[] { 505, 506 };

    long scn = 10;

    // Case when 2 txn have same source and keys

    {

      String key = "key1";

      String oldVal = "val1";

      Map<String, String> keyVals1 = new HashMap<String, String>();

      keyVals1.put(key, oldVal);

      Map<String, String> keyVals2 = new HashMap<String, String>();

      String newVal = "val2";

      keyVals2.put(key, newVal);

      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =

          generateUpdatesForSchema2(sourceIds, keyVals1, scn);

      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =

          generateUpdatesForSchema2(sourceIds, keyVals2, scn);

      List<TransactionState.PerSourceTransactionalUpdate> result =

          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());

      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);

      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      DBUpdateImage dbi = result.get(0).getDbUpdatesSet().iterator().next();

      DBUpdateImage expDbi = newDbUpdates.get(0).getDbUpdatesSet().iterator().next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      String k = (String) dbi.getGenericRecord().get("name1");

      String v = (String) dbi.getGenericRecord().get("name2");

      String expK = (String) expDbi.getGenericRecord().get("name1");

      String expV = (String) expDbi.getGenericRecord().get("name2");

      Assert.assertEquals("Key Check", expK, k);

      Assert.assertEquals("Val Check", expV, v);

      dbi = result.get(1).getDbUpdatesSet().iterator().next();

      expDbi = newDbUpdates.get(1).getDbUpdatesSet().iterator().next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      k = (String) dbi.getGenericRecord().get("name1");

      v = (String) dbi.getGenericRecord().get("name2");

      expK = (String) expDbi.getGenericRecord().get("name1");

      expV = (String) expDbi.getGenericRecord().get("name2");

      Assert.assertEquals("Key Check", expK, k);

      Assert.assertEquals("Val Check", expV, v);

    }

    // Case when 2 txn have same source but different keys

    {

      String key = "key1";

      String oldVal = "val1";

      Map<String, String> keyVals1 = new HashMap<String, String>();

      keyVals1.put(key, oldVal);

      Map<String, String> keyVals2 = new HashMap<String, String>();

      key = "key2";

      String newVal = "val2";

      keyVals2.put(key, newVal);

      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =

          generateUpdatesForSchema2(sourceIds, keyVals1, scn);

      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =

          generateUpdatesForSchema2(sourceIds, keyVals2, scn);

      List<TransactionState.PerSourceTransactionalUpdate> result =

          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());

      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 2);

      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 2);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();

      DBUpdateImage dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      String k = (String) dbi.getGenericRecord().get("name1");

      String v = (String) dbi.getGenericRecord().get("name2");

      dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      String k2 = (String) dbi.getGenericRecord().get("name1");

      String v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))

      {

        Assert.assertEquals("Val Check", keyVals1.get(k), v);

        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);

      }

      else

      {

        Assert.assertEquals("Val Check", keyVals2.get(k), v);

        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);

      }

      itr = result.get(1).getDbUpdatesSet().iterator();

      dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      k = (String) dbi.getGenericRecord().get("name1");

      v = (String) dbi.getGenericRecord().get("name2");

      dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      k2 = (String) dbi.getGenericRecord().get("name1");

      v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))

      {

        Assert.assertEquals("Val Check", keyVals1.get(k), v);

        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);

      }

      else

      {

        Assert.assertEquals("Val Check", keyVals2.get(k), v);

        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);

      }

    }

    // Case when 2 txns have different sources

    {

      String key = "key1";

      String oldVal = "val1";

      Map<String, String> keyVals1 = new HashMap<String, String>();

      keyVals1.put(key, oldVal);

      Map<String, String> keyVals2 = new HashMap<String, String>();

      key = "key2";

      String newVal = "val2";

      keyVals2.put(key, newVal);

      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =

          generateUpdatesForSchema2(new short[] { sourceIds[0] }, keyVals1, scn);

      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =

          generateUpdatesForSchema2(new short[] { sourceIds[1] }, keyVals2, scn);

      List<TransactionState.PerSourceTransactionalUpdate> result =

          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());

      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);

      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();

      DBUpdateImage dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      String k = (String) dbi.getGenericRecord().get("name1");

      String v = (String) dbi.getGenericRecord().get("name2");

      Assert.assertEquals("Val Check", keyVals1.get(k), v);

      itr = result.get(1).getDbUpdatesSet().iterator();

      dbi = itr.next();

      Assert.assertEquals("SCN Check", scn, dbi.getScn());

      k = (String) dbi.getGenericRecord().get("name1");

      v = (String) dbi.getGenericRecord().get("name2");

      Assert.assertEquals("Val Check", keyVals2.get(k), v);

    }

  }

  @Test

  public void testSCNRegressionStats() throws Exception

  {

    short[] sourceIds = new short[] { 505, 506 };

    String[] sourceNames = new String[] { "source1", "source2" };

    PhysicalSourceStaticConfig pssc =

        buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start producer

    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);

    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);

    Method method = getOnTransactionEndMethod();

    // generates the updates

    List<String> keys = new ArrayList<String>();

    keys.add("key1");

    GGParserStatistics ggParserStats = gg.getParserStats();

    // SCN = 10

    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 =

        generateUpdates(sourceIds, keys, 10);

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

        new TransactionInfo(0, 0, timestamp, 10) });

    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;

    Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());

    Assert.assertEquals("NumSCNRegressions", -1, ggParserStats.getLastRegressedScn());

    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());

    // SCN = 5 Regression

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 =

        generateUpdates(sourceIds, keys, 5); // SCN Regression here

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, 5) });

    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());

    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());

    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());

    // SCN = 6 No Regression here

    dbUpdates2 = generateUpdates(sourceIds, keys, 6);

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, 6) });

    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());

    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());

    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());

    // SCN = 3 : SCN regression again

    dbUpdates2 = generateUpdates(sourceIds, keys, 3);

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, 3) });

    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());

    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());

    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());

    // SCN = 11: No regression here

    dbUpdates2 = generateUpdates(sourceIds, keys, 11);

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, 11) });

    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());

    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());

    Assert.assertEquals("MaxScn", 11, ggParserStats.getMaxScn());

  }

  /**

   *

   * Test to cover merging of transactions with same SCNs

   */

  @Test

  public void testTransactionsWithDuplicateSCN() throws Exception

  {

    short[] sourceIds = new short[] { 505, 506 };

    String[] sourceNames = new String[] { "source1", "source2" };

    PhysicalSourceStaticConfig pssc =

        buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start producer

    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);

    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);

    Method method = getOnTransactionEndMethod();

    // generates the updates

    List<String> keys = new ArrayList<String>();

    keys.add("key1");

    // SCN = 10 - Case where both transactions having same SCNs : same set of sources and

    // same keys

    long scn = 10;

    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 =

        generateUpdates(sourceIds, keys, scn);

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

        new TransactionInfo(0, 0, timestamp, scn) });

    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 =

        generateUpdates(sourceIds, keys, scn); // Same SCN as before

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, scn) });

    // Expect no events as events are not yet appended to buffer.

    testStats(gg,

              new EventStatsValues(sourceIds[0], 0, 0, 0, 0, 0),

              new EventStatsValues(sourceIds[1], 0, 0, 0, 0, 0),

              new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                   0,

                                   0,

                                   0,

                                   0,

                                   0));

    // SCN = 11 - Case where both transactions having same SCNs : same set of sources but

    // different keys

    {

      scn = 11;

      timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;

      dbUpdates1 = generateUpdates(sourceIds, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;

      keys.clear();

      keys.add("key2");

      dbUpdates2 = generateUpdates(sourceIds, keys, scn); // Same SCN as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      // Testing for SCN = 10 case

      testStats(gg,

                new EventStatsValues(sourceIds[0], 5, 0, 1, 0, 10),

                new EventStatsValues(sourceIds[1], 5, 0, 1, 0, 10),

                new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                     5,

                                     0,

                                     2,

                                     0,

                                     10));

    }

    // SCN = 12 - Case where both transactions having same SCNs but different set of

    // sources

    {

      scn = 12;

      keys.clear();

      keys.add("key2");

      timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 2;

      dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates2 = generateUpdates(new short[] { sourceIds[0] }, keys, scn); // Same SCN

                                                                             // as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      // Testing for SCN = 11 case

      testStats(gg,

                new EventStatsValues(sourceIds[0], 5, 0, 3, 0, 11),

                new EventStatsValues(sourceIds[1], 5, 0, 3, 0, 11),

                new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                     5,

                                     0,

                                     6,

                                     0,

                                     11));

    }

    // SCN = 13 - Case where more than 2 transactions having same SCNs and keys. The keys

    // will be merged.

    {

      scn = 13;

      timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;

      dbUpdates1 = generateUpdates(sourceIds, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates2 = generateUpdates(sourceIds, keys, scn); // Same SCN as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates1 = generateUpdates(sourceIds, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates2 = generateUpdates(sourceIds, keys, scn); // Same SCN as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      // Testing for SCN = 12 case

      testStats(gg,

                new EventStatsValues(sourceIds[0], 5, 0, 4, 0, 12),

                new EventStatsValues(sourceIds[1], 5, 0, 4, 0, 12),

                new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                     5,

                                     0,

                                     8,

                                     0,

                                     12));

    }

    // SCN = 14 - Case where more than 2 transactions having same SCNs but different keys.

    {

      scn = 14;

      timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;

      dbUpdates1 = generateUpdates(sourceIds, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates2 = generateUpdates(sourceIds, keys, scn); // Same SCN as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      keys.clear();

      keys.add("key5");

      dbUpdates1 = generateUpdates(sourceIds, keys, scn);

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

          new TransactionInfo(0, 0, timestamp, scn) });

      dbUpdates2 = generateUpdates(sourceIds, keys, scn); // Same SCN as before

      method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

          new TransactionInfo(0, 0, timestamp, scn) });

      // Testing for SCN = 13 case

      testStats(gg,

                new EventStatsValues(sourceIds[0], 5, 0, 5, 0, 13),

                new EventStatsValues(sourceIds[1], 5, 0, 5, 0, 13),

                new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                     5,

                                     0,

                                     10,

                                     0,

                                     13));

    }

    // THis is an extra-call but the corresponding events will not be added to EVB.

    // This is needed to flush the events in the above call to EVB

    scn = 15;

    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 4;

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

        new TransactionInfo(0, 0, timestamp, scn) });

    // Testing for SCN = 12 case

    testStats(gg,

              new EventStatsValues(sourceIds[0], 5, 0, 7, 0, 14),

              new EventStatsValues(sourceIds[1], 5, 0, 7, 0, 14),

              new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID,

                                   5,

                                   0,

                                   14,

                                   0,

                                   14));

  }

  private static void testStats(GoldenGateEventProducer gg, EventStatsValues ...eventStatsValues )

  {

    Map<Short, EventStatsValues> eventStatsValuesMap =

        new HashMap<Short, EventStatsValues>();

    for (EventStatsValues e : eventStatsValues)

    {

      eventStatsValuesMap.put(e.getSrcId(), e);

    }

    for (EventSourceStatisticsIface si : gg.getSources())

    {

      EventSourceStatistics ss = si.getStatisticsBean();

      LOG.info(si.getSourceName() + ": scn=" + ss.getMaxScn() + ",averageSize="

          + ss.getAvgEventSerializedSize() + ",numErrors=" + ss.getNumErrors()

          + ",totalEvents=" + ss.getNumTotalEvents() + ",averageFactTime="

          + ss.getAvgEventFactoryTimeMillisPerEvent() + ",timeSinceDb="

          + ss.getTimeSinceLastDBAccess());

      EventStatsValues e = eventStatsValuesMap.get(si.getSourceId());

      Assert.assertEquals(e.getAvgEventSize(), ss.getAvgEventSerializedSize());

      Assert.assertEquals(e.getNumErrors(), ss.getNumErrors());

      Assert.assertEquals(e.getNumTotalEvents(), ss.getNumTotalEvents());

      Assert.assertEquals(e.getAvgEventFactoryTimeMillisPerEvent(),

                          ss.getAvgEventFactoryTimeMillisPerEvent());

      Assert.assertEquals(e.getMaxScn(), ss.getMaxScn());

    }

  }

  private static class EventStatsValues

  {

    private final short srcId;

    private final long  avgEventSize;

    private final long  numErrors;

    private final long  numTotalEvents;

    private final long  avgEventFactoryTimeMillisPerEvent;

    private final long  maxScn;

    public EventStatsValues(short srcId,

                            long avgEventSize,

                            long numErrors,

                            long numTotalEvents,

                            long avgEventFactoryTimeMillisPerEvent,

                            long maxScn)

    {

      super();

      this.srcId = srcId;

      this.avgEventSize = avgEventSize;

      this.numErrors = numErrors;

      this.numTotalEvents = numTotalEvents;

      this.avgEventFactoryTimeMillisPerEvent = avgEventFactoryTimeMillisPerEvent;

      this.maxScn = maxScn;

    }

    public short getSrcId()

    {

      return srcId;

    }

    public long getAvgEventSize()

    {

      return avgEventSize;

    }

    public long getNumErrors()

    {

      return numErrors;

    }

    public long getNumTotalEvents()

    {

      return numTotalEvents;

    }

    public long getAvgEventFactoryTimeMillisPerEvent()

    {

      return avgEventFactoryTimeMillisPerEvent;

    }

    public long getMaxScn()

    {

      return maxScn;

    }

  }

  /**

   * test collection of parser stats, especially lag between parsed and added files

   * @throws Exception

   */

  @Test

  public void testGGParserStats() throws Exception

  {

    short[] sourceIds = new short[] { 505, 506 };

    String[] sourceNames = new String[] { "source1", "source2" };

    // setup trail Files directory

    File ggTrailDir = FileUtils.createTempDir("testGGParserStats");

    // configure physical source

    String uri = "gg://" + ggTrailDir.getAbsolutePath() + ":x3";

    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, uri);

    LOG.info("Uri=" + uri);

    // create schema

    Schema s = Schema.parse(sourceAvroSchema);

    VersionedSchema vs =

        new VersionedSchema(new VersionedSchemaId("source1", (short) 3), s, null);

    // mock for schema registry

    SchemaRegistryService srs = EasyMock.createMock(SchemaRegistryService.class);

    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source1"))

            .andReturn(vs)

            .anyTimes();

    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source2"))

            .andReturn(vs)

            .anyTimes();

    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);

    // mock for MaxSCNReadWriter

    MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);

    EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();

    mscn.saveMaxScn(EasyMock.anyLong());

    EasyMock.expectLastCall().anyTimes();

    EasyMock.replay(mscn);

    EasyMock.replay(srs);

    int totalTransWritten = 0;

    int totalFilesWritten = 0;

    // buffer

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start GG producer

    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, srs, mb, null, mscn);

    // create first 2 files

    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x301"), 100, 4);

    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x302"), 200, 4);

    totalTransWritten = 8;

    totalFilesWritten = 2;

    // get hold of parser stats object

    final GGParserStatistics ggParserStats = gg.getParserStats();

    // all should be 0

    Assert.assertEquals(0, ggParserStats.getNumFilesParsed());

    Assert.assertEquals(0, ggParserStats.getNumFilesAdded());

    Assert.assertEquals(0, ggParserStats.getFilesLag());

    Assert.assertEquals(0, ggParserStats.getTimeLag());

    Assert.assertEquals(0, ggParserStats.getBytesLag());

    try

    {

      LOG.info("starting event producer");

      gg.start(-2); // -2 here does nothing. actual setting happens thru the mock of

                    // MaxSCNReadWriter

      // let it parse first files

      TestUtil.assertWithBackoff(new ConditionCheck()

      {

        @Override

        public boolean check()

        {

          return ggParserStats.getNumFilesParsed() == 2

              && (8 * _transactionPatternSize == ggParserStats.getNumBytesTotalParsed());

        }

      }, "First two files parsed", 2000, LOG);

      // stats in the interim

      Assert.assertEquals(2, ggParserStats.getNumFilesParsed());

      Assert.assertEquals(2, ggParserStats.getNumFilesAdded());

      Assert.assertEquals(0, ggParserStats.getFilesLag());

      Assert.assertEquals(0, ggParserStats.getTimeLag());

      Assert.assertEquals(0, ggParserStats.getBytesLag());

      Assert.assertEquals(totalTransWritten * _transactionPatternSize,

                          ggParserStats.getNumBytesTotalParsed());

      gg.pause();

      // the file will get parsed but not processed

      addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x303"), 300, 4);

      totalTransWritten += 4;

      totalFilesWritten++;

      TestUtil.sleep(2000); // to get more then a ms lag time

      addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 400, 4);

      totalTransWritten += 4;

      totalFilesWritten++;

      TestUtil.sleep(6000); // to guarantee we picked up stats update (stats are updated

                            // every 5 seconds)

      // now we should be 2 files behind. parser thread gets paused AFTER it start

      // processing the file

      // so the actuall value will be 1 file behind

      int lagFiles = 1; // 303(already started being parsed), only 304 is behind

      long lagBytes = 1 * 4 * _transactionPatternSize; // 1 file, 4 transactions each

      /*

       * Assert.assertEquals(totalFilesWritten-1, ggParserStats.getNumFilesParsed());

       * Assert.assertEquals(totalFilesWritten, ggParserStats.getNumFilesAdded());

       * Assert.assertEquals(lagFiles, ggParserStats.getFilesLag()); // because 303 got

       * parsed

       *

       * // we added 4 files and parsed 3 , so the diff should be 1 file size (4

       * trasactions in 1 file) Assert.assertEquals(lagBytes,

       * ggParserStats.getBytesLag()); Assert.assertTrue(ggParserStats.getTimeLag()>0);

       */

      gg.unpause();

      TestUtil.sleep(5000);

      // now we should catchup

      Assert.assertEquals(4, ggParserStats.getNumFilesParsed());

      Assert.assertEquals(4, ggParserStats.getNumFilesAdded());

      Assert.assertEquals(0, ggParserStats.getFilesLag());

      Assert.assertEquals(0, ggParserStats.getTimeLag());

      Assert.assertEquals(0, ggParserStats.getBytesLag());

      // append to a file

      LOG.info("pausing again");

      gg.pause();

      addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 410, 4);

      totalTransWritten += 4;

      TestUtil.sleep(1000);

      addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 420, 4);

      totalTransWritten += 4;

      TestUtil.sleep(2000);

      gg.unpause();

      TestUtil.sleep(5500);

      // should be still up

      Assert.assertEquals(4, ggParserStats.getNumFilesParsed());

      Assert.assertEquals(4, ggParserStats.getNumFilesAdded());

      Assert.assertEquals(0, ggParserStats.getFilesLag());

      Assert.assertEquals(0, ggParserStats.getTimeLag());

      Assert.assertEquals(0, ggParserStats.getBytesLag());

      // assert the stats

      int totalFilesSize = totalTransWritten * _transactionPatternSize;

      Assert.assertEquals((totalFilesSize / totalFilesWritten),

                          ggParserStats.getAvgFileSize());

      Assert.assertEquals(true, ggParserStats.getAvgParseTransactionTimeNs() > 0);

      Assert.assertEquals("part1", ggParserStats.getPhysicalSourceName());

      Assert.assertEquals(totalFilesSize / totalTransWritten,

                          ggParserStats.getAvgTransactionSize());

      Assert.assertEquals(423, ggParserStats.getMaxScn());

      Assert.assertEquals(totalTransWritten * 2, ggParserStats.getNumTotalEvents()); // 2

                                                                                     // events

                                                                                     // per

                                                                                     // transaction

      Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsTotal());

      Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsWithEvents());

      Assert.assertEquals(0, ggParserStats.getNumTransactionsWithoutEvents());

      Assert.assertEquals(true, ggParserStats.getTimeSinceLastAccessMs() > 0);

      Assert.assertEquals(totalTransWritten * _transactionPatternSize,

                          ggParserStats.getNumBytesTotalParsed());

      Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());

      Assert.assertEquals("LastSCNRegressed", -1, ggParserStats.getLastRegressedScn());

    }

    finally

    {

      gg.shutdown();

    }

    return;

  }

  // testing add trail file

  public void testWriter() throws IOException {

    addToTrailFile(new File("/tmp/bbb"), 20, 3);

    addToTrailFile(new File("/tmp/bbb"), 30, 4);

  }

  private void addToTrailFile(File file, long startingSCN, int numTransactions) throws IOException {

    long lenBefore = file.length();

    BufferedWriter writer = new BufferedWriter( new FileWriter( file , true ) );

    for(int i=0; i<numTransactions; i++) {

      String tr = _transactionPattern.replaceAll(SCNPATTERN, String.valueOf(startingSCN++));

      writer.write(tr);

    }

    writer.close();

    Assert.assertEquals(_transactionPatternSize * numTransactions, file.length() - lenBefore);

  }

  private PhysicalSourceStaticConfig buildSimplePssc(short [] sourceIds, String [] sourceNames, String uri) throws InvalidConfigException

  {

    String partitionFunction = "constant:1";

    String pPartName = "part1";

    short pPartId = 0;

    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);

    for(int i=0; i<sourceIds.length; i++) {

      LogicalSourceConfig lc = new LogicalSourceConfig();

      lc.setId(sourceIds[i]);

      lc.setName(sourceNames[i]);

      lc.setPartitionFunction(partitionFunction);

      lc.setUri(pPartName + "." + sourceNames[i]); // this format is expected by GG

      pc.addSource(lc);

    }

    return pc.build();

  }

  @Test

  public void testAddEventToBuffer() throws InvalidConfigException,

      UnsupportedKeyException, DatabusException

  {

    // No rate control

    long rate = 0;

    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);

    long scn = 10;

    DbusEventBuffer mb = (DbusEventBuffer)createBufMult(pssc);

    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb,

        null, null);

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(

        10);

    int sourceId = 505;

    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();

    Object key = new String("name");

    Schema.Type keyType = Schema.Type.RECORD;

    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);

    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(

        1);

    keyPairs.add(kp);

    Schema s = Schema.parse(avroSchema);

    GenericRecord gr = new GenericData.Record(s);

    gr.put("name", "phani");

    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,

        DbUpdateState.DBUpdateImage.OpType.INSERT,false);

    db.add(dbi);

    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(

        sourceId, db);

    dbUpdates.add(dbUpdate);

    long timestamp = System.nanoTime();

    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));

    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);

    DbusEventIterator iter  = mb.acquireIterator("test");

    int count = 0;

    long eventTs = 0;

    while(iter.hasNext()) {

      DbusEvent e = iter.next();

      if(count==1) { // first event prev control event

        eventTs = e.timestampInNanos();

      }

      count ++;

    }

     Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);

    Assert.assertEquals("Got events " , 3, count);

    return;

  }

  @Test

  public void testGGProducerStats() throws InvalidConfigException,

          UnsupportedKeyException, DatabusException, IllegalArgumentException,

          InstantiationException, IllegalAccessException, InvocationTargetException, SecurityException, NoSuchMethodException

  {

    short[] sourceIds = new short[] { 505, 506 };

    String[] sourceNames = new String[] { "source1", "source2" };

    PhysicalSourceStaticConfig pssc =

        buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");

    long scn = 10;

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start producer

    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);

    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);

    Method method = getOnTransactionEndMethod();

    // generates the updates

    List<String> keys = new ArrayList<String>();

    keys.add("keys1");

    // SCN = 10

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates =

        generateUpdates(new short[] { sourceIds[0] }, keys, scn);

    // SCN = 11

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 =

        generateUpdates(new short[] { sourceIds[1] }, keys, scn + 1);

    // SCN = 12

    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 =

        generateUpdates(new short[] { sourceIds[1] }, keys, scn + 2);

    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;

    // gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));

    // gg.addEventToBuffer(dbUpdates1, new TransactionInfo(0, 0, timestamp+1, scn+1));

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates,

        new TransactionInfo(0, 0, timestamp, scn) });

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1,

        new TransactionInfo(0, 0, timestamp + 1, scn + 1) });

    // THis is an extra-call but the corresponding events will not be added to EVB.

    // This is needed to flush the events in the above call to EVB

    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2,

        new TransactionInfo(0, 0, timestamp + 1, scn + 2) });

    for (EventSourceStatisticsIface si : gg.getSources())

    {

      EventSourceStatistics ss = si.getStatisticsBean();

      LOG.info(si.getSourceName() + ": scn=" + ss.getMaxScn() + ",averageSize="

          + ss.getAvgEventSerializedSize() + ",numErrors=" + ss.getNumErrors()

          + ",totalEvents=" + ss.getNumTotalEvents() + ",averageFactTime="

          + ss.getAvgEventFactoryTimeMillisPerEvent() + ",timeSinceDb="

          + ss.getTimeSinceLastDBAccess());

      if (si.getSourceId() == 505)

      {

        Assert.assertEquals(6, ss.getAvgEventSerializedSize());

        Assert.assertEquals(0, ss.getNumErrors());

        Assert.assertEquals(1, ss.getNumTotalEvents());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent()); // we are not

                                                                           // really

                                                                           // reading

        Assert.assertEquals(10, ss.getMaxScn());

      }

      if (si.getSourceId() == 506)

      {

        Assert.assertEquals(6, ss.getAvgEventSerializedSize());

        Assert.assertEquals(0, ss.getNumErrors());

        Assert.assertEquals(1, ss.getNumTotalEvents());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent()); // we are not

                                                                           // really

                                                                           // reading

        Assert.assertEquals(11, ss.getMaxScn());

      }

      if (si.getSourceId() == GoldenGateEventProducer.GLOBAL_SOURCE_ID)

      {

        Assert.assertEquals(6, ss.getAvgEventSerializedSize());

        Assert.assertEquals(0, ss.getNumErrors());

        Assert.assertEquals(2, ss.getNumTotalEvents());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());

        Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent()); // we are not

                                                                           // really

                                                                           // reading

        Assert.assertEquals(11, ss.getMaxScn());

      }

    }

    long approximateTimeSinceLastTransactionMs =

        System.currentTimeMillis() - timestamp / DbusConstants.NUM_NSECS_IN_MSEC;

    long diff =

        gg.getParserStats().getTimeSinceLastTransactionMs()

            - approximateTimeSinceLastTransactionMs;

    Assert.assertTrue("time diff is too big:" + diff, diff >= 0 && diff < 30); // somewhat

                                                                               // close

                                                                               // and IN

                                                                               // MS (not

                                                                               // NS)

    return;

  }

  /**

   *

   * Helper Method to instantiate a private inner class "HandleXMLCallback" which is actually

   * present under another private inner class called "WorkerThread"

   */

  private Object getHandleXmlCallbackInnerInstance(GoldenGateEventProducer gg)