* Copyright 2013 LinkedIn Corp. All rights reserved

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*/

  public static final String MODULE = TestTrailFilePositionSetter.class.getName();

  public static final Logger LOG = Logger.getLogger(MODULE);

  private static final String TRAIL_FILENAME_PREFIX = "x3";

  private static final String[] _txnPattern =

    {

      "<transaction timestamp=\"2013-03-09:02:54:34.000000\">",

      " <dbupdate table=\"TASKMGR.TASKCTL_JOBS_1\" type=\"update\">",

      "   <columns>",

      "     <column name=\"JOB_ID\" key=\"true\">1621745</column>",

      "     <column name=\"GG_MODI_TS\">2013-03-09:02:54:33.996072000</column>",

      "     <column name=\"GG_STATUS\">o</column>",

      "   </columns>",

      "   <tokens>",

      "     <token name=\"TK-XID\">42.8.2681282</token>",

      "     <token name=\"TK-CSN\">${SCN}</token>",

      "   </tokens>",

      " </dbupdate>",

      " <dbupdate table=\"TASKMGR.TASKCTL_JOBS_2\" type=\"update\">",

      "   <columns>",

      "     <column name=\"JOB_ID\" key=\"true\">1621745</column>",

      "     <column name=\"GG_MODI_TS\">2013-03-09:02:54:33.996072000</column>",

      "     <column name=\"GG_STATUS\">o</column>",

      "   </columns>",

      "   <tokens>",

      "     <token name=\"TK-XID\">42.8.2681283</token>",

      "     <token name=\"TK-CSN\">${SCN}</token>",

      "   </tokens>",

      " </dbupdate>",

      "</transaction>"

   }; // currently 24 lines; if startLine > 0, probably want it less than this

  private static final String DONE_STRING =

      "done\n----------------------------------------------------------------------------\n";

  private ArrayList<File> _dirsToDelete = new ArrayList<File>(100);

  private BufferedWriter createWriter(String dir, String file) throws IOException

  {

    BufferedWriter w = new BufferedWriter(new FileWriter(new File(dir + "/" + file)));

    return w;

  }

  private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile,

                                long numLinesPerNewline, String newlineChar, int startLine,

                                long corruptedScn, String corruption,

                                boolean addAlternateCorruption, String altCorruption)

    throws IOException

  {

    HashSet<Long> corruptedScns = new HashSet<Long>(1);

    corruptedScns.add(new Long(corruptedScn));

    createTrailFiles(dir, prefix, numTxns, numLinesPerFile, numLinesPerNewline, newlineChar,

                     startLine, corruptedScns, corruption, addAlternateCorruption, altCorruption);

  }

  private void createTrailFiles(String dir,

                                String prefix,

                                long numTxns,

                                long numLinesPerFile,

                                long numLinesPerNewline,

                                String newlineChar,

                                int startLine,

                                Set<Long> corruptedScns,

                                String corruption,

                                boolean addAlternateCorruption,

                                String altCorruption) throws IOException

  {

    createTrailFiles(dir,

                     prefix,

                     numTxns,

                     numLinesPerFile,

                     numLinesPerNewline,

                     newlineChar,

                     startLine,

                     corruptedScns,

                     corruption,

                     addAlternateCorruption,

                     altCorruption,

                     _txnPattern,

                     1,

                     100);

  }

  private void createTrailFiles(String dir,

                                String prefix,

                                long numTxns,

                                long numLinesPerFile,

                                long numLinesPerNewline,

                                String newlineChar,

                                int startLine,

                                String[] txnPattern,

                                int numDbUpdatesWithSameScn,

                                long startScn) throws IOException

  {

    createTrailFiles(dir,

                     prefix,

                     numTxns,

                     numLinesPerFile,

                     numLinesPerNewline,

                     newlineChar,

                     startLine,

                     new HashSet<Long>(),

                     "",

                     false,

                     "",

                     txnPattern,

                     numDbUpdatesWithSameScn,

                     startScn);

  }

  private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile,

          long numLinesPerNewline, String newlineChar, int startLine,

          Set<Long> corruptedScns, String corruption,

          boolean addAlternateCorruption, String altCorruption, String[] txnPattern, int numDbUpdatesWithSameScn, long currScn)

      throws IOException

{

    long numFiles = ((numTxns * (txnPattern.length))/numLinesPerFile) + 1;

    long numDigits = new Double(Math.log10(numFiles)).longValue() + 1;

    long currFileNum = 0;

    String currFile = prefix + toFixedLengthString(currFileNum, numDigits);

    long lineCount = 0;

    BufferedWriter w = createWriter(dir, currFile);

    int start = startLine;

    int dbUpdates = 0;

    for (long txn = 0; txn < numTxns; ++txn)

    {

      boolean corruptNextTokensEndTag = false;

      if (txn > 0)

        start = 0;

      for (int j = 0 ; j < txnPattern.length; ++j)

      {

        lineCount++;

        String txnLine = txnPattern[j];

        if (txnLine.contains("${SCN}"))

        {

          dbUpdates++;

          txnLine = txnLine.replace("${SCN}", new Long(currScn).toString() + (corruptedScns.contains(currScn)? corruption : ""));

          if (addAlternateCorruption && corruptedScns.contains(currScn))

            corruptNextTokensEndTag = true;

          if (dbUpdates >= numDbUpdatesWithSameScn)

          {

            currScn++;

            dbUpdates = 0;

          }

        }

        if (corruptNextTokensEndTag && txnLine.contains("</tokens>"))

        {

          //txnLine = txnLine.append(newlineChar).append("   ").append(altCorruption);

          txnLine = txnLine + newlineChar + "   " + altCorruption;

          corruptNextTokensEndTag = false;

        }

        if (j >= start)

        {

          w.append(txnLine);

          if (lineCount%numLinesPerNewline == 0)

            w.append(newlineChar);

        }

        if ( (lineCount%numLinesPerFile) == 0)

        {

          w.close();

          currFileNum++;

          currFile = prefix + toFixedLengthString(currFileNum, numDigits);

          w = createWriter(dir, currFile);

        }

      }

    }

    if ( w != null)

      w.close();

  }

  private static String toFixedLengthString(Long num, long numDigits)

  {

    String n = num.toString();

    long numDigitsToBeZeroed = numDigits - n.length();

    StringBuilder bld = new StringBuilder();

    for ( int i = 0; i < numDigitsToBeZeroed; i++)

      bld.append('0');

    bld.append(n);

    return bld.toString();

  }

  @BeforeClass

  public void setUpClass() throws InvalidConfigException

  {

    //set up logging

    // TODO Setting the first argument to 'true' and make DDSDBUS-3577 not happen

    TestUtil.setupLoggingWithTimestampedFile(false, "/tmp/TestTrailFilePositionSetter_", ".log", Level.INFO);  // FATAL

  }

  @AfterClass  // optionally could do this after every test instead

  public void tearDownClass()

  {

    for (File dir : _dirsToDelete)

    {

      FileUtils.deleteQuietly(dir);

    }

    _dirsToDelete.clear();

  }

  private File createTempDir()

      throws IOException

  {

    File dir = File.createTempFile("dir_", null);

    if (!(dir.delete()))

      throw new IOException("Unable to delete temp file " + dir.getAbsolutePath());

    if (!dir.mkdir())

      throw new IOException("Unable to create tempDir: " + dir.getAbsolutePath());

    _dirsToDelete.add(dir);  // deleteOnExit() deletes only if dir empty => use Apache FileUtils helper instead

    return dir;

  }

  /**

   * DDSDBUS-2603 NUllPointerException in GG Relay in Experimental CLuster

   * @throws Exception

   */

  @Test

  public void testDeletedDirectory()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testDeletedDirectory");

    log.info("starting");

    File dir = createTempDir();

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    // Now delete the directory to make the locateFilePosition() see null for listFiles() call.

    boolean deleted = dir.delete();

    Assert.assertTrue(deleted, "Deleted the trail directory successfully");

    FilePositionResult res = posSetter.locateFilePosition(100,finder);

    Assert.assertEquals(res.getStatus(), Status.NO_TXNS_FOUND, "File Position Result Status");

    log.info(DONE_STRING);

  }

  /**

   * Verify that corruption that occurs early in the file causes USE_EARLIEST_SCN to return the

   * first SCN of the first uncorrupted transaction.

   */

  @Test

  public void testFailureMode_MalformedFirstTransactionInFirstFile()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedFirstTransactionInFirstFile");

    log.info("starting");

    File dir = createTempDir();

    // first SCN is hardcoded always to be 100

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     100 /* corrupt first SCN */, "xyzzy", false, "");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // SCN 100 is corrupted, so 101 is the effective oldest SCN => 100 treated as error:

    res = posSetter.locateFilePosition(100, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                        "expected error for exact-match SCN that's corrupted and oldest in all trail files.");

    // SCN 101 is OK (regexQuery() doesn't fully validate XML):

    finder.reset();

    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);

    assertFilePositionResult(res, dir, 101, FilePositionResult.Status.FOUND);

    log.info(DONE_STRING);

  }

  /**

   * Verify that corruption that occurs after the desired SCN/transaction doesn't cause errors

   * when requesting an exact SCN.

   */

  @Test

  public void testFailureMode_MalformedLastTransactionInMiddleFile()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInMiddleFile");

    log.info("starting");

    File dir = createTempDir();

    // first SCN is hardcoded always to be 100

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     307 /* corrupt last SCN in 2nd file */, "plugh", false, "");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // corruption at SCN 307 occurs after SCN 299, so latter should be found OK:

    res = posSetter.locateFilePosition(299, finder);

    assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);

    // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK

    finder.reset();

    res = posSetter.locateFilePosition(306, finder);

    assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND);

    log.info(DONE_STRING);

  }

  /**

   * Verify that corruption in the very last transaction causes USE_LATEST_SCN to return

   * the last SCN from the previous transaction.

   */

  @Test

  public void testFailureMode_MalformedLastTransactionInLastFile()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInLastFile");

    log.info("starting");

    File dir = createTempDir();

    // first SCN is hardcoded always to be 100

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines and 2 SCNs each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     399 /* corrupt last SCN in 3rd file */, "quux", false, "");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)

    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);

    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);

    log.info(DONE_STRING);

  }

  /**

   * Verify that a non-corrupted SCN value within a corrupted transaction causes an exception to be thrown

   * when the exact SCN is requested.

   */

  @Test

  public void testFailureMode_MalformedTransactionForExactScn()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionForExactScn");

    log.info("starting");

    File dir = createTempDir();

    // first SCN is hardcoded always to be 100

    // this adds a non-XML string near the end of the dbupdate for SCN 377, but the SCN value itself is clean:

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     377 /* corrupt a txn in 3rd file */, "", true, "kilroy");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // with a full XML parser, this should throw an exception; with regexQuery(), it's fine:

    finder.reset();

    res = posSetter.locateFilePosition(377, finder);

    assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND);

    log.info(DONE_STRING);

  }

  /**

   * Verify that a transaction with corruption in all of its SCNs doesn't cause problems.

   * Case 1:  bad transaction at the beginning of the first trail file.

   * Case 2:  bad transaction at the beginning of the middle trail file (partial transaction).

   * Case 3:  bad transaction at the beginning of the middle trail file (first full transaction; preceding partial bad).

   * Case 4:  bad transaction in the middle of the middle trail file.

   * Case 5:  bad transaction at the beginning of the last trail file (first full transaction; preceding partial is OK).

   * Case 6:  bad transaction at the end of the last trail file.

   */

  @Test

  public void testFailureMode_MalformedTransactionNoValidScnsInMiddleOfMiddleFile()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionNoValidScns");

    log.info("starting");

    File dir = createTempDir();

    // corrupt both SCNs in each of six transactions:

    HashSet<Long> corruptedScns = new HashSet<Long>(10);

    corruptedScns.add(new Long(100));  // case 1

    corruptedScns.add(new Long(101));  // case 1

    corruptedScns.add(new Long(204));  // case 2

    corruptedScns.add(new Long(205));  // case 2

    corruptedScns.add(new Long(206));  // case 3

    corruptedScns.add(new Long(207));  // case 3

    corruptedScns.add(new Long(250));  // case 4

    corruptedScns.add(new Long(251));  // case 4

    corruptedScns.add(new Long(310));  // case 5

    corruptedScns.add(new Long(311));  // case 5

    corruptedScns.add(new Long(398));  // case 6

    corruptedScns.add(new Long(399));  // case 6

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     corruptedScns, "blargh", false, "");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // SCN 101 is before the earliest (valid) SCN present, so expect ERROR:

    res = posSetter.locateFilePosition(101, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                        "expected error for exact-match SCN that's 'too old'.");

    // For SCN <= the earliest transactions maxSCN, we throw error

    finder.reset();

    res = posSetter.locateFilePosition(102, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                      "expected error for exact-match SCN that's 'too old'.");

    // expect first non-corrupted SCN here, not first "transaction SCN":

    finder.reset();

    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);

    assertFilePositionResult(res, dir, 102, FilePositionResult.Status.FOUND);

    // 107 = max SCN of its transaction = "transaction SCN" => should get FOUND

    finder.reset();

    res = posSetter.locateFilePosition(107, finder);

    assertFilePositionResult(res, dir, 107, FilePositionResult.Status.FOUND);

    // 203 = last valid SCN in first file = max SCN of its transaction = "transaction SCN"

    // => should be FOUND

    finder.reset();

    res = posSetter.locateFilePosition(203, finder);

    assertFilePositionResult(res, dir, 203, FilePositionResult.Status.FOUND);

    // SCN 204 is invalid and is part of a transaction split across first/second files;

    // 209 = next "transaction SCN" and is near the top of the middle file

    finder.reset();

    res = posSetter.locateFilePosition(204, finder);

    assertFilePositionResult(res, dir, 209, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 250 is invalid (as is 251); expect 253 since max SCN of following transaction

    finder.reset();

    res = posSetter.locateFilePosition(250, finder);

    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 251 is invalid (as is 250); expect 253 since max SCN of following transaction

    finder.reset();

    res = posSetter.locateFilePosition(251, finder);

    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 252 is valid and present, but weird corner case => still EXACT_SCN_NOT_FOUND

    finder.reset();

    res = posSetter.locateFilePosition(252, finder);

    assertFilePositionResult(res, dir, 252, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 253 is valid and present and max SCN of its transaction => FOUND

    finder.reset();

    res = posSetter.locateFilePosition(253, finder);

    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.FOUND);

    // SCN 309 is valid and present and max SCN of its transaction => FOUND (even though

    // split across second/third files, and following transaction is corrupted)

    finder.reset();

    res = posSetter.locateFilePosition(309, finder);

    assertFilePositionResult(res, dir, 309, FilePositionResult.Status.FOUND);

    // SCN 310 is invalid (as is 311); expect 313

    finder.reset();

    res = posSetter.locateFilePosition(310, finder);

    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 311 is invalid (as is 310); expect 313

    finder.reset();

    res = posSetter.locateFilePosition(311, finder);

    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 398 is invalid (as is 399) and is in last transaction of last file, but since

    // trail file is expected to continue growing (i.e., eventually to have a valid SCN

    // that's larger than the request), expect EXACT_SCN_NOT_FOUND rather than ERROR.  SCN

    // returned will be that of last valid transaction, i.e., 397.

    // [checks beginning of last valid transaction == 396/397 one at byte offset 35650]

    finder.reset();

    res = posSetter.locateFilePosition(398, finder);

    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // SCN 405 is completely missing (would be after last transaction of last file); expect

    // same behavior as previous case

    finder.reset();

    res = posSetter.locateFilePosition(405, finder);

    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // last valid transaction-SCN is 397

    finder.reset();

    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);

    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.FOUND);

    log.info(DONE_STRING);

  }

  /**

   * Verify that requesting an SCN that's older than the oldest trail file causes ERROR

   * to be returned.  Also check various valid states:  SCN that isn't the max for its

   * transaction; SCN that is the max for its transaction; earliest SCN; etc.

   */

  @Test

  public void testRequestedScnOlderThanOldestTransaction()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRequestedScnOlderThanOldestTransaction");

    //log.setLevel(Level.DEBUG);

    log.info("starting");

    File dir = createTempDir();

    // first SCN is hardcoded always to be 100:

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,

                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,

                     -1 /* no corruption */, "", false, "");

    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    FilePositionResult res;

    // SCN 99 is before the earliest SCN present in the trail files (100), so expect ERROR:

    res = posSetter.locateFilePosition(99, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                        "expected error for exact-match SCN that's too old.");

    // "Transaction-SCN" semantics:  max SCN is 121 in this transaction and 119 in the

    // previous one, so those are the values used for most of the "is it found?" logic.

    // Since 120 falls between the two but doesn't equal either one, it's not considered

    // to be found exactly (even though it's actually present), but the state is still

    // valid, so processing can continue (i.e., it doesn't matter).

    finder.reset();

    res = posSetter.locateFilePosition(120, finder);

    assertFilePositionResult(res, dir, 120, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    // For SCN <= the earliest transactions maxSCN, we throw error

    finder.reset();

    res = posSetter.locateFilePosition(100, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                          "expected error for exact-match SCN that's too old.");

    // Related, weird corner case:  USE_EARLIEST_SCN uses the min SCN in the oldest transaction,

    // so even though an exact match for 100 doesn't return FOUND, this does:

    finder.reset();

    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);

    assertFilePositionResult(res, dir, 100, FilePositionResult.Status.FOUND);

    // For SCN <= the earliest transactions maxSCN, we throw error

    finder.reset();

    res = posSetter.locateFilePosition(101, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                                  "expected error for exact-match SCN that's too old.");

    log.info(DONE_STRING);

  }

  @Test

  public void testScnPositionLocator()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionLocator");

    log.info("starting");

    testScnPositionLocator(0,100,200,102, log); // case where all txn are complete

    log.info(DONE_STRING);

  }

  private void testScnPositionLocator(int startLine, int startScn, int endScn, int beginFoundScn, Logger log)

      throws Exception

  {

    File dir = createTempDir();

    // TODO/FIXME:  why are startScn and endScn being used for numTxns and numLinesPerFile, respectively?

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,"\n",startLine, -1, "", false, "");

    log.info("Directory is: " + dir);

    TrailFilePositionSetter posSetter = null;

    //GoldenGateTransactionSCNFinder finder;

    GGXMLTrailTransactionFinder finder;

    //less than minScn

    for (long i = 0 ; i < beginFoundScn ; i ++)

    {

      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

      //finder = new GoldenGateTransactionSCNFinder();

      finder = new GGXMLTrailTransactionFinder();

      FilePositionResult res = posSetter.locateFilePosition(i,finder);

      Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                          "Result Status for SCN: " + i + ", Result: " + res);

    }

    //Found Case

    for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)

    {

      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

      //finder = new GoldenGateTransactionSCNFinder();

      finder = new GGXMLTrailTransactionFinder();

      FilePositionResult res = posSetter.locateFilePosition(i,finder);

      log.info("For scn (" + i + "):  the result is:  "  + res);

      if (i%2 == 0)

        assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

      else

        assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);

    }

    //Found Case

    for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)

    {

      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

      //finder = new GoldenGateTransactionSCNFinder();

      finder = new GGXMLTrailTransactionFinder();

      FilePositionResult res = posSetter.locateFilePosition(i,finder);

      log.info("For scn (" + i + "):  the result is:  "  + res);

      assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

    }

  }

  @Test

  public void testRepeatSCN() throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRepeatSCN");

    File dir = createTempDir();

    long startScn = 100;

    int numTxnsPerFile = 10;

    /**

     * Create trail files with the following SCN range

     *

     * Trail-File       Number of Txns          SCN Range

     *   x300               10                  100-100

     *   x301               10                  100-100

     *   x302               10                  101-101

     *   x303               10                  101-101

     *   x304               10                  102-102

     *   x305               10                  102-102

     *   x306               10                  103-103

     *   x307               10                  103-103

     *   x308               10                  104-104

     *   x309               10                  104-104

     *

     */

    createTrailFiles(dir.getAbsolutePath(),

                     TRAIL_FILENAME_PREFIX,

                     100L,

                     _txnPattern.length * numTxnsPerFile,

                     1L,

                     "\n",

                     0,

                     _txnPattern,

                     2 * numTxnsPerFile * 2, // Transactions in 2 files have the same SCN

                     startScn);

    log.info("Directory is: " + dir);

    TrailFilePositionSetter posSetter = null;

    // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();

    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

    posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    // SCN 100 is not found because this is the first SCN in the trail file.

    FilePositionResult res = posSetter.locateFilePosition(100, finder);

    Assert.assertEquals(res.getStatus(),

                        FilePositionResult.Status.ERROR,

                        "Result Status for SCN: " + 100 + ", Result: " + res);

    // SCN 101 is found

    res = posSetter.locateFilePosition(101, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");

    Assert.assertEquals(res.getTxnPos().getFile(), "x302", "File found");

    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");

    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");

    Assert.assertEquals(res.getTxnPos().getMinScn(), 101, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getMaxScn(), 101, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");

    // SCN 102 is found

    res = posSetter.locateFilePosition(102, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");

    Assert.assertEquals(res.getTxnPos().getFile(), "x304", "File found");

    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");

    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");

    Assert.assertEquals(res.getTxnPos().getMinScn(), 102, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getMaxScn(), 102, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");

    // SCN 103 is found

    res = posSetter.locateFilePosition(103, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");

    Assert.assertEquals(res.getTxnPos().getFile(), "x306", "File found");

    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");

    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");

    Assert.assertEquals(res.getTxnPos().getMinScn(), 103, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getMaxScn(), 103, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");

    // SCN 104 is found

    res = posSetter.locateFilePosition(104, finder);

    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");

    Assert.assertEquals(res.getTxnPos().getFile(), "x308", "File found");

    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");

    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");

    Assert.assertEquals(res.getTxnPos().getMinScn(), 104, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getMaxScn(), 104, "MinScn check");

    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");

  }

  @Test

  public void testScnPositionSetter1()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter1");

    log.info("starting");

    testScnPositionSetter(0,100,200,100, log); // case where all txn are complete

    testScnPositionSetter(10,100,200,102, log); // case where initial txn is incomplete

    log.info(DONE_STRING);

  }

  private void testScnPositionSetter(int startLine, int startScn, int endScn, int beginFoundScn, Logger log)

    throws Exception

  {

    String[] newLines = { "\n", "\r", "\r\n" };

    for (String newLine : newLines)

    {

      File dir = createTempDir();

      createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,newLine,startLine, -1, "", false, "");

      log.info("Directory is: " + dir);

      TrailFilePositionSetter posSetter = null;

      //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();

      GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

      //less than minScn

      for (long i = 0 ; i < beginFoundScn ; i ++)

      {

        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

        Logger log2 = posSetter._log;

        log2.setLevel(Level.INFO);

        log2.info("Created a TrailFilePositionSetter with suffix x3");

        //finder = new GoldenGateTransactionSCNFinder();

        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);

        Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,

                            "Result Status for SCN: " + i + ", Result: " + res);

      }

      //Found Case

      for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)

      {

        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

        //finder = new GoldenGateTransactionSCNFinder();

        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);

        log.info("For scn (" + i + "):  the result is:  "  + res);

        if (i%2 == 0)

          assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

        else

          assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);

      }

      //Found Case

      for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)

      {

        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

        //finder = new GoldenGateTransactionSCNFinder();

        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);

        log.info("For scn (" + i + "):  the result is:  "  + res);

        assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

      }

    }

  }

  /**

   * Case where XML snippets have newlines at different positions.

   * NOTE:  This test is EXCEEDINGLY slow...

   *

   * @throws Exception

   */

  @Test

  public void testScnPositionSetter2()

    throws Exception

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter2");

    log.info("starting");

    String[] newLines = { "\r", "\n", "\r\n" };

    for (String newLine : newLines)

    {

      // Iterate by Number of Tags per line

      for (int j = 10; j < 32; j++ )

      {

        log.info("NumPerLine: " + j);

        File dir = createTempDir();

        createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,100, 200,j,newLine,0, -1, "", false, "");

        log.info("Directory is: " + dir);

        TrailFilePositionSetter posSetter = null;

        //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();

        GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

        //less than minScn

        log.info("less than MinScn case started !!");

        for (long i = 0 ; i < 100 ; i ++)

        {

          if ( true )break;

          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

          //finder = new GoldenGateTransactionSCNFinder();

          finder = new GGXMLTrailTransactionFinder();

          FilePositionResult res = posSetter.getFilePosition(i,finder);

          Assert.assertEquals(res.getStatus(),FilePositionResult.Status.ERROR,"Result Status");

        }

        log.info("less than MinScn case passed !!");

        //Found Case

        for (long i = 100 ; i < 300 ; i ++)

        {

          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

          finder = new GGXMLTrailTransactionFinder();

          log.info("SCN:  " + i);

          FilePositionResult res = posSetter.getFilePosition(i,finder);

          log.info("For scn (" + i + "):  the result is:  "  + res);

          if (i%2 == 0)

            assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

          else

            assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);

        }

        //Found Case

        FilePositionResult res = null;

        for (long i = 300 ; i < 320 ; i ++)

        {

          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

          //finder = new GoldenGateTransactionSCNFinder();

          finder = new GGXMLTrailTransactionFinder();

          res = posSetter.getFilePosition(i,finder);

          //log.info("For scn (" + i + "):  the result is:  "  + res);

          assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);

        }

        // USE Latest SCN (-1)

        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

        //finder = new GoldenGateTransactionSCNFinder();

        finder = new GGXMLTrailTransactionFinder();

        res = posSetter.getFilePosition(-1,finder);

        //log.info("For scn (" + i + "):  the result is:  "  + res);

        assertFilePositionResult(res,dir,299,FilePositionResult.Status.FOUND);

      }

    }

    log.info(DONE_STRING);

  }

  private void assertFilePositionResult(FilePositionResult res, File dir, long expScn,

                                        FilePositionResult.Status expStatus)

    throws Exception

  {

    Assert.assertEquals(res.getStatus(),expStatus,"Result Status for scn: " + expScn);

    ConcurrentAppendableCompositeFileInputStream c =

        new ConcurrentAppendableCompositeFileInputStream(dir.getAbsolutePath(),

                                                         res.getTxnPos().getFile(),

                                                         res.getTxnPos().getFileOffset(),

                                                         new TrailFilePositionSetter.FileFilter(dir, TRAIL_FILENAME_PREFIX),

                                                         true);

    ReadFirstSCN scnFetcher = new ReadFirstSCN(new BufferedReader(new InputStreamReader(c)),expScn);

    scnFetcher.start();

    scnFetcher.awaitShutdown();

    if(c != null)

      c.close();

    LOG.info("Result is: " + res);

    Assert.assertEquals(scnFetcher.isPatternMatched(), true, "expected SCN " + expScn + " not found at byte offset " +

                        res.getTxnPos().getFileOffset() + " of file " + res.getTxnPos().getFile() + ".");

    LOG.info("Finding result txn (" + res + ") passed !!");

  }

  public static class ReadFirstSCN

     extends DatabusThreadBase

  {

    private final BufferedReader _reader;

    private boolean _patternMatched = false;

    private String _errorMessage = null;

    private long _expScn = -1;

    public ReadFirstSCN(BufferedReader reader, long expScn)

    {

      super("ReadFirSCN");

      _reader = reader;

      _expScn = expScn;

    }

    @Override

    public void run()

    {

      boolean beginSeen = false;

      String line = null;

      try

      {

        String pattern = "<token name=\"tk-csn\">" + _expScn + "</token>";

        while ((line = _reader.readLine()) != null)

        {

         //LOG.info("Line:  (" + line + ")");

          line = line.toLowerCase();

          if ( !beginSeen)

          {

            beginSeen = true;

            if ( ! line.startsWith(GGXMLTrailTransactionFinder.TRANSACTION_BEGIN_PREFIX))

            {

              _errorMessage = "Stream not starting with transaction begin string !!  Line is: " + line;

              return;

            }

          }

          if ( line.contains(pattern) && line.contains(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX))

          {

            int index1 = line.indexOf(pattern);

            int index2 = line.indexOf(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX);

            if ( index1 < index2)

            {

              _patternMatched = true;

            }

            break;

          }

          if (line.contains(pattern))

            _patternMatched = true;

          if (line.contains(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX))

            break;

        }

      }

      catch (IOException e)

      {

        e.printStackTrace();

      } finally {

        if ( null != _reader)

          try

          {

            _reader.close();

          }

          catch (IOException e)

          {

            // TODO Auto-generated catch block

            e.printStackTrace();

          }

          doShutdownNotify();

      }

    }

    public boolean isPatternMatched()

    {

      return _patternMatched;

    }

  }

  @Test

  public void testSplitBytesByNewLines()

  {

    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testSplitBytesByNewLines");

    log.info("starting");

    String s = "abc\r\ndef\r\n";

    List<String> l = new ArrayList<String>();

    List<Integer> endPos = new ArrayList<Integer>();

    String lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(2,l.size(),"Length");

    Assert.assertEquals(endPos.size(),2,"Length");

    Assert.assertEquals(l.get(0),"abc","First in " + l);

    Assert.assertEquals(l.get(1),"def","Second in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 0, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(3,l.size(),"Length");

    Assert.assertEquals(endPos.size(),3,"Length");

    Assert.assertEquals(l.get(0),"","First in " + l);

    Assert.assertEquals(l.get(1),"abc","Second in " + l);

    Assert.assertEquals(l.get(2),"def","Third in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),-1,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(2).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 1, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(3,l.size(),"Length");

    Assert.assertEquals(endPos.size(),3,"Length");

    Assert.assertEquals(l.get(0),"a","First in " + l);

    Assert.assertEquals(l.get(1),"bc","Second in " + l);

    Assert.assertEquals(l.get(2),"def","Third in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),-1,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(2).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 2, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(3,l.size(),"Length");

    Assert.assertEquals(endPos.size(),3,"Length");

    Assert.assertEquals(l.get(0),"ab","First in " + l);

    Assert.assertEquals(l.get(1),"c","Second in " + l);

    Assert.assertEquals(l.get(2),"def","Third in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),-1,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(2).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 3, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(2,l.size(),"Length");

    Assert.assertEquals(endPos.size(),2,"Length");

    Assert.assertEquals(l.get(0),"abc","First in " + l);

    Assert.assertEquals(l.get(1),"def","Second in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 5, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(l.size(),3,"Length");

    Assert.assertEquals(endPos.size(),3,"Length");

    Assert.assertEquals(l.get(0),"abc","First in " + l);

    Assert.assertEquals(l.get(1),"","Second in " + l);

    Assert.assertEquals(l.get(2),"def","Third in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),-1,"First in " + endPos);

    Assert.assertEquals(endPos.get(2).intValue(),10,"Second in " + endPos);

    s = "abc\r\ndef\r\n";

    l = new ArrayList<String>();

    endPos = new ArrayList<Integer>();

    lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 6, null, l, endPos);

    Assert.assertNull(lastLineStr,"Returned String should be null");

    Assert.assertEquals(l.size(),3,"Length");

    Assert.assertEquals(endPos.size(),3,"Length");

    Assert.assertEquals(l.get(0),"abc","First in " + l);

    Assert.assertEquals(l.get(1),"d","Second in " + l);

    Assert.assertEquals(l.get(2),"ef","Third in " + l);

    Assert.assertEquals(endPos.get(0).intValue(),5,"First in " + endPos);

    Assert.assertEquals(endPos.get(1).intValue(),-1,"First in " + endPos);

    Assert.assertEquals(endPos.get(2).intValue(),10,"Second in " + endPos);