Examples of com.linkedin.databus2.core.BackoffTimerStaticConfigBuilder

com.linkedin.databus2.relay.config.LogicalSourceConfig
A builder for error retries static configuration

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     307 /* corrupt last SCN in 2nd file */, "plugh", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 307 occurs after SCN 299, so latter should be found OK:
    res = posSetter.locateFilePosition(299, finder);
    assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);


    // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK
    finder.reset();
    res = posSetter.locateFilePosition(306, finder);
    assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines and 2 SCNs each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     399 /* corrupt last SCN in 3rd file */, "quux", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     377 /* corrupt a txn in 3rd file */, "", true, "kilroy");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // with a full XML parser, this should throw an exception; with regexQuery(), it's fine:
    finder.reset();
    res = posSetter.locateFilePosition(377, finder);
    assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     corruptedScns, "blargh", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 101 is before the earliest (valid) SCN present, so expect ERROR:
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's 'too old'.");


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(102, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                      "expected error for exact-match SCN that's 'too old'.");


    // expect first non-corrupted SCN here, not first "transaction SCN":
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 102, FilePositionResult.Status.FOUND);


    // 107 = max SCN of its transaction = "transaction SCN" => should get FOUND
    finder.reset();
    res = posSetter.locateFilePosition(107, finder);
    assertFilePositionResult(res, dir, 107, FilePositionResult.Status.FOUND);


    // 203 = last valid SCN in first file = max SCN of its transaction = "transaction SCN"
    // => should be FOUND
    finder.reset();
    res = posSetter.locateFilePosition(203, finder);
    assertFilePositionResult(res, dir, 203, FilePositionResult.Status.FOUND);


    // SCN 204 is invalid and is part of a transaction split across first/second files;
    // 209 = next "transaction SCN" and is near the top of the middle file
    finder.reset();
    res = posSetter.locateFilePosition(204, finder);
    assertFilePositionResult(res, dir, 209, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 250 is invalid (as is 251); expect 253 since max SCN of following transaction
    finder.reset();
    res = posSetter.locateFilePosition(250, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 251 is invalid (as is 250); expect 253 since max SCN of following transaction
    finder.reset();
    res = posSetter.locateFilePosition(251, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 252 is valid and present, but weird corner case => still EXACT_SCN_NOT_FOUND
    finder.reset();
    res = posSetter.locateFilePosition(252, finder);
    assertFilePositionResult(res, dir, 252, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 253 is valid and present and max SCN of its transaction => FOUND
    finder.reset();
    res = posSetter.locateFilePosition(253, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.FOUND);


    // SCN 309 is valid and present and max SCN of its transaction => FOUND (even though
    // split across second/third files, and following transaction is corrupted)
    finder.reset();
    res = posSetter.locateFilePosition(309, finder);
    assertFilePositionResult(res, dir, 309, FilePositionResult.Status.FOUND);


    // SCN 310 is invalid (as is 311); expect 313
    finder.reset();
    res = posSetter.locateFilePosition(310, finder);
    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 311 is invalid (as is 310); expect 313
    finder.reset();
    res = posSetter.locateFilePosition(311, finder);
    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 398 is invalid (as is 399) and is in last transaction of last file, but since
    // trail file is expected to continue growing (i.e., eventually to have a valid SCN
    // that's larger than the request), expect EXACT_SCN_NOT_FOUND rather than ERROR.  SCN
    // returned will be that of last valid transaction, i.e., 397.
    // [checks beginning of last valid transaction == 396/397 one at byte offset 35650]
    finder.reset();
    res = posSetter.locateFilePosition(398, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 405 is completely missing (would be after last transaction of last file); expect
    // same behavior as previous case
    finder.reset();
    res = posSetter.locateFilePosition(405, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // last valid transaction-SCN is 397
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     -1 /* no corruption */, "", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 99 is before the earliest SCN present in the trail files (100), so expect ERROR:
    res = posSetter.locateFilePosition(99, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's too old.");


    // "Transaction-SCN" semantics:  max SCN is 121 in this transaction and 119 in the
    // previous one, so those are the values used for most of the "is it found?" logic.
    // Since 120 falls between the two but doesn't equal either one, it's not considered
    // to be found exactly (even though it's actually present), but the state is still
    // valid, so processing can continue (i.e., it doesn't matter).
    finder.reset();
    res = posSetter.locateFilePosition(120, finder);
    assertFilePositionResult(res, dir, 120, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                          "expected error for exact-match SCN that's too old.");


    // Related, weird corner case:  USE_EARLIEST_SCN uses the min SCN in the oldest transaction,
    // so even though an exact match for 100 doesn't return FOUND, this does:
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 100, FilePositionResult.Status.FOUND);


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                                  "expected error for exact-match SCN that's too old.");


    log.info(DONE_STRING);

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,"\n",startLine, -1, "", false, "");
    log.info("Directory is: " + dir);


    TrailFilePositionSetter posSetter = null;
    //GoldenGateTransactionSCNFinder finder;
    GGXMLTrailTransactionFinder finder;


    //less than minScn
    for (long i = 0 ; i < beginFoundScn ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                          "Result Status for SCN: " + i + ", Result: " + res);
    }


    //Found Case
    for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      log.info("For scn (" + i + "):  the result is:  "  + res);
      if (i%2 == 0)
        assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
      else
        assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);
    }


    //Found Case
    for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      log.info("For scn (" + i + "):  the result is:  "  + res);
      assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
    }

View Full Code Here

                     startScn);
    log.info("Directory is: " + dir);


    TrailFilePositionSetter posSetter = null;
    // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);


    // SCN 100 is not found because this is the first SCN in the trail file.
    FilePositionResult res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(),

View Full Code Here

      createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,newLine,startLine, -1, "", false, "");
      log.info("Directory is: " + dir);


      TrailFilePositionSetter posSetter = null;
      //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
      GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();


      //less than minScn
      for (long i = 0 ; i < beginFoundScn ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        Logger log2 = posSetter._log;
        log2.setLevel(Level.INFO);
        log2.info("Created a TrailFilePositionSetter with suffix x3");
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();


        FilePositionResult res = posSetter.getFilePosition(i,finder);
        Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                            "Result Status for SCN: " + i + ", Result: " + res);
      }


      //Found Case
      for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();


        FilePositionResult res = posSetter.getFilePosition(i,finder);
        log.info("For scn (" + i + "):  the result is:  "  + res);
        if (i%2 == 0)
          assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        else
          assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);
      }


      //Found Case
      for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();


        FilePositionResult res = posSetter.getFilePosition(i,finder);
        log.info("For scn (" + i + "):  the result is:  "  + res);
        assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
      }

View Full Code Here

        createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,100, 200,j,newLine,0, -1, "", false, "");
        log.info("Directory is: " + dir);


        TrailFilePositionSetter posSetter = null;
        //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
        GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();


        //less than minScn
        log.info("less than MinScn case started !!");
        for (long i = 0 ; i < 100 ; i ++)
        {
          if ( true )break;
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();


          FilePositionResult res = posSetter.getFilePosition(i,finder);
          Assert.assertEquals(res.getStatus(),FilePositionResult.Status.ERROR,"Result Status");
        }


        log.info("less than MinScn case passed !!");




        //Found Case
        for (long i = 100 ; i < 300 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          finder = new GGXMLTrailTransactionFinder();


          log.info("SCN:  " + i);
          FilePositionResult res = posSetter.getFilePosition(i,finder);
          log.info("For scn (" + i + "):  the result is:  "  + res);
          if (i%2 == 0)
            assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
          else
            assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);


        }


        //Found Case
        FilePositionResult res = null;
        for (long i = 300 ; i < 320 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();


          res = posSetter.getFilePosition(i,finder);
          //log.info("For scn (" + i + "):  the result is:  "  + res);
          assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        }


        // USE Latest SCN (-1)
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();


        res = posSetter.getFilePosition(-1,finder);
        //log.info("For scn (" + i + "):  the result is:  "  + res);
        assertFilePositionResult(res,dir,299,FilePositionResult.Status.FOUND);
      }

View Full Code Here

    String pPartName = "part1";
    short pPartId = 0;


    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for(int i=0; i<sourceIds.length; i++) {
      LogicalSourceConfig lc = new LogicalSourceConfig();
      lc.setId(sourceIds[i]);
      lc.setName(sourceNames[i]);
      lc.setPartitionFunction(partitionFunction);
      lc.setUri(pPartName + "." + sourceNames[i]); // this format is expected by GG
      pc.addSource(lc);
    }




    return pc.build();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.linkedin.databus2.core.BackoffTimerStaticConfigBuilder

com.google.code.or.common.glossary.Row

com.linkedin.databus.bootstrap.server.BootstrapHttpServer

com.linkedin.databus.bootstrap.server.BootstrapRequestProcessor

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain

com.linkedin.databus.bootstrap.utils.BootstrapTableReader

com.linkedin.databus.client.BootstrapPullThread

com.linkedin.databus.client.consumer.TestInternalMetadata

com.linkedin.databus.client.DatabusHttpClientImpl

com.linkedin.databus.client.DatabusSourcesConnection

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.