Examples of com.linkedin.databus2.relay.GoldenGateEventProducer

com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder
The event producer implementation for the xml trail file based event producer. The producer controls the xmlparser (start, pause, shutdown etc.)

              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
      // Get all relevant pConfig attributes
      //TODO add real instantiation
      EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
      if (null == mockProvider)
      {
        throw new DatabusRuntimeException("relay event producer not available: " + "mock");
      }
      producer = mockProvider.createProducer(pConfig, schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                     .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);
    } else if (uri.startsWith("gg:")){
      producer = new GoldenGateEventProducer(pConfig,
                                             schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                 .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);


    } else if (uri.startsWith("mysql:")){
       LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
       final String serviceName = "or";
       EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
       if (null == orProvider)
       {
         throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
       }
       producer = orProvider.createProducer(pConfig, schemaRegistryService,
                                            dbusEventBuffer,
                                            _inBoundStatsCollectors.getStatsCollector(statsCollectorName),
                                            maxScnReaderWriters);
    } else
     {

View Full Code Here

     {
      // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
      RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools =
            new RelayEventProducer.DatabusClientNettyThreadPools(0,getNetworkTimeoutTimer(),getBossExecutorService(),
                                        getIoExecutorService(), getHttpChannelGroup());
      producer = new RelayEventProducer(pConfig, dbusEventBuffer,
          _inBoundStatsCollectors
              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters,nettyThreadPools);
    }

View Full Code Here

    log.info("starting");


    File dir = createTempDir();


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();


    // Now delete the directory to make the locateFilePosition() see null for listFiles() call.
    boolean deleted = dir.delete();
    Assert.assertTrue(deleted, "Deleted the trail directory successfully");

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     100 /* corrupt first SCN */, "xyzzy", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 100 is corrupted, so 101 is the effective oldest SCN => 100 treated as error:
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's corrupted and oldest in all trail files.");


    // SCN 101 is OK (regexQuery() doesn't fully validate XML):
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 101, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     307 /* corrupt last SCN in 2nd file */, "plugh", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 307 occurs after SCN 299, so latter should be found OK:
    res = posSetter.locateFilePosition(299, finder);
    assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);


    // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK
    finder.reset();
    res = posSetter.locateFilePosition(306, finder);
    assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines and 2 SCNs each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     399 /* corrupt last SCN in 3rd file */, "quux", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     377 /* corrupt a txn in 3rd file */, "", true, "kilroy");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // with a full XML parser, this should throw an exception; with regexQuery(), it's fine:
    finder.reset();
    res = posSetter.locateFilePosition(377, finder);
    assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     corruptedScns, "blargh", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 101 is before the earliest (valid) SCN present, so expect ERROR:
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's 'too old'.");


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(102, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                      "expected error for exact-match SCN that's 'too old'.");


    // expect first non-corrupted SCN here, not first "transaction SCN":
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 102, FilePositionResult.Status.FOUND);


    // 107 = max SCN of its transaction = "transaction SCN" => should get FOUND
    finder.reset();
    res = posSetter.locateFilePosition(107, finder);
    assertFilePositionResult(res, dir, 107, FilePositionResult.Status.FOUND);


    // 203 = last valid SCN in first file = max SCN of its transaction = "transaction SCN"
    // => should be FOUND
    finder.reset();
    res = posSetter.locateFilePosition(203, finder);
    assertFilePositionResult(res, dir, 203, FilePositionResult.Status.FOUND);


    // SCN 204 is invalid and is part of a transaction split across first/second files;
    // 209 = next "transaction SCN" and is near the top of the middle file
    finder.reset();
    res = posSetter.locateFilePosition(204, finder);
    assertFilePositionResult(res, dir, 209, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 250 is invalid (as is 251); expect 253 since max SCN of following transaction
    finder.reset();
    res = posSetter.locateFilePosition(250, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 251 is invalid (as is 250); expect 253 since max SCN of following transaction
    finder.reset();
    res = posSetter.locateFilePosition(251, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 252 is valid and present, but weird corner case => still EXACT_SCN_NOT_FOUND
    finder.reset();
    res = posSetter.locateFilePosition(252, finder);
    assertFilePositionResult(res, dir, 252, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 253 is valid and present and max SCN of its transaction => FOUND
    finder.reset();
    res = posSetter.locateFilePosition(253, finder);
    assertFilePositionResult(res, dir, 253, FilePositionResult.Status.FOUND);


    // SCN 309 is valid and present and max SCN of its transaction => FOUND (even though
    // split across second/third files, and following transaction is corrupted)
    finder.reset();
    res = posSetter.locateFilePosition(309, finder);
    assertFilePositionResult(res, dir, 309, FilePositionResult.Status.FOUND);


    // SCN 310 is invalid (as is 311); expect 313
    finder.reset();
    res = posSetter.locateFilePosition(310, finder);
    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 311 is invalid (as is 310); expect 313
    finder.reset();
    res = posSetter.locateFilePosition(311, finder);
    assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 398 is invalid (as is 399) and is in last transaction of last file, but since
    // trail file is expected to continue growing (i.e., eventually to have a valid SCN
    // that's larger than the request), expect EXACT_SCN_NOT_FOUND rather than ERROR.  SCN
    // returned will be that of last valid transaction, i.e., 397.
    // [checks beginning of last valid transaction == 396/397 one at byte offset 35650]
    finder.reset();
    res = posSetter.locateFilePosition(398, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // SCN 405 is completely missing (would be after last transaction of last file); expect
    // same behavior as previous case
    finder.reset();
    res = posSetter.locateFilePosition(405, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // last valid transaction-SCN is 397
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 397, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     -1 /* no corruption */, "", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 99 is before the earliest SCN present in the trail files (100), so expect ERROR:
    res = posSetter.locateFilePosition(99, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's too old.");


    // "Transaction-SCN" semantics:  max SCN is 121 in this transaction and 119 in the
    // previous one, so those are the values used for most of the "is it found?" logic.
    // Since 120 falls between the two but doesn't equal either one, it's not considered
    // to be found exactly (even though it's actually present), but the state is still
    // valid, so processing can continue (i.e., it doesn't matter).
    finder.reset();
    res = posSetter.locateFilePosition(120, finder);
    assertFilePositionResult(res, dir, 120, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                          "expected error for exact-match SCN that's too old.");


    // Related, weird corner case:  USE_EARLIEST_SCN uses the min SCN in the oldest transaction,
    // so even though an exact match for 100 doesn't return FOUND, this does:
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 100, FilePositionResult.Status.FOUND);


    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                                  "expected error for exact-match SCN that's too old.");


    log.info(DONE_STRING);

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,"\n",startLine, -1, "", false, "");
    log.info("Directory is: " + dir);


    TrailFilePositionSetter posSetter = null;
    //GoldenGateTransactionSCNFinder finder;
    GGXMLTrailTransactionFinder finder;


    //less than minScn
    for (long i = 0 ; i < beginFoundScn ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                          "Result Status for SCN: " + i + ", Result: " + res);
    }


    //Found Case
    for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      log.info("For scn (" + i + "):  the result is:  "  + res);
      if (i%2 == 0)
        assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
      else
        assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);
    }


    //Found Case
    for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)
    {
      posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
      //finder = new GoldenGateTransactionSCNFinder();
      finder = new GGXMLTrailTransactionFinder();


      FilePositionResult res = posSetter.locateFilePosition(i,finder);
      log.info("For scn (" + i + "):  the result is:  "  + res);
      assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.linkedin.databus2.relay.GoldenGateEventProducer

com.google.code.or.common.glossary.Row

com.linkedin.databus.bootstrap.server.BootstrapHttpServer

com.linkedin.databus.bootstrap.server.BootstrapRequestProcessor

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain

com.linkedin.databus.bootstrap.utils.BootstrapTableReader

com.linkedin.databus.client.BootstrapPullThread

com.linkedin.databus.client.consumer.TestInternalMetadata

com.linkedin.databus.client.DatabusHttpClientImpl

com.linkedin.databus.client.DatabusSourcesConnection

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.