Package com.linkedin.databus2.relay.TestDatabusRelayMain

Examples of com.linkedin.databus2.relay.TestDatabusRelayMain.ClientRunner


                     startScn);
    log.info("Directory is: " + dir);

    TrailFilePositionSetter posSetter = null;
    // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);

    // SCN 100 is not found because this is the first SCN in the trail file.
    FilePositionResult res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(),
View Full Code Here


      createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,startScn, endScn,1,newLine,startLine, -1, "", false, "");
      log.info("Directory is: " + dir);

      TrailFilePositionSetter posSetter = null;
      //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
      GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

      //less than minScn
      for (long i = 0 ; i < beginFoundScn ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        Logger log2 = posSetter._log;
        log2.setLevel(Level.INFO);
        log2.info("Created a TrailFilePositionSetter with suffix x3");
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);
        Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                            "Result Status for SCN: " + i + ", Result: " + res);
      }

      //Found Case
      for (long i = beginFoundScn ; i < (startScn + endScn) ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);
        log.info("For scn (" + i + "):  the result is:  "  + res);
        if (i%2 == 0)
          assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        else
          assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);
      }

      //Found Case
      for (long i = (startScn + endScn) ; i < (startScn + endScn) + 20 ; i ++)
      {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();

        FilePositionResult res = posSetter.getFilePosition(i,finder);
        log.info("For scn (" + i + "):  the result is:  "  + res);
        assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
      }
View Full Code Here

        createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,100, 200,j,newLine,0, -1, "", false, "");
        log.info("Directory is: " + dir);

        TrailFilePositionSetter posSetter = null;
        //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
        GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

        //less than minScn
        log.info("less than MinScn case started !!");
        for (long i = 0 ; i < 100 ; i ++)
        {
          if ( true )break;
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();

          FilePositionResult res = posSetter.getFilePosition(i,finder);
          Assert.assertEquals(res.getStatus(),FilePositionResult.Status.ERROR,"Result Status");
        }

        log.info("less than MinScn case passed !!");


        //Found Case
        for (long i = 100 ; i < 300 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          finder = new GGXMLTrailTransactionFinder();

          log.info("SCN:  " + i);
          FilePositionResult res = posSetter.getFilePosition(i,finder);
          log.info("For scn (" + i + "):  the result is:  "  + res);
          if (i%2 == 0)
            assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
          else
            assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);

        }

        //Found Case
        FilePositionResult res = null;
        for (long i = 300 ; i < 320 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();

          res = posSetter.getFilePosition(i,finder);
          //log.info("For scn (" + i + "):  the result is:  "  + res);
          assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        }

        // USE Latest SCN (-1)
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();

        res = posSetter.getFilePosition(-1,finder);
        //log.info("For scn (" + i + "):  the result is:  "  + res);
        assertFilePositionResult(res,dir,299,FilePositionResult.Status.FOUND);
      }
View Full Code Here

  {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);

    DatabusRelayTestUtil.RelayRunner r1=null;
    ClientRunner cr = null;
    try
    {
      String[] srcs = { "com.linkedin.events.example.fake.FakeSchema"};

      int pId = 1;
      int srcId = 2;

      int relayPort = Utils.getAvailablePort(11994);;
      final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
      Assert.assertNotNull(relay1);
      r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
      log.info("Relay created");

      DbusEventBufferMult bufMult = relay1.getEventBuffer();


      String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
      PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
      DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
      DbusEventKey key = new DbusEventKey(123L);
      byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
      byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
      DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short)pId, (short)pId, 897L,
                                                  (short)srcId, schemaId, payload, false, true);
      eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
      buf.startEvents();
      buf.appendEvent(key, eventInfo, null);
      buf.endEvents(100L, null);
      r1.start();
      log.info("Relay started");

      // wait until relay comes up
      TestUtil.assertWithBackoff(new ConditionCheck() {
        @Override
        public boolean check() {
          return relay1.isRunningStatus();
        }
      },"Relay hasn't come up completely ", 7000, LOG);

      // now create client:
      String srcSubscriptionString = TestUtil.join(srcs, ",");
      String serverName = "localhost:" + relayPort;
      final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();

      int id = (RngUtils.randomPositiveInt() % 10000) + 1;
      DatabusSourcesConnection clientConn = RelayEventProducer
      .createDatabusSourcesConnection("testProducer", id, serverName,
          srcSubscriptionString, countingConsumer,
          1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000,
          1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id),
          0, DbusEventFactory.DBUS_EVENT_V1,0);

      cr = new ClientRunner(clientConn);

      cr.start();
      log.info("Consumer started");
      // wait till client gets the event
      TestUtil.assertWithBackoff(new ConditionCheck() {
        @Override
        public boolean check() {
View Full Code Here

    int relayPort = Utils.getAvailablePort(11993);

    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1=null;
    ClientRunner cr = null;
    try
    {
      //EventProducer[] producers = relay1.getProducers();
      r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
      log.info("Relay created");

      DbusEventBufferMult bufMult = relay1.getEventBuffer();
      PhysicalPartition pPartition = new PhysicalPartition((int)pId, pSourceName);
      DbusEventBuffer buf = (DbusEventBuffer)bufMult.getDbusEventBufferAppendable(pPartition);

      log.info("create some events");
      long windowScn = 100L;
      ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(),
          pId, DbusEventFactory.DBUS_EVENT_V2);
      ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
      int readEvents = buf.readEvents(channel);
      log.info("successfully read in " + readEvents + " events ");
      channel.close();

      windowScn = 101L;
      serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(),
          pId, DbusEventFactory.DBUS_EVENT_V1);
      channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
      readEvents = buf.readEvents(channel);
      log.info("successfully read in " + readEvents + " events ");
      channel.close();

      log.info("starting relay on port " + relayPort);
      r1.start();
      //TestUtil.sleep(10*1000);

      // wait until relay comes up
      TestUtil.assertWithBackoff(new ConditionCheck() {
        @Override
        public boolean check() {
          return relay1.isRunningStatus();
        }
      },"Relay hasn't come up completely ", 30000, LOG);

      log.info("now create client");
      String srcSubscriptionString = TestUtil.join(srcs, ",");
      String serverName = "localhost:" + relayPort;
      final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();

      int id = (RngUtils.randomPositiveInt() % 10000) + 1;
      DatabusSourcesConnection clientConn = RelayEventProducer
      .createDatabusSourcesConnection("testProducer", id, serverName,
          srcSubscriptionString, countingConsumer,
          1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000,
          1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id),
          0, DbusEventFactory.DBUS_EVENT_V1,0);

      cr = new ClientRunner(clientConn);

      log.info("starting client");
      cr.start();
      // wait till client gets the event
      TestUtil.assertWithBackoff(new ConditionCheck() {
        @Override
        public boolean check() {
          int events = countingConsumer.getNumDataEvents();
          LOG.info("client got " + events + " events");
          return events == 2;
        }
      },"Consumer didn't get 2 events ", 64 * 1024, LOG);

      // asserts
      Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
      Assert.assertEquals(countingConsumer.getNumWindows(), 2);
      Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1),2);
      log.info("shutdown first client");
      clientConn.stop();
      cr.shutdown();
      TestUtil.sleep(1000);
      cr = null;


      log.info("start another client who understands V2");
      final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();

      clientConn = RelayEventProducer
      .createDatabusSourcesConnection("testProducer", id, serverName,
          srcSubscriptionString, countingConsumer1,
          1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000,
          1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id),
          0, DbusEventFactory.DBUS_EVENT_V2,0);

      cr = new ClientRunner(clientConn);

      cr.start();
      log.info("wait till client gets the event");
      TestUtil.assertWithBackoff(new ConditionCheck() {
        @Override
        public boolean check() {
          int events = countingConsumer1.getNumDataEvents();
View Full Code Here

    String pPartName = "part1";
    short pPartId = 0;

    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for(int i=0; i<sourceIds.length; i++) {
      LogicalSourceConfig lc = new LogicalSourceConfig();
      lc.setId(sourceIds[i]);
      lc.setName(sourceNames[i]);
      lc.setPartitionFunction(partitionFunction);
      lc.setUri(pPartName + "." + sourceNames[i]); // this format is expected by GG
      pc.addSource(lc);
    }


    return pc.build();
View Full Code Here

    String resourceKey = "test";
    String partitionFunction = "constant:1";
    boolean skipInfinityScn = false;
    String queryHints = null;
    LogicalSourceStaticConfig[] sources = new LogicalSourceStaticConfig[1];
    LogicalSourceStaticConfig lssc = new LogicalSourceStaticConfig(id, name,
        uri, partitionFunction, partition, skipInfinityScn, queryHints,
        queryHints, queryHints);
    sources[0] = lssc;
    String role = "MASTER";
    long slowSourceQueryThreshold = 0L;
View Full Code Here

    String partitionFunction = "constant:1";

    String pPartName = "part1";
    short pPartId = 0;

    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for(int i=0; i<sourceIds.length; i++) {
      LogicalSourceConfig lc = new LogicalSourceConfig();
      lc.setId(sourceIds[i]);
      lc.setName(sourceNames[i]);
      lc.setPartitionFunction(partitionFunction);
      lc.setUri(pPartName + "." + sourceNames[i]); // this format is expected by GG
      pc.addSource(lc);
    }


    return pc.build();
  }
View Full Code Here

  @Test
  public void testSCNRegressionStats() throws Exception
  {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc =
        buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start producer
View Full Code Here

  @Test
  public void testTransactionsWithDuplicateSCN() throws Exception
  {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc =
        buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");

    DbusEventBufferAppendable mb = createBufMult(pssc);

    // start producer
View Full Code Here

TOP

Related Classes of com.linkedin.databus2.relay.TestDatabusRelayMain.ClientRunner

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.