Examples of com.linkedin.databus2.producers.PartitionFunction

com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder
Interface to return the partition for a given key.

              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
      // Get all relevant pConfig attributes
      //TODO add real instantiation
      EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
      if (null == mockProvider)
      {
        throw new DatabusRuntimeException("relay event producer not available: " + "mock");
      }
      producer = mockProvider.createProducer(pConfig, schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                     .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);
    } else if (uri.startsWith("gg:")){
      producer = new GoldenGateEventProducer(pConfig,
                                             schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                 .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);


    } else if (uri.startsWith("mysql:")){
       LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
       final String serviceName = "or";
       EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
       if (null == orProvider)
       {
         throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
       }
       producer = orProvider.createProducer(pConfig, schemaRegistryService,
                                            dbusEventBuffer,
                                            _inBoundStatsCollectors.getStatsCollector(statsCollectorName),
                                            maxScnReaderWriters);
    } else
     {

View Full Code Here

    if(eventView.toLowerCase().startsWith("sy$"))
    {
      eventView = eventView.substring(3);
    }


    PartitionFunction partitionFunction = buildPartitionFunction(sourceConfig);
    EventFactory factory = createEventFactory(eventViewSchema, eventView, sourceConfig, pConfig,
                                              schema, partitionFunction);


    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());

View Full Code Here

  public GGMonitoredSourceInfo buildGGMonitoredSourceInfo(
            LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig)
                throws DatabusException, InvalidConfigException
  {
    // udpate partition mapping
    PartitionFunction partitionFunction = GGEventGenerationFactory.buildPartitionFunction(sourceConfig);
    _partitionFunctionHashMap.put((int)sourceConfig.getId(), partitionFunction);


    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());


    GGMonitoredSourceInfo sourceInfo =

View Full Code Here

        //Construct the Databus Event key, determine the key type and construct the key
        Object keyObj = obtainKey(dbUpdate);
        DbusEventKey eventKey = new DbusEventKey(keyObj);


        //Get the logicalparition id
        PartitionFunction partitionFunction = _partitionFunctionHashMap.get((int)sourceId);
        short lPartitionId = partitionFunction.getPartition(eventKey);


        record = dbUpdate.getGenericRecord();
        //Write the event to the buffer
        if (record == null)
          throw new DatabusException("Cannot write event to buffer because record = " + record);

View Full Code Here

    {
      eventViewSchema = null;
      eventView = sourceConfig.getUri();
    }


    PartitionFunction partitionFunction = buildPartitionFunction(sourceConfig);
    OpenReplicatorAvroEventFactory factory = createEventFactory(eventViewSchema, eventView, sourceConfig, pConfig,
                                              schema, partitionFunction);
    return factory;
  }

View Full Code Here

     {
      // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
      RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools =
            new RelayEventProducer.DatabusClientNettyThreadPools(0,getNetworkTimeoutTimer(),getBossExecutorService(),
                                        getIoExecutorService(), getHttpChannelGroup());
      producer = new RelayEventProducer(pConfig, dbusEventBuffer,
          _inBoundStatsCollectors
              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters,nettyThreadPools);
    }

View Full Code Here

    log.info("starting");


    File dir = createTempDir();


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();


    // Now delete the directory to make the locateFilePosition() see null for listFiles() call.
    boolean deleted = dir.delete();
    Assert.assertTrue(deleted, "Deleted the trail directory successfully");

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     100 /* corrupt first SCN */, "xyzzy", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // SCN 100 is corrupted, so 101 is the effective oldest SCN => 100 treated as error:
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR,
                        "expected error for exact-match SCN that's corrupted and oldest in all trail files.");


    // SCN 101 is OK (regexQuery() doesn't fully validate XML):
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 101, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     307 /* corrupt last SCN in 2nd file */, "plugh", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 307 occurs after SCN 299, so latter should be found OK:
    res = posSetter.locateFilePosition(299, finder);
    assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);


    // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK
    finder.reset();
    res = posSetter.locateFilePosition(306, finder);
    assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND);


    log.info(DONE_STRING);
  }

View Full Code Here

    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines and 2 SCNs each */,
                     1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0,
                     399 /* corrupt last SCN in 3rd file */, "quux", false, "");


    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;


    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.linkedin.databus2.producers.PartitionFunction

com.google.code.or.common.glossary.Row

com.linkedin.databus.bootstrap.server.BootstrapHttpServer

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader

com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain

com.linkedin.databus.bootstrap.utils.BootstrapTableReader

com.linkedin.databus.client.BootstrapPullThread

com.linkedin.databus.client.consumer.TestInternalMetadata

com.linkedin.databus.client.DatabusHttpClientImpl

com.linkedin.databus.client.DatabusSourcesConnection

com.linkedin.databus.client.DatabusSourcesConnection$Config

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.