Package com.linkedin.databus2.core.filter

Examples of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig


        filterConfig.getMod().setNumBuckets(cluster.getNumTotalPartitions());
        filterConfig.getMod().setBuckets("[" + partition.getPartitionId() + "]");
        compositeConfig.setFilter(s.getLogicalSource().getName(), filterConfig);
    }
   
    DbusKeyCompositeFilterConfig c = new DbusKeyCompositeFilterConfig(compositeConfig.build());
    LOG.info("Generated Mod Partitioned Config for partition (" + partition + ") of cluster (" + cluster + ") is :" + c);
    return c;
  }
View Full Code Here


        filterConfig.getRange().setSize(_rangeWidth);
        filterConfig.getRange().setPartitions("[" + partition.getPartitionId() + "]");
        compositeConfig.setFilter(s.getLogicalSource().getName(), filterConfig);
    }
   
    DbusKeyCompositeFilterConfig c = new DbusKeyCompositeFilterConfig(compositeConfig.build());
    LOG.info("Generated Mod Partitioned Config for partition (" + partition + ") of cluster (" + cluster + ") is :" + c);
    return c;
 
  }
View Full Code Here

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
View Full Code Here

            .andReturn(vs)
            .anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);

    // mock for MaxSCNReadWriter
    MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);
    EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();
    mscn.saveMaxScn(EasyMock.anyLong());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(mscn);
    EasyMock.replay(srs);

    int totalTransWritten = 0;
View Full Code Here

    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();

    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters
        .getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:"
        + maxScnReaderWriters);

    // Get the event buffer
View Full Code Here

  /** get maxScnReaderWriters given a physical source **/
  public MaxSCNReaderWriter getMaxSCNReaderWriter(PhysicalSourceStaticConfig pConfig)
  {
    try
    {
      MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters
        .getOrCreateHandler(pConfig.getPhysicalPartition());
      return maxScnReaderWriters;
    }
    catch (DatabusException e)
    {
View Full Code Here

      PhysicalSourceStaticConfig[] pConfigs) throws IOException,
      InvalidConfigException, DatabusException {
    super(config, pConfigs);
    SequenceNumberHandlerFactory handlerFactory = _relayStaticConfig
        .getDataSources().getSequenceNumbersHandler().createFactory();
    _maxScnReaderWriters = new MultiServerSequenceNumberHandler(
        handlerFactory);
    _producers = new HashMap<PhysicalPartition, EventProducer>(
        _pConfigs.size());
    _monitoringProducers = new HashMap<PhysicalPartition, MonitoringEventProducer>(_pConfigs.size());
    _dbPullerStart = false;
View Full Code Here

  public DatabusRelayMain(HttpRelay.StaticConfig config,
      PhysicalSourceStaticConfig[] pConfigs) throws IOException,
      InvalidConfigException, DatabusException {
    super(config, pConfigs);
    SequenceNumberHandlerFactory handlerFactory = _relayStaticConfig
        .getDataSources().getSequenceNumbersHandler().createFactory();
    _maxScnReaderWriters = new MultiServerSequenceNumberHandler(
        handlerFactory);
    _producers = new HashMap<PhysicalPartition, EventProducer>(
        _pConfigs.size());
View Full Code Here

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      DBUpdateImage dbi = result.get(0).getDbUpdatesSet().iterator().next();
      DBUpdateImage expDbi = newDbUpdates.get(0).getDbUpdatesSet().iterator().next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      String expK = (String) expDbi.getGenericRecord().get("name1");
      String expV = (String) expDbi.getGenericRecord().get("name2");
      Assert.assertEquals("Key Check", expK, k);
      Assert.assertEquals("Val Check", expV, v);

      dbi = result.get(1).getDbUpdatesSet().iterator().next();
      expDbi = newDbUpdates.get(1).getDbUpdatesSet().iterator().next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      expK = (String) expDbi.getGenericRecord().get("name1");
      expV = (String) expDbi.getGenericRecord().get("name2");
      Assert.assertEquals("Key Check", expK, k);
      Assert.assertEquals("Val Check", expV, v);
    }

    // Case when 2 txn have same source but different keys
    {
      String key = "key1";
      String oldVal = "val1";
      Map<String, String> keyVals1 = new HashMap<String, String>();
      keyVals1.put(key, oldVal);
      Map<String, String> keyVals2 = new HashMap<String, String>();
      key = "key2";
      String newVal = "val2";
      keyVals2.put(key, newVal);
      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =
          generateUpdatesForSchema2(sourceIds, keyVals1, scn);
      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =
          generateUpdatesForSchema2(sourceIds, keyVals2, scn);
      List<TransactionState.PerSourceTransactionalUpdate> result =
          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 2);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 2);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();
      DBUpdateImage dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k2 = (String) dbi.getGenericRecord().get("name1");
      String v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))
      {
        Assert.assertEquals("Val Check", keyVals1.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);
      }
      else
      {
        Assert.assertEquals("Val Check", keyVals2.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);
      }

      itr = result.get(1).getDbUpdatesSet().iterator();
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k2 = (String) dbi.getGenericRecord().get("name1");
      v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))
      {
        Assert.assertEquals("Val Check", keyVals1.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);
      }
      else
      {
        Assert.assertEquals("Val Check", keyVals2.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);
      }
    }

    // Case when 2 txns have different sources
    {
      String key = "key1";
      String oldVal = "val1";
      Map<String, String> keyVals1 = new HashMap<String, String>();
      keyVals1.put(key, oldVal);
      Map<String, String> keyVals2 = new HashMap<String, String>();
      key = "key2";
      String newVal = "val2";
      keyVals2.put(key, newVal);
      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =
          generateUpdatesForSchema2(new short[] { sourceIds[0] }, keyVals1, scn);
      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =
          generateUpdatesForSchema2(new short[] { sourceIds[1] }, keyVals2, scn);
      List<TransactionState.PerSourceTransactionalUpdate> result =
          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();
      DBUpdateImage dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      Assert.assertEquals("Val Check", keyVals1.get(k), v);

      itr = result.get(1).getDbUpdatesSet().iterator();
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      Assert.assertEquals("Val Check", keyVals2.get(k), v);
    }
  }
View Full Code Here

    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");

    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,
        DbUpdateState.DBUpdateImage.OpType.INSERT,false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(
        sourceId, db);
    dbUpdates.add(dbUpdate);
View Full Code Here

TOP

Related Classes of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.