Package com.linkedin.databus2.relay.config

Examples of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig

1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
    String xmlEncoding = "";
    String fieldName = "";
    String regex = "";
    ReplicationBitSetterStaticConfig replicationBitSetter = new ReplicationBitSetterStaticConfig(
        ReplicationBitSetterStaticConfig.SourceType.NONE, fieldName, regex, MissingValueBehavior.STOP_WITH_ERROR);
    PhysicalSourceStaticConfig pssc = new PhysicalSourceStaticConfig(name, id,
        uri, resourceKey, sources, role, slowSourceQueryThreshold,
        restartScnOffset, bsc.build(), ct, txnsPerChunk, scnChunkSize,
        chunkedScnThreshold, maxScnDelayMs, eventRatePerSec, eventRateThrottleDuration, dbusEventBuffer,
        largestEventSizeInBytes, largestWindowSizeInBytes,
        errorOnMissingFields, xmlVersion, xmlEncoding, replicationBitSetter);
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
251
252
  @BeforeTest
  public void setUpTest () throws IOException, InvalidConfigException {
    LOG.info("Setting up Test");

    PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build();
    PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalSourceStaticConfig pStatConf3 = convertToPhysicalSourceConfig(_configSource3).build();

    _pConfigs =  new PhysicalSourceStaticConfig [] {pStatConf1, pStatConf2, pStatConf3};

    // generate testData
    int scn = 100;
View Full Code Here
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
  @Test
  public void verifyReadingLogicalPartitionWildcard() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException {
     createBufMult();

     PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build();
     PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build();

     PhysicalPartition pP = pStatConf1.getPhysicalPartition();
     DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(pP);

     buf.startEvents();
     byte [] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
     assertTrue(buf.appendEvent(new DbusEventKey(1), (short)100, (short)0,
                                   System.currentTimeMillis() * 1000000, (short)2,
                                   schema, new byte[100], false, null));

     assertTrue(buf.appendEvent(new DbusEventKey(1), (short)100, (short)1,
         System.currentTimeMillis() * 1000000, (short)2,
         schema, new byte[100], false, null));
     buf.endEvents(100, null);

     pP = pStatConf2.getPhysicalPartition();
     buf = _eventBufferMult.getDbusEventBufferAppendable(pP);
     buf.startEvents();

     assertTrue(buf.appendEvent(new DbusEventKey(1), (short)101, (short)2,
         System.currentTimeMillis() * 1000000, (short)2,
View Full Code Here
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
    Set<Integer> srcIds = new HashSet<Integer>(2);
    srcIds.add(1);

    // total expected events from source 1 is 5
    batchReading(srcIds, 5);
    PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource3).build();
    LOG.info("one more buffer for " + pConfig);
    DbusEventBuffer buf = _eventBufferMult.addNewBuffer(pConfig, _config);
    buf.start(100);

    // add events to the new buffer
    byte [] schema = "ABCDEFGHIJKLMNOP".getBytes(Charset.defaultCharset());
    for(int i=100; i<110; i++) {
      if(i%2 == 0)
        buf.startEvents();
      assertTrue(buf.appendEvent(new DbusEventKey(i),
                                 pConfig.getPhysicalPartition().getId().shortValue(),
                                 (short)0, // logical source id
                                 System.currentTimeMillis(),
                                 (short)(i<105?21:22),
                                 schema,
                                 (""+i).getBytes(Charset.defaultCharset()), false, null));
View Full Code Here
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
  @Test
  public void addRemoveBufferMapping() throws InvalidConfigException, DatabusException {
    createBufMult();

    PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalPartition pPart = _eventBufferMult.getPhysicalPartition(11);
    Assert.assertNotNull(pPart);
    DbusEventBuffer buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource());
    Assert.assertNotNull(buf);
    DbusEventBuffer buf1 = _eventBufferMult.getOneBuffer(pPart);
    Assert.assertNotNull(buf1);
    Assert.assertTrue(buf == buf1, "same references to the buffer");

    // now remove the buffer and the mapping
    _eventBufferMult.removeBuffer(pConfig);
    _eventBufferMult.deallocateRemovedBuffers(true);

    // same should fail
    pPart = _eventBufferMult.getPhysicalPartition(11);
    Assert.assertNull(pPart);
    buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource());
    Assert.assertNull(buf);
    buf1 = _eventBufferMult.getOneBuffer(pPart);
    Assert.assertNull(buf1);

View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
      isr.close();
      isr = new InputStreamReader(IOUtils.toInputStream(_configSource2));
      PhysicalSourceConfig pConfig2 = mapper.readValue(isr, PhysicalSourceConfig.class);

      PhysicalSourceStaticConfig pStatConf1 = pConfig1.build();
      PhysicalSourceStaticConfig pStatConf2 = pConfig2.build();

      _physConfigs =  new PhysicalSourceStaticConfig [] {pStatConf1, pStatConf2};

      DbusEventBuffer.Config conf = new DbusEventBuffer.Config();
      conf.setAllocationPolicy("MMAPPED_MEMORY");
View Full Code Here
541
542
543
544
545
546
547
548
549
550
551
552
    isr.close();
    isr = new InputStreamReader(IOUtils.toInputStream(TestDbusEventBufferMult._configSource2));
    PhysicalSourceConfig pConfig2 = mapper.readValue(isr, PhysicalSourceConfig.class);

    PhysicalSourceStaticConfig pStatConf1 = pConfig1.build();
    PhysicalSourceStaticConfig pStatConf2 = pConfig2.build();

    PhysicalSourceStaticConfig[] _physConfigs =  new PhysicalSourceStaticConfig [] {pStatConf1, pStatConf2};

    return new DbusEventBufferMult(_physConfigs, config, new DbusEventV2Factory());
  }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
    if(uri == null)
      throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
      SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
          if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");

      // if a buffer for this partiton exists - we are overwri
      producer = new OracleEventProducerFactory().buildEventProducer(
View Full Code Here
129
130
131
132
133
134
135
136
137
          break;

        default:
          break;
      }
      SchemaId schemaId = schemaStr != null ? SchemaId.createWithMd5(schemaStr) : null;
      byte[] schemaBytes = schemaId != null? schemaId.getByteArray(): new byte[16];
      initBufferWithEvents(eventsBuf, keyBase, numEvents, srcId,schemaBytes, keyCounts, srcidCounts);
    }
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer()
        .getDbusEventBufferAppendable(pPartition);

    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();

    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
 
View Full Code Here
TOP

Related Classes of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig

  • com.google.code.or.common.glossary.Row
  • com.linkedin.databus.bootstrap.server.BootstrapHttpServer
  • com.linkedin.databus.bootstrap.server.BootstrapRequestProcessor
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain
  • com.linkedin.databus.bootstrap.utils.BootstrapTableReader
  • com.linkedin.databus.client.BootstrapPullThread
  • com.linkedin.databus.client.consumer.TestInternalMetadata
  • com.linkedin.databus.client.DatabusHttpClientImpl
  • com.linkedin.databus.client.DatabusSourcesConnection
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.