Examples of PhysicalPartition

com.linkedin.databus.core.data_model.PhysicalPartition
rp.linkedin.com/wiki/cf/display/ENGS/Databus+2.0+and+Databus+3.0+Data+Model">Databus 2.0 and Databus 3.0 Data Model

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

      // partition (and the offset within that checkpoint).
      // If there was no partial window sourced and a cursor was set, then the cursor was the partition
      // from which we last streamed data. We start streaming from the partition *after* the cursor,
      // unless the cursor itself is not there any more (in which case, we stream from first partition).
      NavigableSet<PhysicalPartitionKey> workingSet = _pKeys;
      PhysicalPartition partialWindowPartition = _checkPoints.getPartialWindowPartition();
      if (partialWindowPartition != null) {
        // Ignore cursorPartition, and construct a working set that includes the partialWindowPartition.
        workingSet = _pKeys.tailSet(new PhysicalPartitionKey(partialWindowPartition), true);
        if (workingSet.size() == 0) {
          // Apparently we sourced an incomplete window from a partition that has since moved away?
          // Throw an exception
          throw new OffsetNotFoundException("Partial window offset not found" + partialWindowPartition);
        }
      } else {
        // We sent a complete window last time (or, this is the first time we are sending something)
        // If cursor partition exists, and we can find it in our pKeys, we have a working set starting
        // from the partition greater than cursor. Otherwise, we go with the entire key set.
        workingSet = _pKeys;
        PhysicalPartition cursorPartition = _checkPoints.getCursorPartition();
        if (cursorPartition != null) {
          PhysicalPartitionKey ppKey = new PhysicalPartitionKey(cursorPartition);
          workingSet = _pKeys.tailSet(ppKey, false);
          if (workingSet.isEmpty() || !_pKeys.contains(ppKey)) {
            workingSet = _pKeys;
          }
        }
      }


      // Initialize a datastructure, that contains the list of all physical partitions for which we invoked
      // streamEvents with streamFromLatest==true.
      // Details described in DDSDBUS-2461, DDSDBUS-2341 and rb 178201
      Set<PhysicalPartitionKey> streamFromLatestState = new HashSet<PhysicalPartitionKey>();


      // Send events from each partition in the working set, iterating through them in an ascending
      // order. Stop sending when we have overflowed the buffer, or there is nothing more to send in
      // any of the buffers.
      // Note that in the first iteration of the outer loop, it may be that we have not scanned all
      // partitions (because our working set was smaller). In that case, even if nothing was streamed,
      // we want to continue through (at least) one more iteration, scanning all partitions.
      // Keep track of  partitions that are not able to send data because the event would not fit
      // into the buffer offered by the client. If we are returning from this method without sending a
      // single event *and* we had events in some of the partitions that would not fit in the client's
      // buffer, then send back the size of the smallest of such events to the client. It could well be
      // that the client can make progress in other partitions, but one partition could be blocked forever
      // because of this event if the client was offering its full buffer size.
      int minPendingEventSize = 0;
      boolean done = false;
      while (!done) {
        boolean somethingStreamed = false;


        // go over relevant buffers
        for(PhysicalPartitionKey pKey : workingSet) {
          DbusEventBuffer buf = _bufsMap.get(pKey);
          if (null == buf)
          {
            // in this case we want to disconnect the client
            String errMsg = "Buffer not found for physicalPartitionKey " + pKey;
            LOG.error(errMsg);
            throw new BufferNotFoundException(errMsg);
          }
          PhysicalPartition pPartition = pKey.getPhysicalPartition();
          DbusEventsStatisticsCollector statsCollector = _statsCollectors == null ? null : _statsCollectors.getStatsCollector(pPartition.toSimpleString());


          Checkpoint cp=null;
          cp = _checkPoints.getCheckpoint(pKey.getPhysicalPartition());// get the corresponding checkpoint
          if(debugEnabled)
            LOG.debug("get Checkpoint by pPartition" + pPartition + ";cp=" + cp);

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

    public void setPhysicalPartition(PhysicalPartition p) {
      _pPartition = p;
    }


    public PhysicalPartitionKey() {
      _pPartition = new PhysicalPartition();
    }

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

    @Override
    public int compareTo(Object other) {
      if (!(other instanceof PhysicalPartitionKey)) {
        throw new ClassCastException("PhysicalPartitionKey class expected instead of " + other.getClass().getSimpleName());
      }
      PhysicalPartition op = ((PhysicalPartitionKey)other).getPhysicalPartition();
      return _pPartition.compareTo(op);
    }

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition


      DbusEventBufferMult bufMult = relay1.getEventBuffer();




      String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
      PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
      DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
      DbusEventKey key = new DbusEventKey(123L);
      byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
      byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
      DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short)pId, (short)pId, 897L,

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

      //EventProducer[] producers = relay1.getProducers();
      r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
      log.info("Relay created");


      DbusEventBufferMult bufMult = relay1.getEventBuffer();
      PhysicalPartition pPartition = new PhysicalPartition((int)pId, pSourceName);
      DbusEventBuffer buf = (DbusEventBuffer)bufMult.getDbusEventBufferAppendable(pPartition);


      log.info("create some events");
      long windowScn = 100L;
      ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(),

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

  {
    TestSetup t = new TestSetup();


    //test single Physical Partition subscription
    DatabusSubscription sub1 =
        DatabusSubscription.createPhysicalPartitionReplicationSubscription(new PhysicalPartition(100, "multBufferTest1"));


    DbusFilter filter1 = t._eventBuffer.constructFilters(Arrays.asList(sub1));
    assertNotNull(filter1);
    assertTrue(filter1 instanceof PhysicalPartitionDbusFilter);
    PhysicalPartitionDbusFilter ppfilter1 = (PhysicalPartitionDbusFilter)filter1;
    assertEquals(ppfilter1.getPhysicalPartition(), new PhysicalPartition(100, "multBufferTest1"));
    assertNull(ppfilter1.getNestedFilter());


    DatabusSubscription sub2 =
        DatabusSubscription.createPhysicalPartitionReplicationSubscription(new PhysicalPartition(101, "multBufferTest2"));


    //test two Physical Partition subscriptions
    DbusFilter filter2 = t._eventBuffer.constructFilters(Arrays.asList(sub1, sub2));
    assertNotNull(filter2);
    assertTrue(filter2 instanceof ConjunctionDbusFilter);
    ConjunctionDbusFilter conjFilter2 = (ConjunctionDbusFilter)filter2;
    boolean hasPP100 = false;
    boolean hasPP101 = false;
    assertEquals(conjFilter2.getFilterList().size(), 2);
    for (DbusFilter f: conjFilter2.getFilterList())
    {
      assertTrue(f instanceof PhysicalPartitionDbusFilter);
      PhysicalPartitionDbusFilter ppf = (PhysicalPartitionDbusFilter)f;
      if (ppf.getPhysicalPartition().getId() == 100) hasPP100 = true;
      else if (ppf.getPhysicalPartition().getId() == 101) hasPP101 = true;
      else fail("unknown physical partition filter:" + ppf.getPhysicalPartition());
    }
    assertTrue(hasPP100);
    assertTrue(hasPP101);


    //test a subcription with a logical source
    DatabusSubscription sub3 =
        DatabusSubscription.createSimpleSourceSubscription(new LogicalSource(2, "srcName2"));


    DbusFilter filter3 = t._eventBuffer.constructFilters(Arrays.asList(sub3));
    assertNotNull(filter3);
    assertTrue(filter3 instanceof PhysicalPartitionDbusFilter);
    PhysicalPartitionDbusFilter ppfilter3 = (PhysicalPartitionDbusFilter)filter3;
    assertEquals(ppfilter3.getPhysicalPartition(), PhysicalPartition.ANY_PHYSICAL_PARTITION);
    DbusFilter ppfilter3_child = ppfilter3.getNestedFilter();
    assertNotNull(ppfilter3_child);
    assertTrue(ppfilter3_child instanceof LogicalSourceAndPartitionDbusFilter);
    LogicalSourceAndPartitionDbusFilter lsourceFilter3 = (LogicalSourceAndPartitionDbusFilter)ppfilter3_child;
    LogicalSourceAndPartitionDbusFilter.LogicalPartitionDbusFilter lpartFilter3_1 =
        lsourceFilter3.getSourceFilter(2);
    assertNotNull(lpartFilter3_1);
    assertTrue(lpartFilter3_1.isAllPartitionsWildcard());


    //test a subcription with a physical and logical partition
    DatabusSubscription sub4 =
        new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE,
                                new PhysicalPartition(101, "multBufferTest2"),
                                new LogicalSourceId(new LogicalSource(2, "srcName2"), (short)2)
                                );


    DbusFilter filter4 = t._eventBuffer.constructFilters(Arrays.asList(sub4));
    assertNotNull(filter4);
    assertTrue(filter4 instanceof PhysicalPartitionDbusFilter);
    PhysicalPartitionDbusFilter ppfilter4 = (PhysicalPartitionDbusFilter)filter4;
    assertEquals(ppfilter4.getPhysicalPartition(), new PhysicalPartition(101, "multBufferTest2"));
    DbusFilter ppfilter4_child = ppfilter4.getNestedFilter();
    assertNotNull(ppfilter4_child);
    assertTrue(ppfilter4_child instanceof LogicalSourceAndPartitionDbusFilter);
    LogicalSourceAndPartitionDbusFilter lsourceFilter4 = (LogicalSourceAndPartitionDbusFilter)ppfilter4_child;
    LogicalSourceAndPartitionDbusFilter.LogicalPartitionDbusFilter lpartFilter4_1 =

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

    final Logger log = Logger.getLogger("TestDbusEventBufferMult.testSubscriptionStream");
    log.info("start");


    TestSetup t = new TestSetup();


    PhysicalPartition pp100 = new PhysicalPartition(100, "multBufferTest1");
    PhysicalPartitionKey pk1 = new PhysicalPartitionKey(pp100);
    PhysicalPartition pp101 = new PhysicalPartition(101, "multBufferTest2");
    PhysicalPartitionKey pk2 = new PhysicalPartitionKey(pp101);


    //generate events in pp100
    byte [] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
    DbusEventBufferAppendable buf100 =
        t._eventBuffer.getDbusEventBufferAppendable(pp100);
    buf100.startEvents();
    assertTrue(buf100.appendEvent(new DbusEventKey(1), (short)100, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)1,
                                  schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(10), (short)100, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)1,
                                  schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(11), (short)100, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)1,
                                  schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(2), (short)100, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    buf100.endEvents(100,  null);
    buf100.startEvents();
    assertTrue(buf100.appendEvent(new DbusEventKey(3), (short)100, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(4), (short)100, (short)1,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    buf100.endEvents(200, null);


    //generate events in pp100
    DbusEventBufferAppendable buf101 =
        t._eventBuffer.getDbusEventBufferAppendable(pp101);
    buf101.startEvents();
    assertTrue(buf101.appendEvent(new DbusEventKey(51), (short)101, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)11,
                                  schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(52), (short)101, (short)0,
                                  System.currentTimeMillis() * 1000000, (short)12,
                                  schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(53), (short)101, (short)2,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    buf101.endEvents(120,  null);
    buf101.startEvents();
    assertTrue(buf101.appendEvent(new DbusEventKey(54), (short)101, (short)2,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(55), (short)101, (short)2,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(56), (short)101, (short)2,
                                  System.currentTimeMillis() * 1000000, (short)2,
                                  schema, new byte[100], false, null));
    buf101.endEvents(200,  null);


    //initialization
    DatabusSubscription sub1 =
        DatabusSubscription.createPhysicalPartitionReplicationSubscription(new PhysicalPartition(100, "multBufferTest1"));


    DbusFilter filter1 = t._eventBuffer.constructFilters(Arrays.asList(sub1));
    assertNotNull(filter1);


    CheckpointMult cpMult1 = new CheckpointMult();
    Checkpoint cp100 = new Checkpoint();
    cp100.init();
    cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp100.setWindowScn(10L);
    cp100.setWindowOffset(-1);
    cpMult1.addCheckpoint(pp100, cp100);


    String[] pnames = {"multBufferTest1:100","multBufferTest2:101"};
    StatsCollectors<DbusEventsStatisticsCollector> statsColls1 = createStats(pnames);
    DbusEventsStatisticsCollector statsCol1 = statsColls1.getStatsCollector("multBufferTest1:100");
    DbusEventsStatisticsCollector statsCol2 = statsColls1.getStatsCollector("multBufferTest2:101");


    //read an entire buffer
    DbusEventBufferBatchReadable reader1 =
        t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1), statsColls1);


    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // Try a call with 20 bytes of fetch size, we should see the event size in the first return with 0 events read.
    StreamEventsResult result = reader1.streamEvents(false, 20, Channels.newChannel(baos),
                                                     Encoding.BINARY, filter1);
    assertEquals(0, result.getNumEventsStreamed());
    assertEquals(161, result.getSizeOfPendingEvent());


    result = reader1.streamEvents(false, 1000000, Channels.newChannel(baos),
                                          Encoding.BINARY, filter1);
    int eventsRead = result.getNumEventsStreamed();
    assertEquals(eventsRead, 8); //4 events + 1 eop + 2 events + 1 eop
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6);
    assertEquals(result.getSizeOfPendingEvent(), 0, "Size of pending event not zero");


    // Now that we have read all the events, we should not see a pending event even if we offer a small fetch size.


    result = reader1.streamEvents(false, 20, Channels.newChannel(baos),
                                                     Encoding.BINARY, filter1);


    assertEquals(0, result.getNumEventsStreamed(), "There should be no more events in the buffer now");
    assertEquals(0, result.getSizeOfPendingEvent(), "We should not see pending event size since there are no events in buffer");
    baos.reset();
    statsCol1.reset(); statsCol2.reset();


    //read from two buffers, filtering out one
    cpMult1 = new CheckpointMult();
    cp100.init();
    cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp100.setWindowScn(10L);
    cp100.setWindowOffset(-1);
    cpMult1.addCheckpoint(pp100, cp100);
    reader1 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1, pk2),
                                                             statsColls1);


    eventsRead = reader1.streamEvents(false, 1000000, Channels.newChannel(baos),
                                      Encoding.BINARY, filter1).getNumEventsStreamed();
    assertEquals(eventsRead, 10); //4 events + 1 eop + 1 eop from the other buffer + 2 events +
                                  //1 eop + 1 eop from the other buffer
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6);


    baos.reset();
    statsCol1.reset();


    //read from one buffer and one source partition
    DatabusSubscription sub2 =
        new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE,
                                new PhysicalPartition(101, "multBufferTest2"),
                                new LogicalSourceId(new LogicalSource(2, "srcName2"), (short)2)
                                );


    DbusFilter filter2 = t._eventBuffer.constructFilters(Arrays.asList(sub2));
    assertNotNull(filter2);

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

  @Test
  public void addRemoveBufferMapping() throws InvalidConfigException, DatabusException {
    createBufMult();


    PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalPartition pPart = _eventBufferMult.getPhysicalPartition(11);
    Assert.assertNotNull(pPart);
    DbusEventBuffer buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource());
    Assert.assertNotNull(buf);
    DbusEventBuffer buf1 = _eventBufferMult.getOneBuffer(pPart);
    Assert.assertNotNull(buf1);

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition

        }


        ppartKeys = new TreeSet<PhysicalPartitionKey>();
        for (DatabusSubscription sub: subs)
        {
          PhysicalPartition ppart = sub.getPhysicalPartition();
          if (ppart.isAnyPartitionWildcard())
          {
            ppartKeys = _eventBuffer.getAllPhysicalPartitionKeys(); break;
          }
          else
          {
            ppartKeys.add(new PhysicalPartitionKey(ppart));
          }
        }
      }
      // TODO
      // The following if statement is a very conservative one just to make sure that there are
      // not some clients out there that send subs, but do not send checkpoint mult. It seems that
      // this was the case during development but never in production, so we should remove this
      // pretty soon (1/28/2013).
      // Need to make sure that we don't have tests that send requests in this form.
      if(subs != null && checkpointStringMult == null && checkpointString != null) {
        throw new RequestProcessingException("Both Subscriptions and CheckpointMult should be present");
      }


      //convert source ids into subscriptions
      if (null == subs) subs = new ArrayList<DatabusSubscription>();
      for (Integer srcId: sourceIds)
      {
        LogicalSource lsource = srcRegistry.getSource(srcId);
        if(lsource == null)
          throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, srcId.toString());
        if(isDebug)
          LOG.debug("registry returns " + lsource  + " for srcid="+ srcId);
        DatabusSubscription newSub = DatabusSubscription.createSimpleSourceSubscription(lsource);
        subs.add(newSub);
      }


      DbusFilter ppartFilters = null;
      if (subs.size() > 0)
      {
        try
        {
          ppartFilters = _eventBuffer.constructFilters(subs);
        }
        catch (DatabusException de)
        {
          throw new RequestProcessingException("unable to generate physical partitions filters:" +
                                               de.getMessage(),
                                               de);
        }
      }


      ConjunctionDbusFilter filters = new ConjunctionDbusFilter();


      // Source filter comes first
      if (v2Mode) filters.addFilter(new SourceDbusFilter(sourceIds));
      else if (null != ppartFilters) filters.addFilter(ppartFilters);


      /*
      // Key range filter comes next
      if ((keyMin >0) && (keyMax > 0))
      {
        filters.addFilter(new KeyRangeFilter(keyMin, keyMax));
      }
      */
      if ( null != keyCompositeFilter)
      {
        filters.addFilter(keyCompositeFilter);
      }


      // need to update registerStreamRequest to support Mult checkpoint TODO (DDSDBUS-80)
      // temp solution
      // 3 options:
      // 1. checkpointStringMult not null - generate checkpoint from it
      // 2. checkpointStringMult null, checkpointString not null - create empty CheckpointMult
      // and add create Checkpoint(checkpointString) and add it to cpMult;
      // 3 both are null - create empty CheckpointMult and add empty Checkpoint to it for each ppartition
      PhysicalPartition pPartition;


      Checkpoint cp = null;
      CheckpointMult cpMult = null;


      if(checkpointStringMult != null) {

View Full Code Here

Examples of com.linkedin.databus.core.data_model.PhysicalPartition


    // Coverage for null processor.
    Assert.assertEquals(req, req.call());


    // Coverqage for setCursorPartition
    PhysicalPartition pp = new PhysicalPartition(15, "SomePartition");
    req.setCursorPartition(pp);
    Assert.assertEquals(pp, req.getCursorPartition());


    Assert.assertFalse(req.cancel(true));
    Assert.assertFalse(req.cancel(false));

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.