Package com.linkedin.databus.core

Examples of com.linkedin.databus.core.Checkpoint


                         sourcesResp);

    //send partial /stream
    callback.clearLastMsg();
    objCapture.clear();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);

View Full Code Here


      OffsetNotFoundException,
      IOException
  {
    HttpRequest msgReq;
    objCapture.clear();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);

View Full Code Here

    {
      tearConnectionAndEnqueuePickServer();
    }
    else
    {
      final Checkpoint cp = curState.getCheckpoint();
      curState.getBstCheckpointHandler().advanceAfterSnapshotPhase(cp);
      curState.getBstCheckpointHandler().advanceAfterTargetScn(cp);
      curState.switchToRequestStream(curState.getCheckpoint());
      enqueueMessage(curState);
    }
View Full Code Here

    int freeBufferThreshold=(int)(_sourcesConn.getConnectionConfig().getFreeBufferThreshold() *
        100.0 / _pullerBufferUtilizationPct);
    int freeSpace = curState.getDataEventsBuffer().getBufferFreeReadSpace();
    if (freeSpace >= freeBufferThreshold)
    {
      Checkpoint cp = curState.getCheckpoint();
      if (debugEnabled) _log.debug("Checkpoint at RequestBootstrapData: " + cp.toString());


      _log.debug("Sending /bootstrap request");

      Map<String, IdNamePair> srcNameMap = curState.getSourcesNameMap();
      String curSrcName = null;
      if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT)
      {
        curSrcName = cp.getSnapshotSource();
      } else {
        curSrcName = cp.getCatchupSource();
      }


      if ( null == _bootstrapFilter)
      {
View Full Code Here

  }

  protected void checkSharedCps(SharedCheckpointPersistenceProvider sharedCp, Vector<String> sources)
  {

    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    Random r = new Random(System.currentTimeMillis());
    long scn = r.nextLong();

    cp.setWindowScn(scn);
    cp.setWindowOffset(-1);
    try
    {
      sharedCp.storeCheckpoint(sources,cp);
    }
    catch (IOException e)
    {
      e.printStackTrace();
      AssertJUnit.assertTrue(false);
    }
    Checkpoint newCp = sharedCp.loadCheckpoint(sources);

    AssertJUnit.assertTrue(newCp != null);
    AssertJUnit.assertTrue(newCp.getWindowScn() == cp.getWindowScn());
    AssertJUnit.assertTrue(newCp.getWindowOffset().equals(cp.getWindowOffset()));
    AssertJUnit.assertTrue(newCp.getConsumptionMode() == cp.getConsumptionMode());

    System.out.printf("Stored scn = %d\n",newCp.getWindowScn());


  }
View Full Code Here

    boolean enqueueMessage = true;

    try
    {
      Checkpoint cp = curState.getCheckpoint();
      DbusEventBuffer eventBuffer = curState.getDataEventsBuffer();

      if (debugEnabled) _log.debug("Sending bootstrap events to buffer");

      //eventBuffer.startEvents();
      DbusEventInternalReadable cpEvent = getEventFactory().createCheckpointEvent(cp);
      byte[] cpEventBytes = new byte[cpEvent.size()];

      if (debugEnabled)
      {
        _log.debug("checkpoint event size: " + cpEventBytes.length);
        _log.debug("checkpoint event:" + cpEvent.toString());
      }

      cpEvent.getRawBytes().get(cpEventBytes);
      ByteArrayInputStream cpIs = new ByteArrayInputStream(cpEventBytes);
      ReadableByteChannel cpRbc = Channels.newChannel(cpIs);

      UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
      sendHeartbeat(unifiedClientStats);
      int ecnt = eventBuffer.readEvents(cpRbc);

      success = (ecnt > 0);

      if (!success)
      {
        _log.error("Unable to write bootstrap phase marker");
      } else {
        ChunkedBodyReadableByteChannel readChannel = curState.getReadChannel();

        String remoteErrorName = RemoteExceptionHandler.getExceptionName(readChannel);
        Throwable remoteError = _remoteExceptionHandler.getException(readChannel);
        if (null != remoteError &&
            remoteError instanceof BootstrapDatabaseTooOldException)
        {
          _log.error("Bootstrap database is too old!");
          _remoteExceptionHandler.handleException(remoteError);
          curState.switchToStreamResponseError();
        }
        else if (null != remoteErrorName)
        {
          //remote processing error
          _log.error("read events error: " + RemoteExceptionHandler.getExceptionMessage(readChannel));
          curState.switchToStreamResponseError();
        }
        else
        {
          sendHeartbeat(unifiedClientStats);
          int eventsNum = eventBuffer.readEvents(readChannel, curState.getListeners(),
                                                 _sourcesConn.getBootstrapEventsStatsCollector());

          if (eventsNum == 0 &&
              _remoteExceptionHandler.getPendingEventSize(readChannel) > eventBuffer.getMaxReadBufferCapacity())
          {
            String err = "ReadBuffer max capacity(" + eventBuffer.getMaxReadBufferCapacity() +
                         ") is less than event size(" +
                         _remoteExceptionHandler.getPendingEventSize(readChannel) +
                         "). Increase databus.client.connectionDefaults.bstEventBuffer.maxEventSize and restart.";
            _log.fatal(err);
            enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new PendingEventTooLargeException(err)));
            return;
          }
          else
          {
            resetServerRetries();

            if (debugEnabled) _log.debug("Sending events to buffer");

            numEventsInCurrentState += eventsNum;

            _log.info("Bootstrap events read so far: " + numEventsInCurrentState);

            String status = readChannel.getMetadata("PhaseCompleted");

            final BootstrapCheckpointHandler ckptHandler = curState.getBstCheckpointHandler();
            if (status != null)
            { // set status in checkpoint to indicate that we are done with the current source
              if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP)
              {
                ckptHandler.finalizeCatchupPhase(cp);
              }
              else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT)
              {
                ckptHandler.finalizeSnapshotPhase(cp);
              }
              else
              {
                 throw new RuntimeException("Invalid bootstrap phase: " + cp.getConsumptionMode());
              }

              _log.info("Bootstrap events read :" + numEventsInCurrentState + " during phase:"
                      + cp.getConsumptionMode() + " [" + cp.getBootstrapSnapshotSourceIndex()
                      + "," + cp.getBootstrapCatchupSourceIndex() + "]");
              numEventsInCurrentState = 0;
            }
            else
            { // keep on reading more for the given snapshot
              // question: how is snapshotOffset maintained in ckpt
              if (eventsNum > 0)
              {
                cp.bootstrapCheckPoint();
              }
            }

            curState.switchToStreamResponseDone();
          }
View Full Code Here

  protected void doStreamResponseDone(ConnectionState curState)
  {
    boolean debugEnabled = _log.isDebugEnabled();

    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled) _log.debug("Checkpoint at EventsDone: " + cp);

    determineNextStateFromCheckpoint(curState);
    // if we successfully got some data - reset retries counter.
    _retriesBeforeCkptCleanup.reset();
View Full Code Here

   */
  private void determineNextStateFromCheckpoint(ConnectionState curState)
  {
    try
    {
      final Checkpoint cp = curState.getCheckpoint();
      final BootstrapCheckpointHandler cpHandler = curState.getBstCheckpointHandler();
      cpHandler.assertBootstrapCheckpoint(cp);
      switch (cp.getConsumptionMode())
      {
      case BOOTSTRAP_SNAPSHOT:
        determineNextStateFromSnapshotCheckpoint(cp, cpHandler, curState);
        break;
      case BOOTSTRAP_CATCHUP:
View Full Code Here

  }

  private void doTestOneDataStreamCommand() throws Exception
  {
    //try to read it
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();

    String streamRequest = "/stream?sources=100&size=100000&output=json&checkPoint=" + cp.toString();

    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, streamRequest);

    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
View Full Code Here

  /** Validates the version checks in the stream calls. */
  private void doTestOneDataClientVerStreamCommand(int ver, boolean expectFail) throws Exception
  {
    //try to read it
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();

    String maxev = "&" + DatabusHttpHeaders.MAX_EVENT_VERSION + "=" + ver;
    // protocol version 2 (versions >= 3 use "subs=")
    String streamRequest = "/stream?sources=100&size=100000&output=json&checkPoint=" + cp.toString() + maxev;

    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, streamRequest);

    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
View Full Code Here

TOP

Related Classes of com.linkedin.databus.core.Checkpoint

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.