Package com.linkedin.databus.client.pub

Examples of com.linkedin.databus.client.pub.CheckpointPersistenceProvider


    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");


    CheckpointPersistenceProvider cp3 = null;
    if (null != _cp3Props)
    {
      CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder =
          new CheckpointPersistenceStaticConfigBuilder();
      ConfigLoader<CheckpointPersistenceStaticConfig> configLoader =
          new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(
              _propPrefix,
              cp3ConfBuilder);
      configLoader.loadConfig(_cp3Props);
      CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
      if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM)
      {
        throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
      }

      cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
    }
    else if (null != _clientProps)
    {
      DatabusHttpClientImpl.Config clientConfBuilder =
          new DatabusHttpClientImpl.Config();
      ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader =
          new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
      configLoader.loadConfig(_clientProps);

      DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
      if (clientConf.getCheckpointPersistence().getType() !=
          CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM)
      {
        throw new RuntimeException("don't know what to do with cp3 type:" +
                                   clientConf.getCheckpointPersistence().getType());
      }

      cp3 = new FileSystemCheckpointPersistenceProvider(
          clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }

    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action)
    {
      cpNew = updateCheckpoint(cpOld);
    }
    else if (Action.CHANGE == _action)
    {
      cpNew = updateCheckpoint(cpOld);

      cp3.storeCheckpoint(sourceList, cpNew);

      //reread as a sanity check
      cpNew = cp3.loadCheckpoint(sourceList);
    }
    else if (Action.DELETE == _action)
    {
      cp3.removeCheckpoint(sourceList);
      cpNew = cp3.loadCheckpoint(sourceList);
    }
    else
    {
      throw new RuntimeException("don't know what to do with action: " + _action);
    }
View Full Code Here


    }

    public CheckpointPersistenceProvider getOrCreateCheckpointPersistenceProvider(DatabusClientGroupMember groupMember)
           throws InvalidConfigException
    {
      CheckpointPersistenceProvider cpPersistenceProvider = null;
      switch (getType())
      {
        case FILE_SYSTEM:
        LOG.info("Creating file-system checkpoint persistence provider");
        cpPersistenceProvider =
View Full Code Here

        if (!connConfig.getEventBuffer().isEnableScnIndex() &&
          connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE)
        {
          throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
        }
        CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
        if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse())
        {
          cpPersistenceProvder.removeCheckpoint(sourcesStrList);
        }

        ServerInfo server0 = _relayGroups.get(subsList).iterator().next();

        ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs =
View Full Code Here

    public void setManagedInstance(DatabusHttpClientImpl managedInstance)
    {
      _managedInstance = managedInstance;
      if (null != _managedInstance)
      {
        CheckpointPersistenceProvider persistenceProvider =
            _managedInstance.getCheckpointPersistenceProvider();
        if (persistenceProvider instanceof FileSystemCheckpointPersistenceProvider)
        {
          _fileSystem.setManagedInstance((FileSystemCheckpointPersistenceProvider)persistenceProvider);
        }
View Full Code Here

        throw new DatabusClientException("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers");
      }

      // Create Registration
      RegistrationId id = new RegistrationId(_id + "-" + partition.getPartitionId());
      CheckpointPersistenceProvider ckptProvider = createCheckpointPersistenceProvider(partition);

      DatabusV2RegistrationImpl reg = createChildRegistration(id, _client, ckptProvider);
      reg.addDatabusConsumers(consumers);

      String[] srcs = new String[_sources.size()];
View Full Code Here

            "Got exception while trying to fetch SCN from bootstrap_producer_state for sources :"
                + sourceNames, ex);
        throw ex;
      }

      CheckpointPersistenceProvider provider = getCheckpointPersistenceProvider();
      Checkpoint cp = provider.loadCheckpoint(sourceNames);
      LOG.info("Bootstrap Producer SCN :" + scn + ", Checkpoint :" + cp);

      if (null != cp)
      {
        if (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)
        {
          // Bootstrapping bootstrap Producer not yet supported !!
          String msg = "Bootstrap Producer starting from non-online consumption mode for sources :"
              + sourceNames + ", Ckpt :" + cp;
          LOG.error(msg);
          throw new BootstrapDBException(msg);
        }
        else
        {
          String msg = null;
          if (((cp.getWindowScn() > scn) && (scn > -1))
              || ((cp.getWindowScn() < scn) && (scn > -1)))
          {

            if (((cp.getWindowScn() > scn) && (scn > -1)))
              LOG.warn("Non-Empty checkpint. Bootstrap Producer is at SCN:"
                  + scn
                  + ", while checkpoint is :"
                  + cp
                  + ", Could result in gap in event consumption. Repairing ckpt !!");
            else
              LOG.info("Non-Empty checkpoint. Bootstrap Producer is at SCN:"
                  + scn + ", while checkpoint is :" + cp
                  + ", Copying producer Scn to checkpoint !!");

            cp.setWindowScn(scn);
            cp.setWindowOffset(-1);
            try
            {
              provider.removeCheckpoint(sourceNames);
              provider.storeCheckpoint(sourceNames, cp);

              // Check if persisted properly
              cp = provider.loadCheckpoint(sourceNames);
              if ((null == cp)
                  || (cp.getWindowScn() != scn)
                  || (cp.getWindowOffset() != -1)
                  || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION))
              {
                msg = "Unable to repair and store the new checkpoint (" + cp
                    + ") to make it same as producer SCN (" + scn + ") !!";
                LOG.fatal(msg);
                throw new BootstrapDBException(msg);
              }
            } catch (IOException ex)
            {
              msg = "Unable to repair and store the new checkpoint (" + cp
                  + ") to make it same as producer SCN (" + scn + ") !!";
              LOG.fatal(msg, ex);
              throw new BootstrapDBException(msg);
            }
          }
        }
      }
      else
      {
        /**
         * Currently since bootstrapping is not available, a null ckpt would
         * result in flexible checkpoint and could result in gap !!
         */
        if (scn > -1)
        {
          String msg = "Empty checkpoint. Bootstrap Producer SCN is at SCN:"
              + scn
              + ", while checkpoint is null !! Could result in gap in event consumption. Repairing ckpt !!";
          LOG.warn(msg);
          cp = new Checkpoint();
          cp.setWindowScn(scn);
          cp.setWindowOffset(-1);
          cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
          try
          {
            provider.removeCheckpoint(sourceNames);
            provider.storeCheckpoint(sourceNames, cp);

            // Check if persisted properly
            cp = provider.loadCheckpoint(sourceNames);
            if ((null == cp)
                || (cp.getWindowScn() != scn)
                || (cp.getWindowOffset() != -1)
                || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION))
            {
View Full Code Here

TOP

Related Classes of com.linkedin.databus.client.pub.CheckpointPersistenceProvider

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.