Examples of Checkpoint


Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

    }
  }

  private void testClient(int relayPort, int fetchSize, long scn, HttpResponseHandler handler) throws Exception
  {
    Checkpoint ckpt = Checkpoint.createOnlineConsumptionCheckpoint(scn);
    //TODO why is this needed
    //ckpt.setCatchupSource("foo");
    String uristr = "/stream?sources=105&output=json&size=" + fetchSize + "&streamFromLatestScn=false&checkPoint=" + ckpt.toString();
    ClientBootstrap bootstrap = new ClientBootstrap(new NioClientSocketChannelFactory(Executors.newCachedThreadPool(),
                                                                                      Executors.newCachedThreadPool()));
    bootstrap.setPipelineFactory(new HttpClientPipelineFactory(handler));
    ChannelFuture future = bootstrap.connect(new InetSocketAddress("localhost", relayPort));
    Channel channel = future.awaitUninterruptibly().getChannel();
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

        && !_dbusConnection.getConnectionStatus().isRunningStatus())
    {
      LOG.info("In RelayEventProducer start:  running =" + _dbusConnection.getConnectionStatus().isRunningStatus());
      //translate relay saved scn to client checkpoint
      LOG.info("Requested sinceSCN = " + sinceSCN);
      Checkpoint cp = getCheckpoint(sinceSCN, _scnReaderWriter);

      //check if the relay chaining consumer has been initialized [it could when the leader passes its active buffer]
      if ((cp != null) && (_consumerEventBuffer.getStartSCN() < 0))
      {
        //note that the restartScnOffset comes into the picture only iff the buffer is empty
        long savedScn = cp.getWindowScn();
        LOG.info("Checkpoint read = " + savedScn + " restartScnOffset=" + _restartScnOffset);

        long newScn = (savedScn >= _restartScnOffset) ? savedScn - _restartScnOffset : 0;
        cp.setWindowScn(newScn);

        LOG.info("Setting start scn of event buffer to " + cp.getWindowScn());
        _consumerEventBuffer.setStartSCN(cp.getWindowScn());
      }
      LOG.info("Eventbuffer start scn = " + _consumerEventBuffer.getStartSCN());

      //now set the checkpoint in the databus client fetcher
      _dbusConnection.getRelayPullThread().getConnectionState().setCheckpoint(cp);
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

    // return no cp if unable to read from saved SCN
    if (scn <= 0)
    {
      return null;
    }
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    // always have greater than semantic
    cp.setWindowOffset(-1);
    cp.setWindowScn(scn);
    return cp;
  }
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  }

  private static Checkpoint updateCheckpoint(Checkpoint cpOld)
          throws JsonParseException, JsonMappingException, IOException
  {
    Checkpoint cpNew = null != cpOld ? new Checkpoint(cpOld.toString()) : new Checkpoint();
    if (null != _scn)
    {
      if (-1L != _scn)
      {
        cpNew.setWindowScn(_scn);
        cpNew.setWindowOffset(0);
      }
      else
      {
        cpNew.setFlexible();
      }
    }

    if (null != _startScn)
    {
      cpNew.setBootstrapStartScn(_startScn);
    }

    if (null != _targetScn)
    {
      cpNew.setBootstrapTargetScn(_targetScn);
    }

    if (null != _cpType)
    {
      cpNew.setConsumptionMode(_cpType);
      switch (_cpType)
      {
        case ONLINE_CONSUMPTION: cpNew.setWindowOffset(0); break;
        /*
         * TODO Disabling as the bootstrap checkpoint creation leaves out important
         * information (e.g. catchup/snashot source index) out of the checkpoint
         * and thus is incorrect. We have to figure out what types of bootstrap
         * checkpoints it makes sense to create.
        case BOOTSTRAP_CATCHUP:
        {
          if (null != _bootstrapSource) cpNew.setCatchupSource(_bootstrapSource);
          cpNew.setCatchupOffset(-1);
          break;
        }*/
        case BOOTSTRAP_SNAPSHOT:
        {
          BootstrapCheckpointHandler handler = new BootstrapCheckpointHandler(_sources);
          cpNew = handler.createInitialBootstrapCheckpoint(cpNew, _sinceScn);
          //if (null != _bootstrapSource) cpNew.setSnapshotSource(_bootstrapSource);
          cpNew.setSnapshotOffset(-1);
          break;
        }
        default:
          throw new DatabusRuntimeException("unsupported checkpoint type: " + _cpType);
      }
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

      cp3 = new FileSystemCheckpointPersistenceProvider(
          clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }

    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action)
    {
      cpNew = updateCheckpoint(cpOld);
    }
    else if (Action.CHANGE == _action)
    {
      cpNew = updateCheckpoint(cpOld);

      cp3.storeCheckpoint(sourceList, cpNew);

      //reread as a sanity check
      cpNew = cp3.loadCheckpoint(sourceList);
    }
    else if (Action.DELETE == _action)
    {
      cp3.removeCheckpoint(sourceList);
      cpNew = cp3.loadCheckpoint(sourceList);
    }
    else
    {
      throw new RuntimeException("don't know what to do with action: " + _action);
    }

    if (null != cpOld) System.out.println("old: " + cpOld.toString());
    else System.out.println("old: null");

    if (null != cpNew) System.out.println("new: " + cpNew.toString());
    else System.out.println("new: null");

  }
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

                                .entrySet())
                        {
                            StringBuilder output = new StringBuilder(64);
                            output.append(mkPair.getKey());
                            output.append(delim);
                            Checkpoint cp = mkPair.getValue();
                            if (cp == null)
                            {
                                output.append(-1);
                                output.append(delim);
                                output.append(-1);
                            }
                            else
                            {
                                if (cp.getConsumptionMode() == DbusClientMode.ONLINE_CONSUMPTION)
                                {
                                    output.append(cp.getWindowScn());
                                    output.append(delim);
                                    output.append(cp.getWindowOffset());
                                }
                                else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP)
                                {
                                    output.append(cp.getWindowScn());
                                    output.append(delim);
                                    output.append(cp.getWindowOffset());
                                }
                                else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT)
                                {
                                    output.append(cp.getBootstrapSinceScn());
                                    output.append(delim);
                                    output.append(-1);
                                }
                            }
                            System.out.println(output.toString());
                        }
                    }
                    else
                    {
                        throw new DatabusClusterUtilException(
                                "readSCN: please specify non-empty sources");
                    }
                }
                else if (function.equals("checkpoint"))
                {
                    List<String> sources = getSources(arg1);
                    if ((sources != null) && !sources.isEmpty())
                    {
                        DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(
                                zkServer, clusterName, sources, partitions,
                                isLegacyChkptLocation);
                        Map<Integer, Checkpoint> ckpts = ckptMgr
                                .readCheckpoint();
                        char delim = '\t';
                        for (Map.Entry<Integer, Checkpoint> mkPair : ckpts
                                .entrySet())
                        {
                            StringBuilder output = new StringBuilder(64);
                            output.append(mkPair.getKey());
                            output.append(delim);
                            Checkpoint cp = mkPair.getValue();
                            if (cp == null)
                            {
                                output.append("null");
                            }
                            else
                            {
                                output.append(cp.toString());
                            }
                            System.out.println(output.toString());
                        }
                    }
                    else
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  public Checkpoint loadPersistentCheckpoint()
  {
    if (_checkpointPersistenceProvider != null)
      return _checkpointPersistenceProvider.loadCheckpointV3(
          getSubscriptions(), _registrationId);
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    return cp;
  }
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

            {
                for (int p : _partitions)
                {
                    ClusterCheckpointPersistenceProvider cpProvider = new ClusterCheckpointPersistenceProvider(
                            p, _clusterConfig);
                    Checkpoint cp = new Checkpoint();
                    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
                    cp.setWindowOffset(-1);
                    cp.setWindowScn(scn);
                    if (_isLegacyCkptLocation)
                    {
                        cpProvider.storeCheckpointLegacy(_sources, cp);
                    }
                    else
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

          {
            for (String k: keyList)
            {
              if (!k.equals(DatabusClientDSCUpdater.DSCKEY))
              {
                Checkpoint cp = (Checkpoint) member.readSharedData(k);
                if (cp != null)
                {
                  System.out.println(k + " " + cp.getWindowScn() + " "+cp.getWindowOffset());
                }
                else
                {
                  System.err.println(k + " null null");
                }
              }
            }
          }
           
        } else if (function.equals("writeSCN")) {
          if (arg1 != null && arg2 != null)
          {
            Checkpoint cp = new Checkpoint();
            cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
            cp.setWindowScn(Long.parseLong(arg2));
            if (fns.length > 3)
            {
              cp.setWindowOffset(Integer.parseInt(fns[3]));
            } else {
              cp.setWindowOffset(-1);
            }
            if (member.writeSharedData(arg1, cp))
            {
              System.out.println(arg1 + " " + cp.getWindowScn() + " " + cp.getWindowOffset());
            }
            else
            {
              System.err.println("Write failed! " + member +  " couldn't write key=" + arg1);
              System.exit(1);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.