Package com.linkedin.databus2.core.filter

Examples of com.linkedin.databus2.core.filter.KeyFilterConfigHolder


        if (null != bootstrapStatsCollector)
        {
          bootstrapStatsCollector.registerErrBootstrap();
        }

        throw new RequestProcessingException(e);
      }

      DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
      if (!componentStatus.isRunningStatus())
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
          if ( isDebug) LOG.debug("ServerSideFilter is :" + keyFilter);
        } catch ( Exception ex) {
          String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
          LOG.error(msg,ex);
          throw new RequestProcessingException(msg,ex);
        }
      }

      String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
      Encoding enc = Encoding.BINARY;

      if ( null != outputFormat)
      {
        try
        {
          enc = Encoding.valueOf(outputFormat.toUpperCase());
        } catch (Exception ex) {
          LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
        }
      }

      processor.setKeyFilter(keyFilter);
      String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);

      int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
      if ( null != _serverHostPort)
      {
        bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
      }

      int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;

        BootstrapEventWriter writer = null;
        if(_config.getPredicatePushDown())
          writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
          writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);

        Checkpoint cp = new Checkpoint(checkpointString);


      DbusClientMode consumptionMode = cp.getConsumptionMode();

      LOG.info("Bootstrap request received: " +
          "fetchSize=" + clientFreeBufferSize +
          ", consumptionMode=" + consumptionMode +
          ", checkpoint=" + checkpointString +
          ", predicatePushDown=" + _config.getPredicatePushDown()
          );

      try
      {
        boolean phaseCompleted = false;
        switch (consumptionMode)
        {
        case BOOTSTRAP_SNAPSHOT:
          phaseCompleted = processor.streamSnapShotRows(new Checkpoint(
              checkpointString), writer);
          break;
        case BOOTSTRAP_CATCHUP:
          phaseCompleted = processor.streamCatchupRows(new Checkpoint(
              checkpointString), writer);
          break;
        default:
          if (null != bootstrapStatsCollector)
            bootstrapStatsCollector.registerErrBootstrap();

          throw new RequestProcessingException("Unexpected mode: "
              + consumptionMode);
        }

        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis()-startTime, clientFreeBufferSize);

        if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0)
        {
          // Append a header to indicate to the client that we do have at least one event to
          // send, but it is too large to fit into client's offered buffer.
          request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE,
                                                   writer.getSizeOfPendingEvent());
          if (isDebug)
          {
            LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
          }
        }
        if (phaseCompleted)
        {
          request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
        }

      }
      catch (BootstrapDatabaseTooOldException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrDatabaseTooOld();

        LOG.error("Bootstrap database is too old!", e);
        throw new RequestProcessingException(e);
      }
      catch (BootstrapDBException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();
        throw new RequestProcessingException(e);
      }
      catch (SQLException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrSqlException();

        throw new RequestProcessingException(e);
      }
      catch (BootstrapProcessingException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

        throw new RequestProcessingException(e);
      }
    }
    finally
    {
      if ( null != processor)
View Full Code Here


            if ((null != expServerInfo) && (! _serverInfo.equals(expServerInfo)))
            {
              String msg = "Bootstrap Server Request should be served by different host : " + bsServerInfo + ", This instance is :" + _serverHostPort;
              LOG.error(msg);
              throw new RequestProcessingException(msg);
            }
          }
        }

        DatabusRequest req = doProcess(request);
View Full Code Here

  protected void initializeBootstrapServerCommandProcessors() throws DatabusException
  {
    LOG.info("Initializing Bootstrap HTTP Server");
    LOG.info("Config=" + _bootstrapServerConfig);
    try{
      RequestProcessorRegistry processorRegistry = getProcessorRegistry();
      processorRegistry.register(ConfigRequestProcessor.COMMAND_NAME,
                                 new ConfigRequestProcessor(null, this));
      processorRegistry.register(BootstrapRequestProcessor.COMMAND_NAME,
                                 new BootstrapRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(StartSCNRequestProcessor.COMMAND_NAME,
                                 new StartSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(TargetSCNRequestProcessor.COMMAND_NAME,
                                 new TargetSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(ContainerOperationProcessor.COMMAND_NAME,
                                 new ContainerOperationProcessor(null, this));
    }
    catch (SQLException sqle)
    {
      throw new DatabusException("command registration failed", sqle);
View Full Code Here

      _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(
          null, this, plist);
    else
      _csEventRequestProcessor.addEventProducers(plist);

    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(
        ControlSourceEventsRequestProcessor.COMMAND_NAME,
        _csEventRequestProcessor);
  }
View Full Code Here

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
View Full Code Here


    DbusKeyCompositeFilter filter = conn.getRelayFilter();
    Assert.assertNotNull(filter);
    // get DbusKeyFilter that we put in
    KeyFilterConfigHolder configHolder = filterConfig.getConfigMap().values().iterator().next();
    DbusKeyFilter inFilter = new DbusKeyFilter(configHolder);

    // get filter from the connection
    Assert.assertEquals(filter.getFilterMap().size(), 1, "1 filter in the composite filter");
    Assert.assertTrue(filter.getFilterMap().entrySet().iterator().hasNext(), "has one filter");
View Full Code Here

    rangeConf.setSize(100);
    rangeConf.setPartitions("[3-4,3-4]");
    partConf2.setRange(rangeConf);

    HashMap<Long, KeyFilterConfigHolder> partConfigMap = new HashMap<Long, KeyFilterConfigHolder>();
    partConfigMap.put(1L, new KeyFilterConfigHolder(partConf1.build()));
    partConfigMap.put(2L, new KeyFilterConfigHolder(partConf2.build()));

    List<Long> keys1 = new ArrayList<Long>();
    List<Long> keys2 = new ArrayList<Long>();
    List<Long> keys3 = new ArrayList<Long>();
View Full Code Here

  {

    KeyFilterConfigHolder.Config partConf =  new KeyFilterConfigHolder.Config();
    partConf.setType("NONE");

    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0 ; i < 1000 ;i++)
    {
      keys.add(i);
View Full Code Here

        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(5);
        modConf.setBuckets("[0,3-9]"); //invalid config
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
      } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
      }

      assertEquals("Got Exception for invalid Config (MaxBucket is more than numBuckets) ", true, isException);

      //Error Case : Min Bucket is more than maxBucket
      isException = false;
      try
      {
        KeyFilterConfigHolder.Config partConf =  new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        modConf.setBuckets("[0,9-3]"); //invalid config
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
      } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
      }
      assertEquals("Got Exception for invalid Config (Min Bucket is more than maxBucket) ", true, isException);


      //Error Case : numBuckets is negative
      isException = false;
      try
      {
        KeyFilterConfigHolder.Config partConf =  new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(-5);
        modConf.setBuckets("[0]"); //invalid config
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
      } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
      }
      assertEquals("Got Exception for invalid Config numBuckets is negative) ", true, isException);


      //Error Case : minBucket is negative
      isException = false;
      try
      {
        KeyFilterConfigHolder.Config partConf =  new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        modConf.setBuckets("[-5,1-3]"); //invalid config
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
      } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
      }
      assertEquals("Got Exception for invalid Config (minBucket is negative) ", true, isException);
View Full Code Here

    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));

    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0 ; i < 1000 ;i++)
    {
View Full Code Here

TOP

Related Classes of com.linkedin.databus2.core.filter.KeyFilterConfigHolder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.