Package com.linkedin.databus2.core.filter

Examples of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig


        if (null != bootstrapStatsCollector)
        {
          bootstrapStatsCollector.registerErrBootstrap();
        }

        throw new RequestProcessingException(e);
      }

      DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
      if (!componentStatus.isRunningStatus())
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
          if ( isDebug) LOG.debug("ServerSideFilter is :" + keyFilter);
        } catch ( Exception ex) {
          String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
          LOG.error(msg,ex);
          throw new RequestProcessingException(msg,ex);
        }
      }

      String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
      Encoding enc = Encoding.BINARY;

      if ( null != outputFormat)
      {
        try
        {
          enc = Encoding.valueOf(outputFormat.toUpperCase());
        } catch (Exception ex) {
          LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
        }
      }

      processor.setKeyFilter(keyFilter);
      String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);

      int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
      if ( null != _serverHostPort)
      {
        bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
      }

      int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;

        BootstrapEventWriter writer = null;
        if(_config.getPredicatePushDown())
          writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
          writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);

        Checkpoint cp = new Checkpoint(checkpointString);


      DbusClientMode consumptionMode = cp.getConsumptionMode();

      LOG.info("Bootstrap request received: " +
          "fetchSize=" + clientFreeBufferSize +
          ", consumptionMode=" + consumptionMode +
          ", checkpoint=" + checkpointString +
          ", predicatePushDown=" + _config.getPredicatePushDown()
          );

      try
      {
        boolean phaseCompleted = false;
        switch (consumptionMode)
        {
        case BOOTSTRAP_SNAPSHOT:
          phaseCompleted = processor.streamSnapShotRows(new Checkpoint(
              checkpointString), writer);
          break;
        case BOOTSTRAP_CATCHUP:
          phaseCompleted = processor.streamCatchupRows(new Checkpoint(
              checkpointString), writer);
          break;
        default:
          if (null != bootstrapStatsCollector)
            bootstrapStatsCollector.registerErrBootstrap();

          throw new RequestProcessingException("Unexpected mode: "
              + consumptionMode);
        }

        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis()-startTime, clientFreeBufferSize);

        if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0)
        {
          // Append a header to indicate to the client that we do have at least one event to
          // send, but it is too large to fit into client's offered buffer.
          request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE,
                                                   writer.getSizeOfPendingEvent());
          if (isDebug)
          {
            LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
          }
        }
        if (phaseCompleted)
        {
          request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
        }

      }
      catch (BootstrapDatabaseTooOldException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrDatabaseTooOld();

        LOG.error("Bootstrap database is too old!", e);
        throw new RequestProcessingException(e);
      }
      catch (BootstrapDBException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();
        throw new RequestProcessingException(e);
      }
      catch (SQLException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrSqlException();

        throw new RequestProcessingException(e);
      }
      catch (BootstrapProcessingException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

        throw new RequestProcessingException(e);
      }
    }
    finally
    {
      if ( null != processor)
View Full Code Here


            if ((null != expServerInfo) && (! _serverInfo.equals(expServerInfo)))
            {
              String msg = "Bootstrap Server Request should be served by different host : " + bsServerInfo + ", This instance is :" + _serverHostPort;
              LOG.error(msg);
              throw new RequestProcessingException(msg);
            }
          }
        }

        DatabusRequest req = doProcess(request);
View Full Code Here

  protected void initializeBootstrapServerCommandProcessors() throws DatabusException
  {
    LOG.info("Initializing Bootstrap HTTP Server");
    LOG.info("Config=" + _bootstrapServerConfig);
    try{
      RequestProcessorRegistry processorRegistry = getProcessorRegistry();
      processorRegistry.register(ConfigRequestProcessor.COMMAND_NAME,
                                 new ConfigRequestProcessor(null, this));
      processorRegistry.register(BootstrapRequestProcessor.COMMAND_NAME,
                                 new BootstrapRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(StartSCNRequestProcessor.COMMAND_NAME,
                                 new StartSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(TargetSCNRequestProcessor.COMMAND_NAME,
                                 new TargetSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(ContainerOperationProcessor.COMMAND_NAME,
                                 new ContainerOperationProcessor(null, this));
    }
    catch (SQLException sqle)
    {
      throw new DatabusException("command registration failed", sqle);
View Full Code Here

      _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(
          null, this, plist);
    else
      _csEventRequestProcessor.addEventProducers(plist);

    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(
        ControlSourceEventsRequestProcessor.COMMAND_NAME,
        _csEventRequestProcessor);
  }
View Full Code Here

  @Test
  public void testRelayPullerThreadV3WithFilter() throws Exception
  {
    RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
    // create filter
    DbusKeyCompositeFilterConfig filterConfig = createFakeFilter();
    bldr.setFilterConfig(filterConfig);
    RelayPullThread relayPuller = bldr.createRelayPullThread();
    relayPuller.getComponentStatus().start();
    ConnectionState connState = relayPuller.getConnectionState();
    connState.switchToPickServer();
    testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);

    // get the created connection
    MockRelayConnection conn = (MockRelayConnection) relayPuller.getLastOpenConnection();
    conn.setProtocolVersion(3);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);


    DbusKeyCompositeFilter filter = conn.getRelayFilter();
    Assert.assertNotNull(filter);
    // get DbusKeyFilter that we put in
    KeyFilterConfigHolder configHolder = filterConfig.getConfigMap().values().iterator().next();
    DbusKeyFilter inFilter = new DbusKeyFilter(configHolder);

    // get filter from the connection
    Assert.assertEquals(filter.getFilterMap().size(), 1, "1 filter in the composite filter");
    Assert.assertTrue(filter.getFilterMap().entrySet().iterator().hasNext(), "has one filter");
View Full Code Here

    List<DbusKeyCompositeFilterConfig> relayFilterConfigs = new ArrayList<DbusKeyCompositeFilterConfig>();
    List<DbusKeyCompositeFilterConfig> bootstrapFilterConfigs = new ArrayList<DbusKeyCompositeFilterConfig>();

    if (null != registrations) {
      for (DatabusV2ConsumerRegistration reg : registrations) {
        DbusKeyCompositeFilterConfig conf = reg.getFilterConfig();

        if (null != conf)
          relayFilterConfigs.add(conf);
      }
    }

    if (null != bootstrapRegistrations) {
      for (DatabusV2ConsumerRegistration reg : bootstrapRegistrations) {
        DbusKeyCompositeFilterConfig conf = reg.getFilterConfig();

        if (null != conf)
          bootstrapFilterConfigs.add(conf);
      }
    }
View Full Code Here

      }
      bootstrapBuilder.setSources(sourcesString.toString());
    }

    // handler server side filtering, can either pass a config file or set from command line
    DbusKeyCompositeFilterConfig filterConfig = createServerSideFilterConfig(_filterConfFile, startupProps);

    // set up listeners
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);

    // dump decoded payload values and raw (undecoded) events
View Full Code Here

      DbusKeyCompositeFilterConfig.Config conf = new DbusKeyCompositeFilterConfig.Config();
      ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig> filterConfigLoader =
        new ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig>(SERVER_SIDE_FILTER_PREFIX, conf);
      DbusKeyCompositeFilterConfig.StaticConfig sConf = filterConfigLoader.loadConfig(props);

      return new DbusKeyCompositeFilterConfig(sConf);
    }
    else
    {
      return null;
    }
View Full Code Here

        return;
      }

      // Call factories to get consumer callbacks and server-side filter config.
      Collection<DatabusCombinedConsumer> consumers = _consumerFactory.createPartitionedConsumers(_clusterInfo, partition);
      DbusKeyCompositeFilterConfig filterConfig = null;

      if (_serverSideFilterFactory != null)
        filterConfig = _serverSideFilterFactory.createServerSideFilter(_clusterInfo, partition);

View Full Code Here

                                                             cli.getOut()); break;
      default: throw new InvalidConfigException("unsupported output format: " + cli.getOutputFormat());
      }
    }

    DbusKeyCompositeFilterConfig filterConfig = null;

    if (_cli.getModPartBase() > 0)
    {
      DbusKeyCompositeFilterConfig.Config builder = new DbusKeyCompositeFilterConfig.Config();
      for (String src: cli.getSources())
      {
        builder.getFilter(src).setType(PartitionType.MOD.toString());
        builder.getFilter(src).getMod().setNumBuckets(cli.getModPartBase());
        builder.getFilter(src).getMod().setBuckets(cli.getModPartIds());
      }
      filterConfig = new DbusKeyCompositeFilterConfig(builder.build());
    }

    _client.registerDatabusStreamListener(_consumer, filterConfig, cli.getSources());
    _client.registerDatabusBootstrapListener(_consumer, filterConfig, cli.getSources());
View Full Code Here

TOP

Related Classes of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.