Examples of IndexerConf

com.ngdata.hbaseindexer.conf.IndexerConf
The configuration for an indexer, i.e. this defines the behavior of the {@link Indexer} and of the parser/mappercalled by the indexer.
Instances of IndexerConf can be created using {@link IndexerConfBuilder} or from XML using an{@link IndexerComponentFactory}.

Examples of com.ngdata.hbaseindexer.conf.IndexerConf

        verifyZeroInteractions(tableA, tableB);
    }


    @Test
    public void testColumnBasedIndexing() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).mappingType(IndexerConf.MappingType.COLUMN).build();


        ResultToSolrMapper mapper = new ResultToSolrMapper() {
            @Override
            public boolean isRelevantKV(KeyValue kv) {
                return Bytes.toString(kv.getFamily()).equals("messages");

View Full Code Here

Examples of com.ngdata.hbaseindexer.conf.IndexerConf

        final HTableInterface multitableA = mock(HTableInterface.class);
        final HTableInterface multitableB = mock(HTableInterface.class);
        when(tablePool.getTable(multiTableAName.getBytes(Charsets.UTF_8))).thenReturn(multitableA);
        when(tablePool.getTable(multiTableBName.getBytes(Charsets.UTF_8))).thenReturn(multitableB);


        final IndexerConf indexerConf = new IndexerConfBuilder().table(tablePrefix + ".*").build();
        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);
        Indexer indexer = Indexer.createIndexer("index name", indexerConf, tablePrefix+ ".*", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, tablePrefix+ ".*", true);
        List<KeyValue> kvs = Lists.newArrayList(
                new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("messages"), Bytes.toBytes("msg1"),

View Full Code Here

Examples of com.ngdata.hbaseindexer.conf.IndexerConf

            indexerProcessId = indexerProcessRegistry.registerIndexerProcess(indexerDef.getName(), hostName);
            indexerProcessIds.put(indexerDef.getName(), indexerProcessId);


            // Create and register the indexer
            IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDef.getConfiguration()), indexerDef.getConnectionParams());
            IndexerConf indexerConf = factory.createIndexerConf();


            ResultToSolrMapper mapper = factory.createMapper(indexerDef.getName());


            Sharder sharder = null;
            SolrInputDocumentWriter solrWriter;
            PoolingClientConnectionManager connectionManager = null;


            if (indexerDef.getConnectionType() == null || indexerDef.getConnectionType().equals("solr")) {
                Map<String, String> connectionParams = indexerDef.getConnectionParams();
                String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams);
                if (solrMode.equals("cloud")) {
                    solrWriter = new DirectSolrInputDocumentWriter(indexerDef.getName(), createCloudSolrServer(connectionParams));
                } else if (solrMode.equals("classic")) {
                    connectionManager = new PoolingClientConnectionManager();
                    connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(connectionParams));
                    connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(connectionParams));


                    httpClient = new DefaultHttpClient(connectionManager);
                    List<SolrServer> solrServers = createHttpSolrServers(connectionParams, httpClient);
                    solrWriter = new DirectSolrClassicInputDocumentWriter(indexerDef.getName(), solrServers);
                    sharder = createSharder(connectionParams, solrServers.size());
                } else {
                    throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode);
                }
            } else {
                throw new RuntimeException(
                        "Invalid connection type: " + indexerDef.getConnectionType() + ". Only 'solr' is supported");
            }


            Indexer indexer = Indexer.createIndexer(indexerDef.getName(), indexerConf, indexerConf.getTable(),
                    mapper, htablePool, sharder, solrWriter);
            IndexingEventListener eventListener = new IndexingEventListener(
                    indexer, indexerConf.getTable(), indexerConf.tableNameIsRegex());


            int threads = hbaseConf.getInt("hbaseindexer.indexer.threads", 10);
            SepConsumer sepConsumer = new SepConsumer(indexerDef.getSubscriptionId(),
                    indexerDef.getSubscriptionTimestamp(), eventListener, threads, hostName,
                    zk, hbaseConf, null);

View Full Code Here

Examples of com.ngdata.hbaseindexer.conf.IndexerConf

        Map<String, IndexInfo> newIndexInfos = new HashMap<String, IndexInfo>();


        Collection<IndexerDefinition> indexDefs = indexerModel.getIndexers();
        for (IndexerDefinition indexDef : indexDefs) {
            byte[] indexerConfXml = indexDef.getConfiguration();
            IndexerConf indexerConf = null;


            try {
                // check if this is a lily index
                // FIXME: check on class name in a non-dependency module
                if (!"org.lilyproject.indexer.hbase.mapper.LilyIndexerComponentFactory".equals(indexDef.getIndexerComponentFactory())) {

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.