Package com.ngdata.hbaseindexer.model.api

Examples of com.ngdata.hbaseindexer.model.api.WriteableIndexerModel


    public void testBasicScenario() throws Exception {
        // Create a table in HBase
        createTable("table1", "family1");

        // Add an indexer
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(
                        Bytes.toBytes("<indexer table='table1'><field name='field1_s' value='family1:qualifier1'/></indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                                                  "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        // Ingest
        HTable table = new HTable(conf, "table1");
        Put put = new Put(b("row1"));
        put.add(b("family1"), b("qualifier1"), b("value1"));
View Full Code Here


    public void testBasicScenario_RegexTableExpression() throws Exception {
        // Create a table in HBase
        createTable("table1", "family1");

        // Add an indexer
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(
                        Bytes.toBytes("<indexer table='regex:table\\d+'><field name='field1_s' " +
                                              "value='family1:qualifier1'/></indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                                                  "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        // Ingest
        HTable table = new HTable(conf, "table1");
        Put put = new Put(b("row1"));
        put.add(b("family1"), b("qualifier1"), b("value1"));
View Full Code Here

    @Test
    public void testIndexerDefinitionChangesPickedUp() throws Exception {
        createTable("table1", "family1");

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(
                        Bytes.toBytes(("<indexer table='table1'><field name='field1_s' " +
                                          "value='family1:qualifier1'/></indexer>")))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        HTable table = new HTable(conf, "table1");
        Put put = new Put(b("row1"));
        put.add(b("family1"), b("qualifier1"), b("value1"));
        table.put(put);

        // Also put a value which should not be processed by the current config
        put = new Put(b("row2"));
        put.add(b("family1"), b("qualifier2"), b("value1"));
        table.put(put);

        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        SepTestUtil.waitOnReplication(conf, 60000L);

        collection1.commit();
        QueryResponse response = collection1.query(new SolrQuery("*:*"));
        assertEquals(1, response.getResults().size());

        // update indexer model
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "<field name='field2_s' value='family1:qualifier2'/>" +
                        "</indexer>"))
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        put = new Put(b("row3"));
        put.add(b("family1"), b("qualifier2"), b("value1"));
        table.put(put);
View Full Code Here

        createTable("table2", "family1");

        assertEquals(0, collection1.query(new SolrQuery("*:*")).getResults().size());
        assertEquals(0, collection2.query(new SolrQuery("*:*")).getResults().size());

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        indexerDef = new IndexerDefinitionBuilder()
                .name("indexer2")
                .configuration(Bytes.toBytes("<indexer table='table2'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection2"))
                .build();

        indexerModel.addIndexer(indexerDef);

        HTable table1 = new HTable(conf, "table1");
        HTable table2 = new HTable(conf, "table2");

        Put put = new Put(b("row1"));
View Full Code Here

    public void testIndexerIncrementalIndexingStates() throws Exception {
        createTable("table1", "family1");

        // First create an index in the default incremental indexing state (SUBSCRIBE_AND_CONSUME)
        markEventCounts();
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();
        indexerModel.addIndexer(indexerDef);
        // wait for 2 events because: first the indexer is added (= first event), then IndexerMaster
        // updates it to assign subscription (= second event), and only then IndexerSupervisor will start the indexer
        waitOnEventsProcessed(2);

        // Make sure the SEP and indexer processes were started
        assertEquals(1, main.getIndexerSupervisor().getRunningIndexers().size());
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));

        // Now change to DO_NOT_SUBSCRIBE_STATE
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.DO_NOT_SUBSCRIBE)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Verify master removed the SEP subscription and unassigned the subscription ID
        SepTestUtil.waitOnReplicationPeerStopped(peerId("indexer1"));
        assertNull(indexerModel.getFreshIndexer("indexer1").getSubscriptionId());

        // Verify supervisor stopped the indexer
        assertEquals(0, main.getIndexerSupervisor().getRunningIndexers().size());

        // Change to SUBSCRIBE_DO_NOT_CONSUME
        markEventCounts();
        lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Verify master registered the SEP subscription and assigned the subscription ID
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        assertNotNull(indexerModel.getFreshIndexer("indexer1").getSubscriptionId());

        // Verify supervisor didn't start the indexer (because in state "do no consume")
        assertEquals(0, main.getIndexerSupervisor().getRunningIndexers().size());

        // Change again to default SUBSCRIBE_AND_CONSUME
        markEventCounts();
        lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_AND_CONSUME)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Verify supervisor started the indexer
        assertEquals(1, main.getIndexerSupervisor().getRunningIndexers().size());
    }
View Full Code Here

    public void testDeleteIndexer() throws Exception {
        createTable("table1", "family1");

        // Create an index
        markEventCounts();
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);
        waitOnEventsProcessed(2);

        // Make sure the SEP and indexer processes were started
        assertEquals(1, main.getIndexerSupervisor().getRunningIndexers().size());
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));

        // Now request to delete the indexer, this is done by changing the lifecycle state to DELETE_REQUESTED
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .lifecycleState(IndexerDefinition.LifecycleState.DELETE_REQUESTED)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Check index was removed
        try {
            indexerModel.getFreshIndexer("indexer1");
            fail("expected an IndexerNotFoundException");
        } catch (IndexerNotFoundException e) {
            // expected
        }
View Full Code Here

        }
       
        // Ensure that the index is added on a different timestamp than the last put
        Thread.sleep(5);

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();
        indexerModel.addIndexer(indexerDef);

        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        SepTestUtil.waitOnReplication(conf, 60000L);

        // Check that records created before the indexer was added are not indexed
View Full Code Here

        Mockito.verify(listener, Mockito.times(builds)).onBatchBuild(Mockito.any(IndexerDefinition.class));
    }


    private void createIndexer1(String indexerConf) throws Exception {
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes(indexerConf.toString()))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                                                  "solr.collection", "collection1"))
                .build();
        indexerModel.addIndexer(indexerDef);
    }
View Full Code Here

    @PUT
    @Path("{name}")
    @Consumes("application/json")
    @Produces("application/json")
    public IndexerDefinition put(@PathParam("name") String indexName, ObjectNode json) throws Exception {
        WriteableIndexerModel model = getModel();
        ObjectMapper m = new ObjectMapper();

        IndexerDefinition oldIndexer = model.getIndexer(indexName);
        IndexerDefinition indexerDefinition = IndexerDefinitionJsonSerDeser.INSTANCE.fromJson(json,
                new IndexerDefinitionBuilder().startFrom(oldIndexer)).build();

        IndexerDefinition.LifecycleState lifecycleState = json.has("lifecycleState") ?
                IndexerDefinition.LifecycleState.valueOf(json.get("lifecycleState").getTextValue()) : null;

        String lock = model.lockIndexer(indexName);
        try {
            if (!oldIndexer.equals(indexerDefinition)) {
                model.updateIndexer(indexerDefinition, lock);
                //System.out.println("Index updated: " + indexName);
            } else {
                //System.out.println("Index already matches the specified settings, did not update it.");
            }
        } finally {
            // In case we requested deletion of an index, it might be that the lock is already removed
            // by the time we get here as part of the index deletion.
            boolean ignoreMissing = lifecycleState != null && lifecycleState == IndexerDefinition.LifecycleState.DELETE_REQUESTED;
            model.unlockIndexer(lock, ignoreMissing);
        }

        return indexerDefinition;
    }
View Full Code Here

    @Test
    public void testEvents() throws Exception {
        ZooKeeperItf zk1 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        ZooKeeperItf zk2 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        WriteableIndexerModel model1 = null;
        WriteableIndexerModel model2 = null;
        try {
            TestListener listener = new TestListener();

            model1 = new IndexerModelImpl(zk1, "/test");
            model1.registerListener(listener);

            // Create an indexer -- verify INDEXER_ADDED event
            IndexerDefinition indexer1 = new IndexerDefinitionBuilder()
                    .name("indexer1")
                    .configuration("my-conf".getBytes("UTF-8"))
                    .build();
            model1.addIndexer(indexer1);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_ADDED, "indexer1"));

            // Verify that a fresh indexer model has the index
            model2 = new IndexerModelImpl(zk2, "/test");
            Collection<IndexerDefinition> indexers = model2.getIndexers();
            assertEquals("Expected indexer1, got " + indexers, 1, indexers.size());
            assertTrue(model2.hasIndexer("indexer1"));

            // Update the indexer -- verify INDEXER_UPDATED event
            indexer1 = new IndexerDefinitionBuilder()
                    .startFrom(indexer1)
                    .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
View Full Code Here

TOP

Related Classes of com.ngdata.hbaseindexer.model.api.WriteableIndexerModel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.