Package com.ngdata.hbaseindexer.model.api

Examples of com.ngdata.hbaseindexer.model.api.IndexerDefinition


        SOLR_ZK = "127.0.0.1:" + zkClientPort + "/solr";
        INDEXER_ZK = "localhost:" + zkClientPort;
        ZooKeeperItf zkItf = ZkUtil.connect(INDEXER_ZK, 15000);
        INDEXER_MODEL = new IndexerModelImpl(zkItf, "/ngdata/hbaseindexer");
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                                                .name("zkindexerdef")
                                                .indexerComponentFactory(DefaultIndexerComponentFactory.class.getName())
                                                .configuration(Resources.toByteArray(Resources.getResource(
                                                        HBaseMapReduceIndexerToolGoLiveTest.class, "user_indexer.xml")))
                                                .connectionParams(ImmutableMap.of(
View Full Code Here


     */
    @GET
    @Path("{name}/config")
    @Produces("application/json")
    public Response getConfig(@PathParam("name") String name) throws IndexerNotFoundException, IOException {
        IndexerDefinition index = getModel().getIndexer(name);

        ObjectMapper m = new ObjectMapper();
        ObjectNode json = m.createObjectNode();
        json.put("occVersion", index.getOccVersion());
        json.put("config", new String(index.getConfiguration(), Charsets.UTF_8));

        return Response.ok(m.writeValueAsString(json), new MediaType("application", "json")).build();
    }
View Full Code Here

    @Produces("application/json")
    public IndexerDefinition put(@PathParam("name") String indexName, ObjectNode json) throws Exception {
        WriteableIndexerModel model = getModel();
        ObjectMapper m = new ObjectMapper();

        IndexerDefinition oldIndexer = model.getIndexer(indexName);
        IndexerDefinition indexerDefinition = IndexerDefinitionJsonSerDeser.INSTANCE.fromJson(json,
                new IndexerDefinitionBuilder().startFrom(oldIndexer)).build();

        IndexerDefinition.LifecycleState lifecycleState = json.has("lifecycleState") ?
                IndexerDefinition.LifecycleState.valueOf(json.get("lifecycleState").getTextValue()) : null;
View Full Code Here

        }
    }

    private String fetchIndexerTableName(String indexerName) throws Exception{
        // best effort since this could be a pattern ...
        IndexerDefinition indexerDefinition = get(indexerName);
        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDefinition.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDefinition.getConfiguration()), indexerDefinition.getConnectionParams());
        String tableName = factory.createIndexerConf().getTable();

        // TODO we should fail if the table does not exist
        return tableName;
    }
View Full Code Here

    @Override
    public void run(OptionSet options) throws Exception {
        super.run(options);


        IndexerDefinition indexer = null;
        try {
            IndexerDefinitionBuilder builder = buildIndexerDefinition(options, null);
            indexer = builder.build();
        } catch (IllegalArgumentException e) {
            System.err.printf("Error adding indexer: %s\n", e.getMessage());
            return;
        }

        model.addIndexer(indexer);

        System.out.println("Indexer added: " + indexer.getName());
    }
View Full Code Here

                    IndexerModelEvent event = eventQueue.take();
                    log.debug("Took event from queue: " + event.toString());
                    if (event.getType() == INDEXER_ADDED || event.getType() == INDEXER_UPDATED) {
                        try {
                            IndexerDefinition indexerDef = indexerModel.getIndexer(event.getIndexerName());
                            if (shouldRunIndexer(indexerDef)) {
                                if (indexers.containsKey(indexerDef.getName())) {
                                    restartIndexer(indexerDef);
                                } else {
                                    startIndexer(indexerDef);
                                }
                            } else {
                                stopIndexer(indexerDef.getName());
                            }
                        } catch (IndexerNotFoundException e) {
                            stopIndexer(event.getIndexerName());
                        } catch (Throwable t) {
                            log.error("Error in IndexerWorker's IndexerModelListener.", t);
View Full Code Here

            model1 = new IndexerModelImpl(zk1, "/test");
            model1.registerListener(listener);

            // Create an indexer -- verify INDEXER_ADDED event
            IndexerDefinition indexer1 = new IndexerDefinitionBuilder()
                    .name("indexer1")
                    .configuration("my-conf".getBytes("UTF-8"))
                    .build();
            model1.addIndexer(indexer1);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_ADDED, "indexer1"));

            // Verify that a fresh indexer model has the index
            model2 = new IndexerModelImpl(zk2, "/test");
            Collection<IndexerDefinition> indexers = model2.getIndexers();
            assertEquals("Expected indexer1, got " + indexers, 1, indexers.size());
            assertTrue(model2.hasIndexer("indexer1"));

            // Update the indexer -- verify INDEXER_UPDATED event
            indexer1 = new IndexerDefinitionBuilder()
                    .startFrom(indexer1)
                    .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
                    .build();
            String lock = model1.lockIndexer("indexer1");
            model1.updateIndexer(indexer1, lock);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_UPDATED, "indexer1"));
            model1.unlockIndexer(lock);

            // Delete the indexer -- verify INDEXER_DELETED event
            model1.deleteIndexerInternal("indexer1");
            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_DELETED, "indexer1"));

            // Create some more indexes and verify we get the correct number of INDEXER_ADDED events
            IndexerModelEvent[] expectedEvents = new IndexerModelEvent[9];
            for (int i = 2; i <= 10; i++) {
                String name = "indexer" + i;
                IndexerDefinition indexer = new IndexerDefinitionBuilder()
                        .name(name)
                        .configuration("my-conf".getBytes("UTF-8"))
                        .build();
                model1.addIndexer(indexer);
                expectedEvents[i - 2] = new IndexerModelEvent(IndexerModelEventType.INDEXER_ADDED, name);
            }

            listener.waitForEvents(9);
            listener.verifyEvents(expectedEvents);

            // Terminate ZK connections: clients should automatically re-establish the connection and things
            // should work as before
            assertEquals(2, terminateZooKeeperConnections());

            // Do another index update and check we get an event
            IndexerDefinition indexer2 = new IndexerDefinitionBuilder()
                    .name("indexer2")
                    .incrementalIndexingState(IncrementalIndexingState.DO_NOT_SUBSCRIBE)
                    .configuration("my-conf".getBytes(Charsets.UTF_8))
                    .build();
            lock = model1.lockIndexer("indexer2");
View Full Code Here

        try {
            model1 = new IndexerModelImpl(zk1, "/test");
            model2 = new IndexerModelImpl(zk2, "/test");

            // Create an index
            IndexerDefinition indexer1 = new IndexerDefinitionBuilder()
                    .name(indexerName)
                    .configuration("foo".getBytes(Charsets.UTF_8))
                    .build();
            model1.addIndexer(indexer1);
View Full Code Here

    }

    private void waitForDeletion(String indexerName) throws InterruptedException, KeeperException {
        System.out.printf("Deleting indexer '%s'", indexerName);
        while (model.hasIndexer(indexerName)) {
            IndexerDefinition indexerDef;
            try {
                indexerDef = model.getFreshIndexer(indexerName);
            } catch (IndexerNotFoundException e) {
                // The indexer was deleted between the call to hasIndexer and getIndexer, that's ok
                break;
            }

            switch (indexerDef.getLifecycleState()) {
            case DELETE_FAILED:
                System.err.println("\nDelete failed");
                return;
            case DELETE_REQUESTED:
            case DELETING:
                System.out.print(".");
                Thread.sleep(500);
                continue;
            default:
                throw new IllegalStateException("Illegal lifecycle state while deleting: "
                        + indexerDef.getLifecycleState());
            }
        }
        System.out.printf("\nDeleted indexer '%s'\n", indexerName);
    }
View Full Code Here

        if (!model.hasIndexer(indexerName)) {
            throw new CliException("Indexer does not exist: " + indexerName);
        }

        IndexerDefinition indexerDef = model.getIndexer(indexerName);

        if (indexerDef.getLifecycleState() == LifecycleState.DELETE_REQUESTED
                || indexerDef.getLifecycleState() == LifecycleState.DELETING) {
            System.err.printf("Delete of '%s' is already in progress\n", indexerName);
            return;
        }

        IndexerDefinitionBuilder builder = new IndexerDefinitionBuilder();
View Full Code Here

TOP

Related Classes of com.ngdata.hbaseindexer.model.api.IndexerDefinition

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.