Package com.ngdata.hbaseindexer.model.api

Examples of com.ngdata.hbaseindexer.model.api.IndexerDefinition


        if (!model.hasIndexer(indexName)) {
            throw new CliException("Indexer does not exist: " + indexName);
        }

        IndexerDefinition newIndexer = null;
        String lock = model.lockIndexer(indexName);
        try {
            IndexerDefinition indexer = model.getFreshIndexer(indexName);

            IndexerDefinitionBuilder builder = buildIndexerDefinition(options, indexer);
            newIndexer = builder.build();

            if (newIndexer.equals(indexer)) {
View Full Code Here


import org.junit.Test;

public class IndexerDefinitionJsonSerDeserTest {
    @Test
    public void testMinimal() {
        IndexerDefinition indexer = new IndexerDefinitionBuilder()
                .name("index1").build();

        IndexerDefinitionJsonSerDeser serdeser = new IndexerDefinitionJsonSerDeser();
        byte[] json = serdeser.toJsonBytes(indexer);

        IndexerDefinition indexer2 = serdeser.fromJsonBytes(json).build();

        assertEquals(indexer, indexer2);
        assertEquals("index1", indexer.getName());
    }
View Full Code Here

        assertEquals("index1", indexer.getName());
    }

    @Test
    public void testFull() {
        IndexerDefinition indexer = new IndexerDefinitionBuilder()
                .name("index1")
                .lifecycleState(LifecycleState.DELETE_REQUESTED)
                .batchIndexingState(BatchIndexingState.BUILDING)
                .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
                .indexerComponentFactory("testReader")
                .configuration("config1".getBytes(Charsets.UTF_8))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("p1", "v1", "p2", "v2"))
                .subscriptionId("my-subscription")
                .subscriptionTimestamp(5L)
                .defaultBatchIndexCliArguments(new String[]{"arg1", "arg2"})
                .batchIndexCliArguments(new String[]{"arg3"})
                .activeBatchBuildInfo(
                        new BatchBuildInfo(10L, null, ImmutableMap.of("job-id-1", "url-1"), new String[]{"arg1", "arg2"}))
                .lastBatchBuildInfo(
                        new BatchBuildInfo(11L, false, ImmutableMap.of("job-id-2", "url-2"), new String[]{"arg3"}))
                .occVersion(5).build();

        IndexerDefinitionJsonSerDeser serdeser = new IndexerDefinitionJsonSerDeser();
        byte[] json = serdeser.toJsonBytes(indexer);

        IndexerDefinition indexer2 = serdeser.fromJsonBytes(json).build();

        assertEquals("index1", indexer2.getName());
        assertEquals(LifecycleState.DELETE_REQUESTED, indexer2.getLifecycleState());
        assertEquals(BatchIndexingState.BUILDING, indexer2.getBatchIndexingState());
        assertEquals(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME, indexer2.getIncrementalIndexingState());
        assertEquals("testReader", indexer2.getIndexerComponentFactory());
        assertArrayEquals("config1".getBytes(Charsets.UTF_8), indexer2.getConfiguration());
        assertEquals("solr", indexer.getConnectionType());
        assertEquals("v1", indexer.getConnectionParams().get("p1"));
        assertEquals("v2", indexer.getConnectionParams().get("p2"));
        assertEquals("my-subscription", indexer2.getSubscriptionId());
        assertEquals(5L, indexer2.getSubscriptionTimestamp());
        assertArrayEquals(new String[]{"arg1", "arg2"}, indexer2.getDefaultBatchIndexCliArguments());
        assertArrayEquals(new String[]{"arg3"}, indexer2.getBatchIndexCliArguments());

        assertNotNull(indexer2.getActiveBatchBuildInfo());
        assertEquals(10L, indexer2.getActiveBatchBuildInfo().getSubmitTime());
        assertEquals(ImmutableMap.of("job-id-1", "url-1"), indexer2.getActiveBatchBuildInfo().getMapReduceJobTrackingUrls());
        assertNull(indexer2.getActiveBatchBuildInfo().isFinishedSuccessful());
        assertArrayEquals(new String[]{"arg1", "arg2"}, indexer2.getActiveBatchBuildInfo().getBatchIndexCliArguments());

        assertNotNull(indexer2.getLastBatchBuildInfo());
        assertEquals(11L, indexer2.getLastBatchBuildInfo().getSubmitTime());
        assertEquals(ImmutableMap.of("job-id-2", "url-2"), indexer2.getLastBatchBuildInfo().getMapReduceJobTrackingUrls());
        assertFalse(indexer2.getLastBatchBuildInfo().isFinishedSuccessful());
        assertArrayEquals(new String[]{"arg3"}, indexer2.getLastBatchBuildInfo().getBatchIndexCliArguments());

        assertEquals(5, indexer2.getOccVersion());

        assertEquals(indexer, indexer2);
    }
View Full Code Here

            // Disable incremental index updating
            WriteableIndexerModel indexerModel = lilyProxy.getLilyServerProxy().getIndexerModel();
            String lock = indexerModel.lockIndexer(indexName);
            String subscriptionId;
            try {
                IndexerDefinition index = indexerModel.getIndexer(indexName);
                subscriptionId = index.getSubscriptionId();

                indexerModel.updateIndexer(new IndexerDefinitionBuilder()
                        .startFrom(index)
                        .incrementalIndexingState(IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE)
                        .build(), lock);
View Full Code Here

        connectionParams.put(SolrConnectionParams.ZOOKEEPER, "localhost:2181/solr");
        connectionParams.put(SolrConnectionParams.COLLECTION, collectionName);
        connectionParams.put(LResultToSolrMapper.ZOOKEEPER_KEY, "localhost:2181");
        connectionParams.put(LResultToSolrMapper.REPO_KEY, repositoryName);

        IndexerDefinition index = new IndexerDefinitionBuilder()
                .name(indexName)
                .connectionType("solr")
                .connectionParams(connectionParams)
                .indexerComponentFactory(LilyIndexerComponentFactory.class.getName())
                .configuration(indexerConfiguration)
View Full Code Here

        if (!indexerModel.hasIndexer(indexName)) {
            log.info("Index '" + indexName + "' not known to indexerModel within " + timeout + "ms");
            return subscriptionId;
        }

        IndexerDefinition indexDefinition = indexerModel.getIndexer(indexName);
        subscriptionId = indexDefinition.getSubscriptionId();
        while (subscriptionId == null && System.currentTimeMillis() < tryUntil) {
            Thread.sleep(10);
            subscriptionId = indexerModel.getIndexer(indexName).getSubscriptionId();
        }
        if (subscriptionId == null) {
View Full Code Here

        WriteableIndexerModel model = getIndexerModel();

        try {
            String lock = model.lockIndexer(indexName);
            try {
                IndexerDefinition index = model.getIndexer(indexName);
                IndexerDefinitionBuilder builder = new IndexerDefinitionBuilder()
                        .startFrom(index)
                        .batchIndexingState(IndexerDefinition.BatchIndexingState.BUILD_REQUESTED)
                        .batchIndexCliArguments(batchCliArgs);
                model.updateIndexer(builder.build(), lock);
            } finally {
                model.unlockIndexer(lock);
            }
        } catch (Exception e) {
            throw new Exception("Error launching batch index build.", e);
        }

        try {
            // Now wait until its finished
            long tryUntil = System.currentTimeMillis() + timeOut;
            while (System.currentTimeMillis() < tryUntil) {
                Thread.sleep(100);
                IndexerDefinition definition = model.getIndexer(indexName);

                if (definition.getBatchIndexingState() == IndexerDefinition.BatchIndexingState.INACTIVE) {
                    Long amountFailed = null;
                    //Long amountFailed = definition.getLastBatchBuildInfo().getCounters().get(COUNTER_NUM_FAILED_RECORDS);
                    boolean successFlag = definition.getLastBatchBuildInfo().isFinishedSuccessful();
                    if (successFlag && (amountFailed == null || amountFailed == 0L)) {
                        return;
                    } else {
                        System.out.println(definition.getLastBatchBuildInfo().getMapReduceJobTrackingUrls());
                        throw new Exception("Batch index build did not finish successfully: success flag = " +
                                successFlag + ", amount failed records = " + amountFailed);
                    }
                }
            }
View Full Code Here

        connectionParams.put(SolrConnectionParams.COLLECTION, "core0");
        connectionParams.put(LResultToSolrMapper.ZOOKEEPER_KEY, "localhost:2181");
        connectionParams.put(LResultToSolrMapper.REPO_KEY, REPO_NAME);
        connectionParams.put(LResultToSolrMapper.TABLE_KEY, indexTableName);
        connectionParams.put(LResultToSolrMapper.ENABLE_DEREFMAP_KEY, Boolean.toString(enableDerefMap));
        IndexerDefinition indexDef = new IndexerDefinitionBuilder().name(indexName)
                .connectionType("solr")
                .connectionParams(connectionParams)
                .indexerComponentFactory(LilyIndexerComponentFactory.class.getName())
                .configuration(ByteStreams.toByteArray(IndexerTest.class.getResourceAsStream(confName)))
                        //.solrShards(Collections.singletonMap("shard1", "http://somewhere/"))
View Full Code Here

    private static void cleanupIndex(String indexName) throws Exception {
        if (indexerModel != null) {
            if (indexerModel.hasIndexer(indexName)) {
                System.out.println("doing the cleanup of " + indexName);
                String lock = indexerModel.lockIndexer(indexName);
                IndexerDefinition def = indexerModel.getIndexer(indexName);
                indexerModel.updateIndexer(new IndexerDefinitionBuilder()
                        .startFrom(def)
                        .lifecycleState(IndexerDefinition.LifecycleState.DELETE_REQUESTED).build(), lock);

                listener.waitForEvents(2);
View Full Code Here

        Map<String, String> connectionParams = Maps.newHashMap();
        connectionParams.put(SolrConnectionParams.ZOOKEEPER, "localhost:2181/solr");
        connectionParams.put(SolrConnectionParams.COLLECTION, "core0");
        connectionParams.put(LResultToSolrMapper.REPO_KEY, REPO_NAME);
        connectionParams.put(LResultToSolrMapper.ZOOKEEPER_KEY, "localhost:2181");
        IndexerDefinition index = new IndexerDefinitionBuilder()
                .name(INDEX_NAME)
                .connectionType("solr")
                .connectionParams(connectionParams)
                /*
                Map<String, String> solrShards = new HashMap<String, String>();
View Full Code Here

TOP

Related Classes of com.ngdata.hbaseindexer.model.api.IndexerDefinition

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.