Package com.ngdata.hbaseindexer.conf

Examples of com.ngdata.hbaseindexer.conf.IndexerConfBuilder

51
52
53
54
55
56
57
58
59
60
61
    private RowBasedIndexer indexer;
   
    @Before
    public void setUp() {
       
        indexerConf = spy(new IndexerConfBuilder().table(TABLE_NAME).mappingType(MappingType.ROW).build());
        ResultToSolrMapper mapper = IndexingEventListenerTest.createHbaseToSolrMapper(true);
       
        tablePool = mock(HTablePool.class);
        solrWriter = mock(DirectSolrInputDocumentWriter.class);
       
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    private SolrUpdateCollector updateCollector;
    private ColumnBasedIndexer indexer;

    @Before
    public void setUp() {
        indexerConf = spy(new IndexerConfBuilder().table(TABLE_NAME).build());
        mapper = IndexingEventListenerTest.createHbaseToSolrMapper(true);
        solrWriter = mock(DirectSolrInputDocumentWriter.class);
        updateCollector = new SolrUpdateCollector(10);
        indexer = new ColumnBasedIndexer("column-based", indexerConf, TABLE_NAME, mapper, null, solrWriter);
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexingSpec.getIndexerComponentFactory(), new ByteArrayInputStream(indexingSpec.getConfiguration()), Maps.<String, String>newHashMap());
        IndexerConf indexerConf = factory.createIndexerConf();

        if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
            LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
            indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
        }

        Map<String, String> params = indexerConf.getGlobalParams();
        if (indexingOpts.morphlineFile != null) {
            params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
        // TODO This would be better-placed in the top-level job setup -- however, there isn't currently any
        // infrastructure to handle converting an in-memory model into XML (we can only interpret an
        // XML doc into the internal model), so we need to do this here for now
        if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
            LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
            indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
        }
        indexerConf.setGlobalParams(params);

        try {
            indexer = createIndexer(indexName, context, indexerConf, tableName, mapper, indexConnectionParams);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
     * When receiving an event for a different table than the one specified in the IndexerConf, the Indexer should not
     * interact with HBase or Solr.
     */
    @Test
    public void testNonmatchedTable() {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).build();

        Indexer indexer = Indexer.createIndexer("index name", conf, TABLE_A, null, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

        SepEvent event = new SepEvent(Bytes.toBytes(TABLE_B), null, null, null);
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
     * When we get a sep event for a row which does not exist anymore (or at least, doesn't contain any relevant
     * columns), then we expect a delete on Solr.
     */
    @Test
    public void testNonExistingRow() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).build();

        when(tableA.get(any(Get.class))).thenReturn(new Result());

        ResultToSolrMapper mapper = mock(ResultToSolrMapper.class);
        when(mapper.isRelevantKV(any(KeyValue.class))).thenReturn(true);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
        };
    }

    @Test
    public void testRowBasedIndexing_RowReadModeNever() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.NEVER).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);

        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
        verifyZeroInteractions(tableA, tableB);
    }

    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_RereadRequired() throws IOException, SolrServerException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(false);

        when(tableA.get(any(Get.class))).thenReturn(newResult(Lists.newArrayList(new KeyValue())));
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
        verify(tableA).get(any(Get.class));
    }

    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_NoRereadRequired() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);

        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
        verifyZeroInteractions(tableA, tableB);
    }

    @Test
    public void testColumnBasedIndexing() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).mappingType(IndexerConf.MappingType.COLUMN).build();

        ResultToSolrMapper mapper = new ResultToSolrMapper() {
            @Override
            public boolean isRelevantKV(KeyValue kv) {
                return Bytes.toString(kv.getFamily()).equals("messages");
View Full Code Here
TOP

Related Classes of com.ngdata.hbaseindexer.conf.IndexerConfBuilder

  • com.ngdata.hbaseindexer.indexer.ColumnBasedIndexerTest
  • com.ngdata.hbaseindexer.indexer.IndexingEventListenerTest
  • com.ngdata.hbaseindexer.indexer.RowBasedIndexerTest
  • com.ngdata.hbaseindexer.mr.HBaseIndexerMapper
  • com.ngdata.hbaseindexer.mr.IndexerDryRun
  • org.lilyproject.indexer.hbase.mapper.LilyIndexerComponentFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.