Package com.ngdata.hbaseindexer.parse

Examples of com.ngdata.hbaseindexer.parse.ResultToSolrMapper

232
233
234
235
236
237
238
239
240
241
242
243
244
245
            // Only scan the column families and/or cells that the indexer requires
            // if we're running in row-indexing mode       
            if (indexerConf.getMappingType() == MappingType.ROW) {
                MorphlineClasspathUtil.setupJavaCompilerClasspath();

                ResultToSolrMapper resultToSolrMapper = factory.createMapper(
                        hbaseIndexingSpecification.getIndexerName()
                );
                Get get = resultToSolrMapper.getGet(HBaseShims.newGet().getRow());
                hbaseScan.setFamilyMap(get.getFamilyMap());
            }
            hbaseScan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, tableName);

            scans.add(hbaseScan);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
   
    @Before
    public void setUp() {
       
        indexerConf = spy(new IndexerConfBuilder().table(TABLE_NAME).mappingType(MappingType.ROW).build());
        ResultToSolrMapper mapper = IndexingEventListenerTest.createHbaseToSolrMapper(true);
       
        tablePool = mock(HTablePool.class);
        solrWriter = mock(DirectSolrInputDocumentWriter.class);
       
        updateCollector = new SolrUpdateCollector(10);
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
            }
        }

        MorphlineClasspathUtil.setupJavaCompilerClasspath();
       
        ResultToSolrMapper resultToSolrMapper = ResultToSolrMapperFactory.createResultToSolrMapper(
                indexingSpec.getIndexerName(),
                indexerConf);
       
        Indexer indexer = Indexer.createIndexer(
                                indexingSpec.getIndexerName(),
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
          if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) {
              params.put(entry.getKey(), entry.getValue());
          }
        }

        ResultToSolrMapper mapper = factory.createMapper(indexName);

        // TODO This would be better-placed in the top-level job setup -- however, there isn't currently any
        // infrastructure to handle converting an in-memory model into XML (we can only interpret an
        // XML doc into the internal model), so we need to do this here for now
        if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
    public void testNonExistingRow() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).build();

        when(tableA.get(any(Get.class))).thenReturn(new Result());

        ResultToSolrMapper mapper = mock(ResultToSolrMapper.class);
        when(mapper.isRelevantKV(any(KeyValue.class))).thenReturn(true);
        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

        List<KeyValue> kvs = Lists.newArrayList(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("cf"),
                Bytes.toBytes("qual"), Bytes.toBytes("value")));
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
    /**
     * Create a dummy HBaseToSolrMapper that returns the given value on any calls to containsRequiredData().
     */
    static ResultToSolrMapper createHbaseToSolrMapper(final boolean containsRequiredDataReturnVal) {
        return new ResultToSolrMapper() {
            @Override
            public boolean isRelevantKV(KeyValue kv) {
                return true;
            }

View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
    @Test
    public void testRowBasedIndexing_RowReadModeNever() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.NEVER).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);

        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

        List<KeyValue> kvs = Lists.newArrayList(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("cf"),
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_RereadRequired() throws IOException, SolrServerException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(false);

        when(tableA.get(any(Get.class))).thenReturn(newResult(Lists.newArrayList(new KeyValue())));

        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_NoRereadRequired() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();

        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);

        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

        List<KeyValue> kvs = Lists.newArrayList(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("cf"),
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
    @Test
    public void testColumnBasedIndexing() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).mappingType(IndexerConf.MappingType.COLUMN).build();

        ResultToSolrMapper mapper = new ResultToSolrMapper() {
            @Override
            public boolean isRelevantKV(KeyValue kv) {
                return Bytes.toString(kv.getFamily()).equals("messages");
            }
View Full Code Here
TOP

Related Classes of com.ngdata.hbaseindexer.parse.ResultToSolrMapper

  • com.ngdata.hbaseindexer.indexer.IndexingEventListenerTest
  • com.ngdata.hbaseindexer.indexer.ResultToSolrMapperFactory
  • com.ngdata.hbaseindexer.indexer.RowBasedIndexerTest
  • com.ngdata.hbaseindexer.mr.HBaseIndexerMapper
  • com.ngdata.hbaseindexer.mr.HBaseIndexingOptions
  • com.ngdata.hbaseindexer.mr.IndexerDryRun
  • com.ngdata.hbaseindexer.supervisor.IndexerSupervisor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.