Package org.solbase.indexer

Examples of org.solbase.indexer.ParsedDoc

214
215
216
217
218
219
220
221
222
223
224
225
        }
      } else {
        try {
          logger.info("adding doc: " + docNumber);
         
          ParsedDoc parsedDoc = writer.parseDoc(doc, schema.getAnalyzer(), indexName, docNumber, indexUtil.getSortFieldNames());
          List<TermDocMetadata> termDocMetas = parsedDoc.getTermDocMetadatas();
          // TODO: possible problem
          // doc is not in cache, cluster isn't responsible for update store
          // doc never gets updated in hbase, nor cache
          // for loop below will update tv with this new doc.
          // when searched, it will throw null point exception on this doc
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
      if(cachedObj == null || cachedObj.getValue() == null) {
        // document doesn't exist, so let's just bail out here
        return true;
      }
     
      ParsedDoc parsedDoc = new ParsedDoc(newDoc);
      parsedDoc.setIndexName(indexName);
      parsedDoc.setIndexUtil(indexUtil);
      parsedDoc.setIndexWriter(writer);
      parsedDoc.setUpdateStore(updateStore);
     
      int shardNum = SolbaseShardUtil.getShardNum(indexName);
      int startDocId = SolbaseShardUtil.getStartDocId(shardNum);
      int endDocId = SolbaseShardUtil.getEndDocId(shardNum);
     
View Full Code Here
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
    try {
      CachedObjectWrapper<Document, Long> wrapper = ReaderCache.getDocument(docId, null, indexName, 0, 0);
     
      boolean updateStore = cmd.getUpdateStore();
     
      ParsedDoc parsedDoc = new ParsedDoc();
      parsedDoc.setIndexName(indexName);
      parsedDoc.setIndexUtil(indexUtil);
      parsedDoc.setIndexWriter(writer);
      parsedDoc.setUpdateStore(updateStore);

      int shardNum = SolbaseShardUtil.getShardNum(indexName);
      int startDocId = SolbaseShardUtil.getStartDocId(shardNum);
      int endDocId = SolbaseShardUtil.getEndDocId(shardNum);
     
View Full Code Here
599
600
601
602
603
604
605
606
607
    }
   
    // Finally, Store meta-data so we can delete this document
    documentPut.add(Bytes.toBytes("allTerms"), Bytes.toBytes("allTerms"), SolbaseUtil.toBytes(allIndexedTerms).array());

    ParsedDoc parsedDoc = new ParsedDoc(metadatas, doc, documentPut, fieldCache.entrySet(), allIndexedTerms);
    return parsedDoc;

  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    // incrementing chunking sequence (lucene doc id)
    this.idCounter++;

    try {
      ParsedDoc parsedDoc = indexerUtil.getIndexWriter().parseDoc(doc, indexerUtil.getAnalyzer(), "", docId, indexerUtil.getSortFieldNames());

      List<TermDocMetadata> metadatas = parsedDoc.getTermDocMetadatas();
     
      MapWritable mapWritable = new MapWritable();
      DocumentPutWritable docWritable = new DocumentPutWritable(parsedDoc.getFieldsMap(), parsedDoc.getAllTerms(), docId, globalId);
      mapWritable.put(new BytesWritable(Bytes.toBytes("doc")), docWritable);

      for (TermDocMetadata metadata : metadatas) {
        byte[] key = metadata.getFieldTermKey();
        ByteBuffer buf = metadata.serialize();
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
290
291
  private ParsedDoc deleteDocument(Document oldDoc, int docId, String indexName, IndexWriter writer, SolbaseIndexUtil indexUtil, boolean updateStore, int startDocId, int endDocId){
    try {
      // clone so read won't conflict
      oldDoc = new Document(oldDoc);
      oldDoc.removeField("docId");
      ParsedDoc parsedDoc = writer.parseDoc(oldDoc, schema.getAnalyzer(), indexName, docId, indexUtil.getSortFieldNames());

      List<TermDocMetadata> metadatas = parsedDoc.getTermDocMetadatas();

      // TODO: doing duplicate work here - once from updateObject and again from updateObjectStore
      for (TermDocMetadata metadata : metadatas) {
        ReaderCache.updateTermDocsMetadata(metadata.getTerm(), metadata, indexName, writer, LayeredCache.ModificationType.DELETE, updateStore, startDocId, endDocId);
      }
View Full Code Here
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
        oldDoc = new Document(oldDoc);
        oldDoc.removeField("docId");

        // parsing old doc to get all terms
        try {
          ParsedDoc oldParsedDoc = writer.parseDoc(oldDoc, schema.getAnalyzer(), indexName, docNumber, indexUtil.getSortFieldNames());

          List<Term> oldTerms = oldParsedDoc.getAllTerms();
          List<TermDocMetadata> oldTermDocMetas = oldParsedDoc.getTermDocMetadatas();

          Document mergedDoc = mergeOldAndNew(oldDoc, newDoc);
          ParsedDoc parsedDoc = writer.parseDoc(mergedDoc, schema.getAnalyzer(), indexName, docNumber, indexUtil.getSortFieldNames());

          List<TermDocMetadata> newTermDocMetas = parsedDoc.getTermDocMetadatas();
          List<Term> newTerms = parsedDoc.getAllTerms();

          List<Term> updateList = new ArrayList<Term>(oldTerms);
          List<Term> deleteList = new ArrayList<Term>(oldTerms);
          List<Term> addList = new ArrayList<Term>(newTerms);

          Collections.copy(updateList, oldTerms);
          Collections.copy(deleteList, oldTerms);
          Collections.copy(addList, newTerms);

          updateList.retainAll(newTerms);
          deleteList.removeAll(newTerms);
          addList.removeAll(oldTerms);
          int shardNum = SolbaseShardUtil.getShardNum(indexName);
          int startDocId = SolbaseShardUtil.getStartDocId(shardNum);
          int endDocId = SolbaseShardUtil.getEndDocId(shardNum);
          // updating tv first
          for (TermDocMetadata termDocMeta : newTermDocMetas) {
            Term term = termDocMeta.getTerm();
            if (updateList.contains(term)) {
              logger.debug("updating this term: " + term.toString());
              ReaderCache.updateTermDocsMetadata(term, termDocMeta, indexName, writer, LayeredCache.ModificationType.UPDATE, updateStore, startDocId, endDocId);
            } else if (addList.contains(term)) {
              ReaderCache.updateTermDocsMetadata(term, termDocMeta, indexName, writer, LayeredCache.ModificationType.ADD, updateStore, startDocId, endDocId);
            }
          }

          // clean up deletes
          if (deleteList.size() > 0) {
            for (TermDocMetadata termDocMeta : oldTermDocMetas) {
              Term term = termDocMeta.getTerm();

              if (deleteList.contains(term)) {
                ReaderCache.updateTermDocsMetadata(term, termDocMeta, indexName, writer, LayeredCache.ModificationType.DELETE, updateStore, startDocId, endDocId);

              }
            }
          }

          parsedDoc.getDocument().add(docIdField);

          return parsedDoc;
        } catch (NullPointerException e) {
          return null;
        }
      } else {
        Document mergedDoc = mergeOldAndNew(oldDoc, newDoc);

        ParsedDoc parsedDoc = writer.parseDoc(mergedDoc, schema.getAnalyzer(), indexName, docNumber, indexUtil.getSortFieldNames());

        return parsedDoc;
      }
    } catch (IOException e) {
      // TODO Auto-generated catch block
View Full Code Here
TOP

Related Classes of org.solbase.indexer.ParsedDoc

  • org.solbase.indexer.mapreduce.SolbaseInitialIndexMapper
  • org.solbase.lucenehbase.DocumentLoader
  • org.solbase.lucenehbase.IndexWriter
  • org.solbase.SolbaseIndexWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.