Package org.apache.lucene.analysis.snowball

Examples of org.apache.lucene.analysis.snowball.SnowballAnalyzer.tokenStream()

98
99
100
101
102
103
104
105
106
107
108
                                        org.apache.lucene.util.Version.LUCENE_CURRENT,
                                        "Spanish",
                                        SPANISH_STOP_WORDS);

                       
                        TokenStream tokenStream = analyzer.tokenStream(
                                        "content",
                                        new StringReader(indexCleanedOfHTMLTags));
                       
                        Token token = new Token();
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
      TokenStream ts ;
      TermAttribute termAtt;
     
      // Query data
      doc = (Document)ir.document(testDocId);
      ts = analyzer.tokenStream("title", new StringReader(doc.get("title")));
      termAtt = ts.addAttribute(TermAttribute.class); 
      while (ts.incrementToken())
      {
        rTitle.add(termAtt.term());
      }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
      while (ts.incrementToken())
      {
        rTitle.add(termAtt.term());
      }
     
      ts = analyzer.tokenStream("content", new StringReader(doc.get("content")));
      termAtt = ts.addAttribute(TermAttribute.class); 
      while (ts.incrementToken())
      {
        rContent.add(termAtt.term());
      }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
      while (ts.incrementToken())
      {
        rContent.add(termAtt.term());
      }
     
      ts = analyzer.tokenStream("desc", new StringReader(doc.get("desc")));
      termAtt = ts.addAttribute(TermAttribute.class); 
      while (ts.incrementToken())
      {
        rDesc.add(termAtt.term());
      }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
      {
        int docId = (Integer)trainIterator.next();
        doc = (Document)ir.document(docId);
               
        //********************** Title Similarity Score ***************
        ts = analyzer.tokenStream("title", new StringReader(doc.get("title")));
        termAtt = ts.addAttribute(TermAttribute.class);   
       
        // Construct a HashMap of Train record title
        titleMap = new HashMap<String , Integer>();
        titleNI = 0;
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
              titleSimScore += ((double)titleMap.get(tempToken) + (100 * titleVocabMap.get(tempToken)/titleLength))/(titleNI + 100);
          }
        }
       
        //********************** Description Similarity Score ****************
        ts = analyzer.tokenStream("desc", new StringReader(doc.get("desc")));
        termAtt = ts.addAttribute(TermAttribute.class); 
               
        // Construct a HashMap of Train record description
        descMap = new HashMap<String , Integer>();
        descNI = 0;
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
            descSimScore += ((double)descMap.get(tempToken) + (100 * descVocabMap.get(tempToken))/descLength)/(descNI + 100);
          }
        }
       
        //********************** Content Similarity Score ****************
        ts = analyzer.tokenStream("content", new StringReader(doc.get("content")));
        termAtt = ts.addAttribute(TermAttribute.class); 
               
        // Construct a HashMap of Train record content
        contentMap = new HashMap<String , Integer>();
        contentNI = 0;
View Full Code Here
380
381
382
383
384
385
386
387
388
389
390
   
    Iterator trainIterator = trainDocIds.iterator();
    while (trainIterator.hasNext())
    {
      doc = (Document)ir.document((Integer)trainIterator.next());
      ts = analyzer.tokenStream("title", new StringReader(doc.get("title")));
      termAtt = ts.addAttribute(TermAttribute.class);   
     
      while(ts.incrementToken())
      {
        tempToken = termAtt.term();
View Full Code Here
398
399
400
401
402
403
404
405
406
407
408
          else
            titleVocabMap.put(tempToken, 1);
        }
      }
     
      ts = analyzer.tokenStream("content", new StringReader(doc.get("content")));
      termAtt = ts.addAttribute(TermAttribute.class);   
     
      while(ts.incrementToken())
      {
        tempToken = termAtt.term();
View Full Code Here
417
418
419
420
421
422
423
424
425
426
427
          else
            contentVocabMap.put(tempToken, 1);
        }
      }
     
      ts = analyzer.tokenStream("desc", new StringReader(doc.get("desc")));
      termAtt = ts.addAttribute(TermAttribute.class);   
     
      while(ts.incrementToken())
      {
        tempToken = termAtt.term();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.