Package org.wltea.analyzer

Examples of org.wltea.analyzer.Lexeme

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
            if(hit.getBegin() > doneIndex + 1){
              //输出并处理从doneIndex+1 到 seg.start - 1之间的未知词段
              processUnknown(segmentBuff , context , doneIndex + 1 , hit.getBegin()- 1);
            }
            //输出当前的词
            Lexeme newLexeme = new Lexeme(context.getBuffOffset() , hit.getBegin() , context.getCursor() - hit.getBegin() + 1 , Lexeme.TYPE_CJK_NORMAL);
            context.addLexeme(newLexeme);
            //更新goneIndex，标识已处理
            if(doneIndex < context.getCursor()){
              doneIndex = context.getCursor();
            }
           
            if(hit.isPrefix()){//同时也是前缀
             
            }else{ //后面不再可能有匹配了
              //移出当前的hit
              hitList.remove(hit);
            }
           
          }else if(hit.isPrefix()){//前缀，未匹配成词
           
          }else if(hit.isUnmatch()){//不匹配
            //移出当前的hit
            hitList.remove(hit);
          }
        }
      }
     
      //处理以input为开始的一个新hit
      Hit hit = Dictionary.matchInMainDict(segmentBuff, context.getCursor() , 1);
      if(hit.isMatch()){//匹配成词
        //判断是否有不可识别的词段
        if(context.getCursor() > doneIndex + 1){
          //输出并处理从doneIndex+1 到 context.getCursor()- 1之间的未知
          processUnknown(segmentBuff , context , doneIndex + 1 , context.getCursor()- 1);
        }
        //输出当前的词
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , context.getCursor() , 1 , Lexeme.TYPE_CJK_NORMAL);
        context.addLexeme(newLexeme);
        //更新doneIndex，标识已处理
        if(doneIndex < context.getCursor()){
          doneIndex = context.getCursor();
        }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
   * @param segmentBuff
   * @param uBegin 起始位置
   * @param uEnd 终止位置
   */
  private void processUnknown(char[] segmentBuff , Context context , int uBegin , int uEnd){
    Lexeme newLexeme = null;
   
    Hit hit = Dictionary.matchInPrepDict(segmentBuff, uBegin, 1);   
    if(hit.isUnmatch()){//不是副词或介词     
      if(uBegin > 0){//处理姓氏
        hit = Dictionary.matchInSurnameDict(segmentBuff, uBegin - 1 , 1);
        if(hit.isMatch()){
          //输出姓氏
          newLexeme = new Lexeme(context.getBuffOffset() , uBegin - 1 , 1 , Lexeme.TYPE_CJK_SN);
          context.addLexeme(newLexeme);   
        }
      }     
    }
   
    //以单字输出未知词段
    for(int i = uBegin ; i <= uEnd ; i++){
      newLexeme = new Lexeme(context.getBuffOffset() , i , 1  , Lexeme.TYPE_CJK_UNKNOWN);
      context.addLexeme(newLexeme);   
    }
   
    hit = Dictionary.matchInPrepDict(segmentBuff, uEnd, 1);
    if(hit.isUnmatch()){//不是副词或介词
      int length = 1;
      while(uEnd < context.getAvailable() - length){//处理后缀词
        hit = Dictionary.matchInSuffixDict(segmentBuff, uEnd + 1 , length);
        if(hit.isMatch()){
          //输出后缀
          newLexeme = new Lexeme(context.getBuffOffset() , uEnd + 1  , length , Lexeme.TYPE_CJK_SF);
          context.addLexeme(newLexeme);
          break;
        }
        if(hit.isUnmatch()){
          break;
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
  @Override
  public final boolean incrementToken() throws IOException {
    //清除所有的词元属性
    clearAttributes();
    Lexeme nextLexeme = _IKImplement.next();
    if(nextLexeme != null){
      //将Lexeme转成Attributes
      //设置词元文本
      termAtt.setTermBuffer(nextLexeme.getLexemeText());
      //设置词元长度
      termAtt.setTermLength(nextLexeme.getLength());
      //设置词元位移
      offsetAtt.setOffset(nextLexeme.getBeginPosition(), nextLexeme.getEndPosition());
      //记录分词的最后位置
      finalOffset = nextLexeme.getEndPosition();
      //返会true告知还有下个词元
      return true;
    }
    //返会false告知词元输出完毕
    return false;
View Full Code Here
578
579
580
581
582
583
584
585
586
587
   * @param context
   */
  private void outputNumLexeme(Context context){
    if(nStart > -1 && nEnd > -1){
      //生成已切分的词元
      Lexeme newLexeme = new Lexeme(context.getBuffOffset() ,nStart , nEnd - nStart + 1 , Lexeme.TYPE_NUM );
      context.addLexeme(newLexeme);
      fCaN = true;
    }
  }
View Full Code Here
591
592
593
594
595
596
597
598
599
600
   * @param context
   */
  private void outputCountLexeme(Context context){
    if(countStart > -1 && countEnd > -1){
      //生成已切分的词元
      Lexeme countLexeme = new Lexeme(context.getBuffOffset() ,countStart , countEnd - countStart + 1 , Lexeme.TYPE_NUMCOUNT);
      context.addLexeme(countLexeme);
    }

  } 
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
        //不在忽略尾部的链接字符
        end = context.getCursor();         
       
      }else{
        //生成已切分的词元
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , start , end - start + 1 , Lexeme.TYPE_LETTER);
        context.addLexeme(newLexeme);
        //设置当前分词器状态为“待处理”
        start = -1;
        end = -1;
      }     
    }
   
    //context.getCursor() == context.getAvailable() - 1读取缓冲区最后一个字符，直接输出
    if(context.getCursor() == context.getAvailable() - 1){
      if(start != -1 && end != -1){
        //生成已切分的词元
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , start , end - start + 1 , Lexeme.TYPE_LETTER);
        context.addLexeme(newLexeme);
      }
      //设置当前分词器状态为“待处理”
      start = -1;
      end = -1;
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
      if(CharacterHelper.isEnglishLetter(input)){
        //记录当前指针位置为结束位置
        letterEnd =  context.getCursor();
      }else{
        //生成已切分的词元
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , letterStart , letterEnd - letterStart + 1 , Lexeme.TYPE_LETTER);
        context.addLexeme(newLexeme);
        //设置当前分词器状态为“待处理”
        letterStart = -1;
        letterEnd = -1;
      }
    }
   
    //context.getCursor() == context.getAvailable() - 1读取缓冲区最后一个字符，直接输出
    if(context.getCursor() == context.getAvailable() - 1){
      if(letterStart != -1 && letterEnd != -1){
        //生成已切分的词元
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , letterStart , letterEnd - letterStart + 1 , Lexeme.TYPE_LETTER);
        context.addLexeme(newLexeme);
      }
      //设置当前分词器状态为“待处理”
      letterStart = -1;
      letterEnd = -1;
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
//      e.printStackTrace();
//    }
    System.out.println(t); 
    IKSegmentation ikSeg = new IKSegmentation(new StringReader(t) ,false);
    try {
      Lexeme l = null;
      while( (l = ikSeg.next()) != null){
        System.out.println(l);
      }
    } catch (IOException e) {
      // TODO Auto-generated catch block
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
   
    for(String t : testStr){
      System.out.println(t); 
      IKSegmentation ikSeg = new IKSegmentation(new StringReader(t) , false);
      try {
        Lexeme l = null;
        while( (l = ikSeg.next()) != null){
          System.out.println(l);
        }
      } catch (IOException e) {
        // TODO Auto-generated catch block
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
//    testStr.add("浙江省长兴县城管在光天化日之下明抢东东");
    for(String t : testStr){
      System.out.println(t); 
      IKSegmentation ikSeg = new IKSegmentation(new StringReader(t) , false);
      try {
        Lexeme l = null;
        while( (l = ikSeg.next()) != null){
          System.out.println(l);
        }
      } catch (IOException e) {
        // TODO Auto-generated catch block
View Full Code Here
TOP

Related Classes of org.wltea.analyzer.Lexeme

  • org.wltea.analyzer.lucene.IKTokenizer
  • org.wltea.analyzer.seg.CJKSegmenter
  • org.wltea.analyzer.seg.LetterSegmenter
  • org.wltea.analyzer.seg.QuantifierSegmenter
  • org.wltea.analyzer.test.SegmentorTester
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.