Package org.wltea.analyzer.dic

Examples of org.wltea.analyzer.dic.Hit

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
          }
        }
      }
     
      //处理以input为开始的一个新hit
      Hit hit = Dictionary.matchInMainDict(segmentBuff, context.getCursor() , 1);
      if(hit.isMatch()){//匹配成词
        //判断是否有不可识别的词段
        if(context.getCursor() > doneIndex + 1){
          //输出并处理从doneIndex+1 到 context.getCursor()- 1之间的未知
          processUnknown(segmentBuff , context , doneIndex + 1 , context.getCursor()- 1);
        }
        //输出当前的词
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , context.getCursor() , 1 , Lexeme.TYPE_CJK_NORMAL);
        context.addLexeme(newLexeme);
        //更新doneIndex，标识已处理
        if(doneIndex < context.getCursor()){
          doneIndex = context.getCursor();
        }

        if(hit.isPrefix()){//同时也是前缀
          //向词段队列增加新的Hit
          hitList.add(hit);
        }
       
      }else if(hit.isPrefix()){//前缀，未匹配成词
        //向词段队列增加新的Hit
        hitList.add(hit);
       
      }else if(hit.isUnmatch()){//不匹配，当前的input不是词，也不是词前缀，将其视为分割性的字符
        if(doneIndex >= context.getCursor()){
          //当前不匹配的字符已经被处理过了，不需要再processUnknown
          return;
        }
       
        //输出从doneIndex到当前字符（含当前字符）之间的未知词
        processUnknown(segmentBuff , context , doneIndex + 1 , context.getCursor());
        //更新doneIndex，标识已处理
        doneIndex = context.getCursor();
      }
     
    }else {//输入的不是中文(CJK)字符
      if(hitList.size() > 0
          &&  doneIndex < context.getCursor() - 1){
        for(Hit hit : hitList){
          //判断是否有不可识别的词段
          if(doneIndex < hit.getEnd()){
            //输出并处理从doneIndex+1 到 seg.end之间的未知词段
            processUnknown(segmentBuff , context , doneIndex + 1 , hit.getEnd());
          }
        }
      }
      //清空词段队列
      hitList.clear();
      //更新doneIndex，标识已处理
      if(doneIndex < context.getCursor()){
        doneIndex = context.getCursor();
      }
    }
   
    //缓冲区结束临界处理
    if(context.getCursor() == context.getAvailable() - 1){ //读取缓冲区结束的最后一个字符     
      if( hitList.size() > 0 //队列中还有未处理词段
        && doneIndex < context.getCursor()){//最后一个字符还未被输出过
        for(Hit hit : hitList){
          //判断是否有不可识别的词段
          if(doneIndex < hit.getEnd() ){
            //输出并处理从doneIndex+1 到 seg.end之间的未知词段
            processUnknown(segmentBuff , context , doneIndex + 1 , hit.getEnd());
          }
        }
      }
      //清空词段队列
      hitList.clear();;
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
   * @param uEnd 终止位置
   */
  private void processUnknown(char[] segmentBuff , Context context , int uBegin , int uEnd){
    Lexeme newLexeme = null;
   
    Hit hit = Dictionary.matchInPrepDict(segmentBuff, uBegin, 1);   
    if(hit.isUnmatch()){//不是副词或介词     
      if(uBegin > 0){//处理姓氏
        hit = Dictionary.matchInSurnameDict(segmentBuff, uBegin - 1 , 1);
        if(hit.isMatch()){
          //输出姓氏
          newLexeme = new Lexeme(context.getBuffOffset() , uBegin - 1 , 1 , Lexeme.TYPE_CJK_SN);
          context.addLexeme(newLexeme);   
        }
      }     
    }
   
    //以单字输出未知词段
    for(int i = uBegin ; i <= uEnd ; i++){
      newLexeme = new Lexeme(context.getBuffOffset() , i , 1  , Lexeme.TYPE_CJK_UNKNOWN);
      context.addLexeme(newLexeme);   
    }
   
    hit = Dictionary.matchInPrepDict(segmentBuff, uEnd, 1);
    if(hit.isUnmatch()){//不是副词或介词
      int length = 1;
      while(uEnd < context.getAvailable() - length){//处理后缀词
        hit = Dictionary.matchInSuffixDict(segmentBuff, uEnd + 1 , length);
        if(hit.isMatch()){
          //输出后缀
          newLexeme = new Lexeme(context.getBuffOffset() , uEnd + 1  , length , Lexeme.TYPE_CJK_SF);
          context.addLexeme(newLexeme);
          break;
        }
        if(hit.isUnmatch()){
          break;
        }
        length++;
      }
    }   
View Full Code Here
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
   * 处理中文量词
   * @param segmentBuff
   * @param context
   */
  private void processCount(char[] segmentBuff , Context context){
    Hit hit = null;

    if(countStart == -1){
      hit = Dictionary.matchInQuantifierDict(segmentBuff , context.getCursor() , 1);
    }else{
      hit = Dictionary.matchInQuantifierDict(segmentBuff , countStart , context.getCursor() - countStart + 1);
    }
   
    if(hit != null){
      if(hit.isPrefix()){
        if(countStart == -1){
          //设置量词的开始
          countStart = context.getCursor();
        }
      }
     
      if(hit.isMatch()){
        if(countStart == -1){
          countStart = context.getCursor();
        }
        //设置量词可能的结束
        countEnd = context.getCursor();
        //输出可能存在的量词
        outputCountLexeme(context);
      }
     
      if(hit.isUnmatch()){
        if(countStart != -1){
          //重置量词状态
          countStart = -1;
          countEnd = -1;
        }
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
      e1.printStackTrace();
    }
   
    System.out.println(new Date() + " begin march");
    long begintime = System.currentTimeMillis();
    Hit hit = null;
    int umCount = 0;
    int mCount = 0;
    for(String word : allWords){
      char[] chars = word.toCharArray();
      hit = _root_.match(chars , 0, chars.length);
      if(hit.isUnmatch()){
        //System.out.println(word);
        umCount++;
      }else{
        mCount++;
        //System.out.println(mCount + " : " + word);
View Full Code Here
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
        e1.printStackTrace();
      }
     
      System.out.println(new Date() + " begin march");
      long begintime = System.currentTimeMillis();
      Hit hit = null;
      int umCount = 0;
      int mCount = 0;
      for(String word : allWords){     
        hit = Dictionary.matchInMainDict(word.toCharArray(), 0, word.length());
        if(hit.isUnmatch()){
          System.out.println(word);
          umCount++;
        }else{
          mCount++;
        }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
          }
        }
      }
     
      //处理以input为开始的一个新hit
      Hit hit = Dictionary.matchInMainDict(segmentBuff, context.getCursor() , 1);
      if(hit.isMatch()){//匹配成词
        //判断是否有不可识别的词段
        if(context.getCursor() > doneIndex + 1){
          //输出并处理从doneIndex+1 到 context.getCursor()- 1之间的未知
          processUnknown(segmentBuff , context , doneIndex + 1 , context.getCursor()- 1);
        }
        //输出当前的词
        Lexeme newLexeme = new Lexeme(context.getBuffOffset() , context.getCursor() , 1 , Lexeme.TYPE_CJK_NORMAL);
        context.addLexeme(newLexeme);
        //更新doneIndex，标识已处理
        if(doneIndex < context.getCursor()){
          doneIndex = context.getCursor();
        }

        if(hit.isPrefix()){//同时也是前缀
          //向词段队列增加新的Hit
          hitList.add(hit);
        }
       
      }else if(hit.isPrefix()){//前缀，未匹配成词
        //向词段队列增加新的Hit
        hitList.add(hit);
       
      }else if(hit.isUnmatch()){//不匹配，当前的input不是词，也不是词前缀，将其视为分割性的字符
        if(doneIndex >= context.getCursor()){
          //当前不匹配的字符已经被处理过了，不需要再processUnknown
          return;
        }
       
        //输出从doneIndex到当前字符（含当前字符）之间的未知词
        processUnknown(segmentBuff , context , doneIndex + 1 , context.getCursor());
        //更新doneIndex，标识已处理
        doneIndex = context.getCursor();
      }
     
    }else {//输入的不是中文(CJK)字符
      if(hitList.size() > 0
          &&  doneIndex < context.getCursor() - 1){
        for(Hit hit : hitList){
          //判断是否有不可识别的词段
          if(doneIndex < hit.getEnd()){
            //输出并处理从doneIndex+1 到 seg.end之间的未知词段
            processUnknown(segmentBuff , context , doneIndex + 1 , hit.getEnd());
          }
        }
      }
      //清空词段队列
      hitList.clear();
      //更新doneIndex，标识已处理
      if(doneIndex < context.getCursor()){
        doneIndex = context.getCursor();
      }
    }
   
    //缓冲区结束临界处理
    if(context.getCursor() == context.getAvailable() - 1){ //读取缓冲区结束的最后一个字符     
      if( hitList.size() > 0 //队列中还有未处理词段
        && doneIndex < context.getCursor()){//最后一个字符还未被输出过
        for(Hit hit : hitList){
          //判断是否有不可识别的词段
          if(doneIndex < hit.getEnd() ){
            //输出并处理从doneIndex+1 到 seg.end之间的未知词段
            processUnknown(segmentBuff , context , doneIndex + 1 , hit.getEnd());
          }
        }
      }
      //清空词段队列
      hitList.clear();;
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
   * @param uEnd 终止位置
   */
  private void processUnknown(char[] segmentBuff , Context context , int uBegin , int uEnd){
    Lexeme newLexeme = null;
   
    Hit hit = Dictionary.matchInPrepDict(segmentBuff, uBegin, 1);   
    if(hit.isUnmatch()){//不是副词或介词     
      if(uBegin > 0){//处理姓氏
        hit = Dictionary.matchInSurnameDict(segmentBuff, uBegin - 1 , 1);
        if(hit.isMatch()){
          //输出姓氏
          newLexeme = new Lexeme(context.getBuffOffset() , uBegin - 1 , 1 , Lexeme.TYPE_CJK_SN);
          context.addLexeme(newLexeme);   
        }
      }     
    }
   
    //以单字输出未知词段
    for(int i = uBegin ; i <= uEnd ; i++){
      newLexeme = new Lexeme(context.getBuffOffset() , i , 1  , Lexeme.TYPE_CJK_UNKNOWN);
      context.addLexeme(newLexeme);   
    }
   
    hit = Dictionary.matchInPrepDict(segmentBuff, uEnd, 1);
    if(hit.isUnmatch()){//不是副词或介词
      int length = 1;
      while(uEnd < context.getAvailable() - length){//处理后缀词
        hit = Dictionary.matchInSuffixDict(segmentBuff, uEnd + 1 , length);
        if(hit.isMatch()){
          //输出后缀
          newLexeme = new Lexeme(context.getBuffOffset() , uEnd + 1  , length , Lexeme.TYPE_CJK_SF);
          context.addLexeme(newLexeme);
          break;
        }
        if(hit.isUnmatch()){
          break;
        }
        length++;
      }
    }   
View Full Code Here
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
   * 处理中文量词
   * @param segmentBuff
   * @param context
   */
  private void processCount(char[] segmentBuff , Context context){
    Hit hit = null;

    if(countStart == -1){
      hit = Dictionary.matchInQuantifierDict(segmentBuff , context.getCursor() , 1);
    }else{
      hit = Dictionary.matchInQuantifierDict(segmentBuff , countStart , context.getCursor() - countStart + 1);
    }
   
    if(hit != null){
      if(hit.isPrefix()){
        if(countStart == -1){
          //设置量词的开始
          countStart = context.getCursor();
        }
      }
     
      if(hit.isMatch()){
        if(countStart == -1){
          countStart = context.getCursor();
        }
        //设置量词可能的结束
        countEnd = context.getCursor();
        //输出可能存在的量词
        outputCountLexeme(context);
      }
     
      if(hit.isUnmatch()){
        if(countStart != -1){
          //重置量词状态
          countStart = -1;
          countEnd = -1;
        }
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
      e1.printStackTrace();
    }
   
    System.out.println(new Date() + " begin march");
    long begintime = System.currentTimeMillis();
    Hit hit = null;
    int umCount = 0;
    int mCount = 0;
    for(String word : allWords){     
      hit = _root_.match(word.toCharArray());
      if(hit.isUnmatch()){
        System.out.println(word);
        umCount++;
      }else{
        mCount++;
        System.out.println(mCount + " : " + word);
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
        e1.printStackTrace();
      }
     
      System.out.println(new Date() + " begin march");
      long begintime = System.currentTimeMillis();
      Hit hit = null;
      int umCount = 0;
      int mCount = 0;
      for(String word : allWords){     
        hit = Dictionary.matchInMainDict(word.toCharArray(), 0, word.length());
        if(hit.isUnmatch()){
          System.out.println(word);
          umCount++;
        }else{
          mCount++;
        }
View Full Code Here
TOP

Related Classes of org.wltea.analyzer.dic.Hit

  • org.wltea.analyzer.core.CJKSegmenter
  • org.wltea.analyzer.core.CN_QuantifierSegmenter
  • org.wltea.analyzer.seg.CJKSegmenter
  • org.wltea.analyzer.seg.QuantifierSegmenter
  • org.wltea.analyzer.test.DictionaryTester
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.