Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token

42
43
44
45
46
47
48
   */
  public void collect(String word, int begin, int end) {
    if (tokens == null) {
      this.tokens = new LinkedList<Token>();
    }
    this.tokens.add(new Token(word, begin, end));
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
      OffsetAttribute offsetAtt = (OffsetAttribute)ts.getAttribute(OffsetAttribute.class);
      TypeAttribute typeAtt = (TypeAttribute)ts.getAttribute(TypeAttribute.class);
     
      System.out.println("("+termAtt.term()+","+offsetAtt.startOffset()+","+offsetAtt.endOffset()+",type="+typeAtt.type()+")");
    }*/
    for(Token t= new Token(); (t=TokenUtils.nextToken(ts, t)) !=null;) {
      System.out.println(t);
    }
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
    private Token reusableToken;
 
  public CutLetterDigitFilter(TokenStream input) {
    super(input);
   
    reusableToken = new Token();
    termAtt = (TermAttribute)addAttribute(TermAttribute.class);
    offsetAtt = (OffsetAttribute)addAttribute(OffsetAttribute.class);
    typeAtt = (TypeAttribute)addAttribute(TypeAttribute.class);
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
  private Token nextToken(Token reusableToken) throws IOException {
    assert reusableToken != null;
   
    //先使用上次留下来的。
    Token nextToken = tokenQueue.poll();
    if(nextToken != null) {
      return nextToken;
    }
   
    /*//在 TokenUtils.nextToken 已经调用了 inc
    if(!input.incrementToken()) {
      return null;
    }*/
   
    /*TermAttribute termAtt = (TermAttribute)input.getAttribute(TermAttribute.class);
    OffsetAttribute offsetAtt = (OffsetAttribute)input.getAttribute(OffsetAttribute.class);
    TypeAttribute typeAtt = (TypeAttribute)input.getAttribute(TypeAttribute.class);
   
    nextToken = reusableToken.reinit(termAtt.termBuffer(), 0, termAtt.termLength(), offsetAtt.startOffset(), offsetAtt.endOffset(), typeAtt.type());*/
   
    nextToken = TokenUtils.nextToken(input, reusableToken);
   
    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.termBuffer();
      final int length = nextToken.termLength();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
      int termBufferOffset = 0;
      int termBufferLength = 0;
      for(int i=0;i<length;i++) {
        byte type = (byte) Character.getType(buffer[i]);
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
   
    return nextToken;
  }
 
  private void addToken(Token oriToken, int termBufferOffset, int termBufferLength, byte type) {
    Token token = new Token(oriToken.termBuffer(), termBufferOffset, termBufferLength,
        oriToken.startOffset()+termBufferOffset, oriToken.startOffset()+termBufferOffset+termBufferLength);
   
    if(type == Character.DECIMAL_DIGIT_NUMBER) {
      token.setType(Word.TYPE_DIGIT);
    } else {
      token.setType(Word.TYPE_LETTER);
    }
   
    tokenQueue.offer(token);
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    tokenQueue.clear();
  }

  public boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
        long start = System.currentTimeMillis();
        while((line = reader.readLine()) != null) {
          bw.append("--------------------------").append("\r\n");;
          bw.append(line).append("\r\n");
          TokenStream ts = analyzer.tokenStream("text", new StringReader(line));
          for(Token t= new Token(); (t=TokenUtils.nextToken(ts, t)) !=null;) {
            bw.append(new String(t.termBuffer(), 0, t.termLength())).append(" | ");
          }
          bw.append("\r\n");
        }
       
        long t = System.currentTimeMillis() - start;
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
        size += s;
        TokenStream ts = analyzer.tokenStream("text", new InputStreamReader(ftxt));
        OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(new File(txt.getAbsoluteFile()+"."+outputChipName+".word")));
        BufferedWriter bw = new BufferedWriter(osw);
        long start = System.currentTimeMillis();
        for(Token t= new Token(); (t=TokenUtils.nextToken(ts, t)) !=null;) {
          bw.append(new String(t.term())).append("\r\n");
        }
        long t = System.currentTimeMillis() - start;
        time += t;
        System.out.println("size="+(s/1024)+"kb, use "+t+"ms, speed="+speed(s, t)+"kb/s, file="+txt.getName());
        bw.close();
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import org.apache.lucene.analysis.TokenStream;

public class TestSmartChineseAnalyzer extends TestCase {

  public void testChineseAnalyzer() throws IOException {
    Token nt = new Token();
    Analyzer ca = new SmartChineseAnalyzer(true);
    Reader sentence = new StringReader("我购买了道具和服装。");
    String[] result = { "我", "购买", "了", "道具", "和", "服装" };
    TokenStream ts = ca.tokenStream("sentence", sentence);
    int i = 0;
    nt = ts.next(nt);
    while (nt != null) {
      assertEquals(result[i], nt.term());
      i++;
      nt = ts.next(nt);
    }
    ts.close();
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
   * @throws FileNotFoundException
   * @throws IOException
   */
  private void sampleMethod() throws UnsupportedEncodingException,
      FileNotFoundException, IOException {
    Token nt = new Token();
    Analyzer ca = new SmartChineseAnalyzer(true);
    Reader sentence = new StringReader(
        "我从小就不由自主地认为自己长大以后一定得成为一个象我父亲一样的画家, 可能是父母潜移默化的影响。其实我根本不知道作为画家意味着什么，我是否喜欢，最重要的是否适合我，我是否有这个才华。其实人到中年的我还是不确定我最喜欢什么，最想做的是什么？我相信很多人和我一样有同样的烦恼。毕竟不是每个人都能成为作文里的宇航员，科学家和大教授。知道自己适合做什么，喜欢做什么，能做好什么其实是个非常困难的问题。"
            + "幸运的是，我想我的孩子不会为这个太过烦恼。通过老大，我慢慢发现美国高中的一个重要功能就是帮助学生分析他们的专长和兴趣，从而帮助他们选择大学的专业和未来的职业。我觉得帮助一个未成形的孩子找到她未来成长的方向是个非常重要的过程。"
            + "美国高中都有专门的职业顾问，通过接触不同的课程，和各种心理，个性，兴趣很多方面的问答来帮助每个学生找到最感兴趣的专业。这样的教育一般是要到高年级才开始， 可老大因为今年上计算机的课程就是研究一个职业走向的软件项目，所以她提前做了这些考试和面试。看来以后这样的教育会慢慢由电脑来测试了。老大带回家了一些试卷，我挑出一些给大家看看。这门课她花了2个多月才做完，这里只是很小的一部分。"
            + "在测试里有这样的一些问题："
            + "你是个喜欢动手的人吗？ 你喜欢修东西吗？你喜欢体育运动吗？你喜欢在室外工作吗？你是个喜欢思考的人吗？你喜欢数学和科学课吗？你喜欢一个人工作吗？你对自己的智力自信吗？你的创造能力很强吗？你喜欢艺术，音乐和戏剧吗？  你喜欢自由自在的工作环境吗？你喜欢尝试新的东西吗？ 你喜欢帮助别人吗？你喜欢教别人吗？你喜欢和机器和工具打交道吗？你喜欢当领导吗？你喜欢组织活动吗？你什么和数字打交道吗？");
    TokenStream ts = ca.tokenStream("sentence", sentence);

    System.out.println("start: " + (new Date()));
    long before = System.currentTimeMillis();
    nt = ts.next(nt);
    while (nt != null) {
      System.out.println(nt.term());
      nt = ts.next(nt);
    }
    ts.close();
    long now = System.currentTimeMillis();
    System.out.println("time: " + (now - before) / 1000.0 + " s");
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.Token

  • com.chenlb.mmseg4j.analysis.CutLetterDigitFilter
  • com.flaptor.hounder.classifier.util.TupleTokenizer
  • com.stimulus.archiva.search.EmailFilter
  • com.stimulus.archiva.search.FileNameFilter
  • net.nutch.analysis.CommonGrams
  • net.nutch.analysis.CommonGrams$Filter
  • net.nutch.analysis.NutchDocumentTokenizer
  • org.apache.cocoon.bean.query.SimpleLuceneCriterionBean
  • org.apache.jackrabbit.core.query.lucene.AbstractExcerpt
  • org.apache.jackrabbit.core.query.lucene.SearchIndex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.