Examples of org.apache.lucene.analysis.Token.type()

org.apache.lucene.analysis.Token.type()
Returns this Token's lexical type. Defaults to "word".

      if (increment > 0) 
      { position = position + increment;
        System.out.println();
        System.out.print(position + ": ");
      }
      System.out.print("\t [" + token.termText() + ": " + token.type() + "] " + token.startOffset() + ":" + token.endOffset());
      
     } //*-- end of for
    System.out.println("");
  }

View Full Code Here

  StandardBgramAnalyzer analyzer = new StandardBgramAnalyzer(); analyzer.setExtractEntities(true);
  TokenStream stream = analyzer.tokenStream("contents", new StringReader(question));
  ArrayList<Token> tokenList = new ArrayList<Token>(); Token token = null;
  entities = new ArrayList<String>();    //*-- list of entities in the question
  while ( (token = stream.next()) != null) 
   { tokenList.add(token); if (token.type().equals("<ENTITY>")) entities.add(token.termText()); }
 
  //*-------------------------------------------------------------------
  //*-- build the query with the five components
  //*--
  //*-- 1. First identify the entity types for the query

View Full Code Here

        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }


    catch (IllegalStateException e) {

View Full Code Here

    nextToken = reusableToken.reinit(termAtt.termBuffer(), 0, termAtt.termLength(), offsetAtt.startOffset(), offsetAtt.endOffset(), typeAtt.type());*/
    
    nextToken = TokenUtils.nextToken(input, reusableToken);
    
    if(nextToken != null && 
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.termBuffer();
      final int length = nextToken.termLength();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类

View Full Code Here

    
    nextToken = TokenUtils.nextToken(input, reusableToken);
    
    if(nextToken != null && 
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.termBuffer();
      final int length = nextToken.termLength();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
      int termBufferOffset = 0;

View Full Code Here

    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
    }

View Full Code Here

    clearAttributes();
    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
    posIncrAtt.setPositionIncrement(token.getPositionIncrement());
    flagsAtt.setFlags(token.getFlags());
    offsetAtt.setOffset(token.startOffset(), token.endOffset());
    typeAtt.setType(token.type());
    payloadAtt.setPayload(token.getPayload());
    return true;
  }


  private Token getNextInputToken(Token token) throws IOException {

View Full Code Here

        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }

View Full Code Here

      Token prototype = (Token) iterator.next();
      termAtt.setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
      posIncrAtt.setPositionIncrement(prototype.getPositionIncrement());
      flagsAtt.setFlags(prototype.getFlags());
      offsetAtt.setOffset(prototype.startOffset(), prototype.endOffset());
      typeAtt.setType(prototype.type());
      payloadAtt.setPayload(prototype.getPayload());


      return true;
    }

View Full Code Here


        Token token = null;


        while ((token = ts.next()) != null) {
            System.out.println("ReTokenizeFile.reTokenize(File): " + token.termText() + " " +
                token.startOffset() + " " + token.endOffset() + " " + token.type());
        }


        return file.getAbsolutePath();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.