Examples of org.apache.lucene.analysis.tokenattributes.CharTermAttribute.buffer()

Package org.apache.lucene.analysis.tokenattributes

Class org.apache.lucene.analysis.tokenattributes.CharTermAttribute

Examples of org.apache.lucene.analysis.tokenattributes.CharTermAttribute.buffer()

org.apache.lucene.analysis.tokenattributes.CharTermAttribute.buffer()
Returns the internal termBuffer character array which you can then directly alter. If the array is too small for your token, use {@link #resizeBuffer(int)} to increase it. Afteraltering the buffer be sure to call {@link #setLength} to record the number of validcharacters that were placed into the termBuffer.
NOTE: The returned buffer may be larger than the valid {@link #length()}.

      {
         while (ts.incrementToken())
         {
            OffsetAttribute offset = ts.getAttribute(OffsetAttribute.class);
            CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
            String termText = new String(term.buffer(), 0, term.length());
            TermVectorOffsetInfo[] info = termMap.get(termText);
            if (info == null)
            {
               info = new TermVectorOffsetInfo[1];
            }

View Full Code Here

      int tokenCount = 0;
      // for every token
      while (ts.incrementToken())
      {
         CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
         String word = new String(term.buffer(), 0, term.length());
         tokenCount++;
         if (tokenCount > maxNumTokensParsed)
         {
            break;
         }

View Full Code Here

         {
            String word;
            while (ts.incrementToken())
            {


               word = new String(term.buffer(), 0, term.length());
               //            while ((t = ts.next()) != null)
               //            {
               String origWord = statement.substring(offset.startOffset(), offset.endOffset());
               if (positionIncrement.getPositionIncrement() > 0)
               {

View Full Code Here

                           //t = field.tokenStreamValue().next(t);
                           field.tokenStreamValue().incrementToken();
                           CharTermAttribute term = field.tokenStreamValue().getAttribute(CharTermAttribute.class);
                           PayloadAttribute payload = field.tokenStreamValue().getAttribute(PayloadAttribute.class);


                           String value = new String(term.buffer(), 0, term.length());


                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());

View Full Code Here

    @Override
    public boolean incrementToken() throws IOException {
        final LinkedList<String> tokens = getTokens();
        final CharTermAttribute termAtt = getTermAtt();
        if (tokens.size() == 0 && input.incrementToken()) {
            final String version = new String(termAtt.buffer(), 0, termAtt.length());
            final String[] toAnalyze = version.split("[_-]");
            //ensure we analyze the whole string as one too
            analyzeVersion(version);
            for (String str : toAnalyze) {
                analyzeVersion(str);

View Full Code Here

    @Override
    public boolean incrementToken() throws IOException {
        final LinkedList<String> tokens = getTokens();
        final CharTermAttribute termAtt = getTermAtt();
        if (tokens.size() == 0 && input.incrementToken()) {
            final String text = new String(termAtt.buffer(), 0, termAtt.length());
            if (UrlStringUtils.containsUrl(text)) {
                final String[] parts = text.split("\\s");
                for (String part : parts) {
                    if (UrlStringUtils.isUrl(part)) {
                        try {

View Full Code Here

    CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
    stream.reset();


    while ( stream.incrementToken() ) {
      if ( attribute.length() > 0 ) {
        String term = new String( attribute.buffer(), 0, attribute.length() );
        terms.add( term );
      }
    }
    stream.end();
    stream.close();

View Full Code Here

        if (termsAtSamePosition == null) {
          termsAtSamePosition = new ArrayList<Term>();
          termsPerPosition.put( position, termsAtSamePosition );
        }


        String termString = new String( termAttribute.buffer(), 0, termAttribute.length() );
        termsAtSamePosition.add( new Term( fieldName, termString ) );
        if ( termsAtSamePosition.size() > 1 ) {
          isMultiPhrase = true;
        }
      }

View Full Code Here

    CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
    stream.reset();


    while ( stream.incrementToken() ) {
      if ( attribute.length() > 0 ) {
        String term = new String( attribute.buffer(), 0, attribute.length() );
        terms.add( term );
      }
    }
    stream.end();
    stream.close();

View Full Code Here

    CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
    TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
    PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
    
    while (nptf.incrementToken()) {
      assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.buffer()[0]))));
      assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
      String type = new String(payloadAtt.getPayload().getData(), "UTF-8");
      assertTrue(type + " is not equal to " + typeAtt.type(), type.equals(typeAtt.type()) == true);
      count++;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.