Examples of org.apache.lucene.analysis.tokenattributes.OffsetAttribute.startOffset()

Package org.apache.lucene.analysis.tokenattributes

Class org.apache.lucene.analysis.tokenattributes.OffsetAttribute

Examples of org.apache.lucene.analysis.tokenattributes.OffsetAttribute.startOffset()

org.apache.lucene.analysis.tokenattributes.OffsetAttribute.startOffset()
Returns this Token's starting offset, the position of the first character corresponding to this token in the source text.
Note that the difference between {@link #endOffset()} and startOffset() may not be equal to termText.length(), as the term text may have been altered by a stemmer or some other filter. @see #setOffset(int,int)

      TokenStream ts = analyzer.tokenStream("foo", s);
      try {
        ts.reset();
        OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
        while (ts.incrementToken()) {
          String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
          for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
            cp = highlightedText.codePointAt(j);
            assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
          }
        }

View Full Code Here

    final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
    final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
    tk.reset();
    for (int i = minGram; i <= Math.min(codePointCount, maxGram); ++i) {
      assertTrue(tk.incrementToken());
      assertEquals(0, offsetAtt.startOffset());
      assertEquals(s.length(), offsetAtt.endOffset());
      final int end = Character.offsetByCodePoints(s, 0, i);
      assertEquals(s.substring(0, end), termAtt.toString());
    }
    assertFalse(tk.incrementToken());

View Full Code Here

      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
      OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
      while (ts.incrementToken()) {
        assertNotNull(offsetAtt);
        assertNotNull(termAtt);
        System.out.println("token '" + termAtt.toString() + "' has offset " + offsetAtt.startOffset() + "," + offsetAtt.endOffset());
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getLocalizedMessage());
    }

View Full Code Here

        val = ((CharTermAttribute)att).toString();
      } else if (attClass.equals("FlagsAttribute")) {
        val = Integer.toHexString(((FlagsAttribute)att).getFlags());
      } else if (attClass.equals("OffsetAttribute")) {
        OffsetAttribute off = (OffsetAttribute)att;
        val = off.startOffset() + "-" + off.endOffset();
      } else if (attClass.equals("PayloadAttribute")) {
        BytesRef payload = ((PayloadAttribute)att).getPayload();
        if (payload != null) {
          val = Util.bytesToHex(payload.bytes, payload.offset, payload.length, false);
        } else {

View Full Code Here

    Object inputText = app.find(myUi, "inputText");
    if (as.hasAttribute(OffsetAttribute.class)) {
      OffsetAttribute off = (OffsetAttribute)as.getAttribute(OffsetAttribute.class);
      app.setInteger(inputText, "start", 0);
      app.setInteger(inputText, "end", off.endOffset());
      app.setInteger(inputText, "start", off.startOffset());
      app.requestFocus(inputText);
    }
  }
}

View Full Code Here

        
        TestCase.assertEquals(tokenStrings[pos], cta.toString());
        TestCase.assertEquals(positionIncrements[pos], pta.getPositionIncrement());
        
        if (pta.getPositionIncrement() == 0) {
          TestCase.assertEquals(lastStart, oa.startOffset());
          TestCase.assertEquals(lastEnd, oa.endOffset());
        }
        
        if (!cta.toString().startsWith("NE_")) {
          TestCase.assertEquals(input.substring(oa.startOffset(), oa.endOffset()), cta.toString());

View Full Code Here

          TestCase.assertEquals(lastStart, oa.startOffset());
          TestCase.assertEquals(lastEnd, oa.endOffset());
        }
        
        if (!cta.toString().startsWith("NE_")) {
          TestCase.assertEquals(input.substring(oa.startOffset(), oa.endOffset()), cta.toString());
        }
        
        lastStart = oa.startOffset();
        lastEnd   = oa.endOffset();

View Full Code Here

        
        if (!cta.toString().startsWith("NE_")) {
          TestCase.assertEquals(input.substring(oa.startOffset(), oa.endOffset()), cta.toString());
        }
        
        lastStart = oa.startOffset();
        lastEnd   = oa.endOffset();
        
        pos++;
      }

View Full Code Here

    PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
    
    while (ts.incrementToken()){
      Token token = new Token();
      token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
      token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
      token.setType(typeAtt.type());
      token.setFlags(flagsAtt.getFlags());
      token.setPayload(payloadAtt.getPayload());
      token.setPositionIncrement(posIncAtt.getPositionIncrement());
      result.add(token);

View Full Code Here

          terms.put(term, positions);
        }
        if (stride == 1) {
          positions.add(pos);
        } else {
          positions.add(pos, offsetAtt.startOffset(), offsetAtt.endOffset());
        }
      }
      stream.end();


      // ensure infos.numTokens > 0 invariant; needed for correct operation of terms()

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.