Examples of org.apache.lucene.analysis.TokenStream.end()

Class org.apache.lucene.analysis.TokenStream

Examples of org.apache.lucene.analysis.TokenStream.end()

org.apache.lucene.analysis.TokenStream.end()
This method is called by the consumer after the last token has been consumed, after {@link #incrementToken()} returned false(using the new TokenStream API). Streams implementing the old API should upgrade to use this feature.
This method can be used to perform any end-of-stream operations, such as setting the final offset of a stream. The final offset of a stream might differ from the offset of the last token eg in case one or more whitespaces followed after the last token, but a {@link WhitespaceTokenizer} was used. @throws IOException

            fieldSetting.fieldLength++;
            if (fieldSetting.fieldLength > maxFieldLength) {
              break;
            }
          }
          tokenStream.end();
          tokenStream.close();
        } else {
          // untokenized
          String fieldVal = field.stringValue();
          Token token = new Token(0, fieldVal.length(), "untokenized");

View Full Code Here

              q.insertWithOverflow(st);
            }
          }
        }
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

      MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class);
      CharTermAttribute charTerm = ts.getAttribute(CharTermAttribute.class);
      while (ts.incrementToken()) {
        System.out.println(charTerm.toString() + " => " + attribute.getTags());
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

    try {
      CharTermAttribute termAtt_1 = ts_1.getAttribute(CharTermAttribute.class);
      ts_1.reset();
      ts_1.incrementToken();
      assertEquals("first stream", "liście", termAtt_1.toString());
      ts_1.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts_1);
    }


    TokenStream ts_2 = a.tokenStream("dummy", "danych");

View Full Code Here

    try {
      CharTermAttribute termAtt_2 = ts_2.getAttribute(CharTermAttribute.class);
      ts_2.reset();
      ts_2.incrementToken();
      assertEquals("second stream", "dany", termAtt_2.toString());
      ts_2.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts_2);
    }
  }

View Full Code Here

        "subst:sg:voc:m3");


      assertPOSToken(ts, "lista", 
        "subst:sg:dat:f",
        "subst:sg:loc:f");
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

        assertEquals(offsets[start], offsetAtt.startOffset());
        assertEquals(offsets[end], offsetAtt.endOffset());
      }
    }
    assertFalse(grams.incrementToken());
    grams.end();
    assertEquals(s.length(), offsetAtt.startOffset());
    assertEquals(s.length(), offsetAtt.endOffset());
  }


  public void testLargeInput() throws IOException {

View Full Code Here

          for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
            cp = highlightedText.codePointAt(j);
            assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
          }
        }
        ts.end();
      } finally {
        IOUtils.closeWhileHandlingException(ts);
      }
    }
    // just for fun

View Full Code Here

          for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
            cp = highlightedText.codePointAt(j);
            assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
          }
        }
        ts.end();
      } finally {
        IOUtils.closeWhileHandlingException(ts);
      }
    }
    // just for fun

View Full Code Here

                if (termAtt.length() > 0) {
                    String term = termAtt.toString();
                        result.add(term);
                }
            }
            stream.end();
            stream.close();
        } catch (IOException e) {
            throw new RuntimeException("Impossible error", e);
        }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.