Examples of org.apache.lucene.analysis.CachingTokenFilter

org.apache.lucene.analysis.CachingTokenFilter
This class can be used if the token attributes of a TokenStream are intended to be consumed more than once. It caches all token attribute states locally in a List.
CachingTokenFilter implements the optional method {@link TokenStream#reset()}, which repositions the stream to the first Token.

  /**
   * This method intended for use with <tt>testHighlightingWithDefaultField()</tt>
   */
  private static String highlightField(Query query, String fieldName, String text)
      throws IOException {
    CachingTokenFilter tokenStream = new CachingTokenFilter(new StandardAnalyzer().tokenStream(
        fieldName, new StringReader(text)));
    // Assuming "<B>", "</B>" used to highlight
    SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
    Highlighter highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName,
        tokenStream, FIELD_NAME));
    highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
    tokenStream.reset();
    String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");
    return rv.length() == 0 ? text : rv;
  }

View Full Code Here


    int maxNumFragmentsRequired = 2;


    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);
    }

View Full Code Here


    int maxNumFragmentsRequired = 2;


    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(this,
          new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);
    }

View Full Code Here


    int maxNumFragmentsRequired = 2;


    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(this,
          new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);
    }

View Full Code Here


    int maxNumFragmentsRequired = 2;


    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(this,
          new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);

View Full Code Here


    int maxNumFragmentsRequired = 2;


    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(this,
          new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);

View Full Code Here

      int maxNumFragmentsRequired = 2;
      String fragmentSeparator = "...";
      SpanScorer scorer = null;
      TokenStream tokenStream = null;


      tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
          new StringReader(text)));
      
      SpanScorer.setHighlightCnstScrRngQuery(true);
      scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME, (CachingTokenFilter) tokenStream);

View Full Code Here

        // highlighting respects fieldnames used in query


        Scorer fieldSpecificScorer = null;
        if (mode == this.SPAN) {
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
          CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
          fieldSpecificScorer = new SpanScorer(query, FIELD_NAME, ctf);
        } else if (mode == this.STANDARD) {
          fieldSpecificScorer = new QueryScorer(query, "contents");
        }
        Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
            fieldSpecificScorer);
        fieldSpecificHighlighter.setTextFragmenter(new NullFragmenter());
        String result = fieldSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);
        assertEquals("Should match", result, "<B>fred</B> is one of the people");


        // highlighting does not respect fieldnames used in query
        Scorer fieldInSpecificScorer = null;
        if (mode == this.SPAN) {
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
          CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
          fieldInSpecificScorer = new SpanScorer(query, null, ctf);
        } else if (mode == this.STANDARD) {
          fieldInSpecificScorer = new QueryScorer(query);
        }

View Full Code Here


  public void assertExpectedHighlightCount(final int maxNumFragmentsRequired,
      final int expectedHighlights) throws Exception {
    for (int i = 0; i < hits.length(); i++) {
      String text = hits.doc(i).get(FIELD_NAME);
      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
          new StringReader(text)));
      Highlighter highlighter = new Highlighter(this,
          new SpanScorer(query, FIELD_NAME, tokenStream));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));
      tokenStream.reset();


      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      System.out.println("\t" + result);

View Full Code Here

    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
        Formatter formatter) {
      if (mode == STANDARD) {
        return new Highlighter(formatter, new QueryScorer(query));
      } else if (mode == SPAN) {
        CachingTokenFilter tokenStream = new CachingTokenFilter(stream);
        Highlighter highlighter;
        try {
          highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName, tokenStream));
          tokenStream.reset();
        } catch (IOException e) {
          throw new RuntimeException(e);
        }


        return highlighter;

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.lucene.analysis.CachingTokenFilter

org.apache.lucene.analysis.shingle.TestShingleMatrixFilter

org.apache.lucene.index.TestIndexWriter

org.apache.lucene.index.TestTermVectorsWriter

org.apache.lucene.queryparser.classic.QueryParserBase

org.apache.lucene.queryparser.flexible.standard.processors.AnalyzerQueryNodeProcessor

org.apache.lucene.queryParser.QueryParser

org.apache.lucene.queryParser.standard.processors.AnalyzerQueryNodeProcessor

org.apache.lucene.search.highlight.HighlighterTest

org.apache.lucene.search.highlight.SynonymTokenizer$TestHighlightRunner

org.apache.lucene.search.highlight.WeightedSpanTermExtractor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.