Examples of org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator

      FieldDescription fieldDescription) throws FieldBuildingException {
    TokenStream tokenStream = null;
    if (fieldDescription.getMerge())
      tokenStream = getTokenStreamMerger(tokenStreams);
    else if (tokenStreams.size() > 1)
      tokenStream = new TokenStreamConcatenator(tokenStreams);
    else if (tokenStreams.size() == 1)
      tokenStream = tokenStreams.iterator().next();
    return tokenStream;
  }

    tokens.add(newToken("token6", 34, 40));


    tokenStream = new CollectionTokenStream(tokens );
    tokenStreams.add(tokenStream);
    
    TokenStreamConcatenator concatenator = new TokenStreamConcatenator(tokenStreams);
    
    Token nextToken = new Token(); 
    concatenator.next(nextToken);
    assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    
    concatenator.reset();
    nextToken = concatenator.next(nextToken);
    assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    nextToken = concatenator.next(nextToken);
    assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));    
  }

  {
    TokenStream tokenStream = null;
    if (fieldDescription.getMerge())
      tokenStream = getTokenStreamMerger(tokenStreams);
    else if (tokenStreams.size() > 1)
      tokenStream = new TokenStreamConcatenator(tokenStreams);
    else if (tokenStreams.size() == 1)
      tokenStream = tokenStreams.iterator().next();
    return tokenStream;
  }

Examples of org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator

Related Classes of org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator