Examples of UniqueFilter

    
    @Override
    public Generator createGenerator() {


        Generator generator = new FSGenerator(crawlPath);
        generator = new UniqueFilter(new IntervalFilter(new URLRegexFilter(generator, getRegexs())));
        return generator;
    }

    tokens.add(newToken("token2", 21, 27));
    tokens.add(newToken("token3", 28, 33));
    tokens.add(newToken("token4", 34, 40));
    
    TokenStream tokenStream = new CollectionTokenStream(tokens);
    TokenFilter filter = new UniqueFilter(tokenStream);
    
    Token nextToken = new Token(); 
    filter.next(nextToken);
    assertNotNull(nextToken);
    assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    
    nextToken = filter.next(nextToken);
    assertNotNull(nextToken);
    assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
    
    nextToken = filter.next(nextToken);
    assertNotNull(nextToken);
    assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));


    nextToken = filter.next(nextToken);
    assertNotNull(nextToken);
    assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));


    assertNull(filter.next(nextToken));
  }

Examples of UniqueFilter

Examples of cn.edu.hfut.dmic.webcollector.generator.filter.UniqueFilter

Examples of org.apache.uima.lucas.indexer.analysis.UniqueFilter