Package com.atilika.kuromoji

Examples of com.atilika.kuromoji.Token

76
77
78
79
80
81
82
83
84
        String input = "シロクロ";
        String[] surfaceForms = {"シロ", "クロ"};
        List<Token> tokens = tokenizer.tokenize(input);

        assertEquals(surfaceForms.length, tokens.size());
        Token token = tokens.get(1);
        String actual = token.getSurfaceForm() + "\t" + token.getAllFeatures();
        assertEquals("クロ\tカスタム名詞,*,*,*,*,*,*,クロ,*", actual);
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
        List<Token> tokens = tokenizer.tokenize("1234");

        // If any of the assertions below fail, the dictionary filter wasn't applies correctly
        assertEquals(1, tokens.size());

        Token token = tokens.get(0);

        assertEquals("1234", token.getSurfaceForm());
        assertTrue(token.isUnknown());
    }
View Full Code Here
TOP

Related Classes of com.atilika.kuromoji.Token

  • com.atilika.kuromoji.entities.TokenizerTest
  • com.atilika.kuromoji.ipadic.UserDictionaryTokenizerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.