Package com.github.pmerienne.trident.ml.preprocessing

Examples of com.github.pmerienne.trident.ml.preprocessing.TwitterTokenizer

28
29
30
31
32
33
34
35
36
37
38
39
40
41
  @Test
  public void testRemoveUsername() {
    // Given
    String tweet = "@PrincessSuperC Hey Cici";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("hei", "cici");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  @Test
  public void testRemoveHashTagIndicator() {
    // Given
    String tweet = "arg #word!";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("arg", "word");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  @Test
  public void testRemoveCharacterRepetitions() {
    // Given
    String tweet = "so huuunggryy!";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("so", "hungri");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  @Test
  public void testRemoveWordsStartingNumber() {
    // Given
    String tweet = "it's 15PM we're in the 20th century";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("we'r", "centuri");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
  @Test
  public void testRemoveHTMLMarkum() {
    // Given
    String tweet = "fb &gt; tw";
    TwitterTokenizer tokenizer = new TwitterTokenizer();

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("fb", "tw");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  @Test
  public void testWithNGram() {
    // Given
    String tweet = "it's not bad movie";
    TwitterTokenizer tokenizer = new TwitterTokenizer(2, 2);

    // When
    List<String> actualTokens = tokenizer.tokenize(tweet);

    // Then
    List<String> expectedTokens = Arrays.asList("_ bad", "bad", "bad movi", "movi");
    assertEquals(expectedTokens, actualTokens);
  }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
    }
  }

  protected static void loadTwitterData() throws IOException {
    TWITTER_SAMPLES = new ArrayList<TextInstance<Boolean>>();
    TwitterTokenizer tokenizer = new TwitterTokenizer(2, 2);

    FileInputStream is = new FileInputStream(TWITTER_FILE);
    BufferedReader br = new BufferedReader(new InputStreamReader(is));
    try {
      String line;
      while ((line = br.readLine()) != null) {
        try {
          String[] values = line.split(",");

          Boolean label = !values[0].equals("0");
          String text = line.substring(line.indexOf(",") + 1);

          TWITTER_SAMPLES.add(new TextInstance<Boolean>(label, tokenizer.tokenize(text)));
        } catch (Exception ex) {
          System.err.println("Skipped twitter sample because it can't be parsed : " + line);
        }
      }
View Full Code Here
TOP

Related Classes of com.github.pmerienne.trident.ml.preprocessing.TwitterTokenizer

  • com.github.pmerienne.trident.ml.preprocessing.TwitterTokenizerTest
  • com.github.pmerienne.trident.ml.testing.data.Datasets
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.