Package com.clearnlp.util.pair

Examples of com.clearnlp.util.pair.StringBooleanPair

180
181
182
183
184
185
186
187
188
189
190
191
192
193
 
  private void handleSingleToken(List<StringBooleanPair> lTokens)
  {
    if (lTokens.size() == 1)
    {
      StringBooleanPair p = lTokens.get(0);
     
      if (p.s.equalsIgnoreCase("no."))
      {
        p.s = p.s.substring(0, p.s.length()-1);
        lTokens.add(new StringBooleanPair(".", false));
      }
    }
  }
View Full Code Here
408
409
410
411
412
413
414
415
416
  protected List<StringBooleanPair> tokenizeWhiteSpaces(String str)
  {
    List<StringBooleanPair> tokens = new ArrayList<StringBooleanPair>();
   
    for (String token : PTLib.splitWhiteSpaces(str))
      tokens.add(new StringBooleanPair(token, false));
   
    return tokens;
  }
View Full Code Here
498
499
500
501
502
503
504
505
506
507
508
      if (oToken.b || (idx = M_COMPOUNDS.get(oToken.s.toLowerCase()) - 1) < 0)
        nTokens.add(oToken);
      else
      {
        for (IntIntPair p : L_COMPOUNDS.get(idx))
          nTokens.add(new StringBooleanPair(oToken.s.substring(p.i1, p.i2), true));
      }
    }
   
    return nTokens;
  }
View Full Code Here
525
526
527
528
529
530
531
532
533
534
  private void tokenizePatternsAux(List<StringBooleanPair> tokens, Replacer rep, String str)
  {
    for (String token : P_DELIM.split(rep.replace(str).trim()))
    {
      if (token.startsWith(S_PROTECTED))
        tokens.add(new StringBooleanPair(token.substring(N_PROTECTED), true));
      else if (!token.isEmpty())
        tokens.add(new StringBooleanPair(token, false));
    }
  }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
  private void handleSingleToken(List<StringBooleanPair> lTokens)
  {
    if (lTokens.size() == 1)
    {
      StringBooleanPair p = lTokens.get(0);
     
      if (p.s.equalsIgnoreCase("no."))
      {
        p.s = p.s.substring(0, p.s.length()-1);
        lTokens.add(new StringBooleanPair(".", false));
      }
    }
  }
View Full Code Here
432
433
434
435
436
437
438
439
440
  protected List<StringBooleanPair> tokenizeWhiteSpaces(String str)
  {
    List<StringBooleanPair> tokens = new ArrayList<StringBooleanPair>();
   
    for (String token : PTLib.splitWhiteSpaces(str))
      tokens.add(new StringBooleanPair(token, false));
   
    return tokens;
  }
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
      if (oToken.b || (idx = M_COMPOUNDS.get(oToken.s.toLowerCase()) - 1) < 0)
        nTokens.add(oToken);
      else
      {
        for (IntIntPair p : L_COMPOUNDS.get(idx))
          nTokens.add(new StringBooleanPair(oToken.s.substring(p.i1, p.i2), true));
      }
    }
   
    return nTokens;
  }
View Full Code Here
549
550
551
552
553
554
555
556
557
558
  private void tokenizePatternsAux(List<StringBooleanPair> tokens, Replacer rep, String str)
  {
    for (String token : P_DELIM.split(rep.replace(str).trim()))
    {
      if (token.startsWith(S_PROTECTED))
        tokens.add(new StringBooleanPair(token.substring(N_PROTECTED), true));
      else if (!token.isEmpty())
        tokens.add(new StringBooleanPair(token, false));
    }
  }
View Full Code Here
TOP

Related Classes of com.clearnlp.util.pair.StringBooleanPair

  • com.clearnlp.tokenization.EnglishTokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.