Examples of com.ibm.icu.text.RuleBasedBreakIterator

com.ibm.icu.text.RuleBasedBreakIterator
Rule Based Break Iterator This is a port of the C++ class RuleBasedBreakIterator from ICU4C. @stable ICU 2.0

    
    /**
     * Tests the method hashCode() of RuleBasedBreakIterator
     **/
    public void TestHashCode() {
        RuleBasedBreakIterator bi1 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.getDefault()); 
        RuleBasedBreakIterator bi3 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.getDefault()); 
        RuleBasedBreakIterator bi2 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault());
        logln("Testing hashCode()");
        bi1.setText("Hash code");
        bi2.setText("Hash code");
        bi3.setText("Hash code");
        RuleBasedBreakIterator bi1clone = (RuleBasedBreakIterator) bi1.clone();
        RuleBasedBreakIterator bi2clone = (RuleBasedBreakIterator) bi2.clone();
        if (bi1.hashCode() != bi1clone.hashCode()
            || bi1.hashCode() != bi3.hashCode()
            || bi1clone.hashCode() != bi3.hashCode()
            || bi2.hashCode() != bi2clone.hashCode())
            errln("ERROR: identical objects have different hashcodes");
        
        if (bi1.hashCode() == bi2.hashCode()
            || bi2.hashCode() == bi3.hashCode()
            || bi1clone.hashCode() == bi2clone.hashCode()
            || bi1clone.hashCode() == bi2.hashCode())
            errln("ERROR: different objects have same hashcodes");
    }

View Full Code Here

    public void TestGetSetText() {
        logln("Testing getText setText ");
        String str1 = "first string.";
        String str2 = "Second string.";
        //RuleBasedBreakIterator charIter1 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.getDefault()); 
        RuleBasedBreakIterator wordIter1 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault()); 
        CharacterIterator text1 = new StringCharacterIterator(str1);
        //CharacterIterator text1Clone = (CharacterIterator) text1.clone();
        //CharacterIterator text2 = new StringCharacterIterator(str2);
        wordIter1.setText(str1);
        if (!wordIter1.getText().equals(text1))
            errln("ERROR:1 error in setText or getText ");
        if (wordIter1.current() != 0)
            errln("ERROR:1 setText did not set the iteration position to the beginning of the text, it is"
                   + wordIter1.current() + "\n"); 
        wordIter1.next(2);
        wordIter1.setText(str2);
        if (wordIter1.current() != 0)
            errln("ERROR:2 setText did not reset the iteration position to the beginning of the text, it is"
                    + wordIter1.current() + "\n"); 
        //ICU4J has remove the method adoptText
        /*
        charIter1.adoptText(text1Clone);
        if (wordIter1.getText() == charIter1.getText()
            || wordIter1.getText() != text2

View Full Code Here

    public void TestFirstNextFollowing() {
        int p, q;
        String testString = "This is a word break. Isn't it? 2.25";
        logln("Testing first() and next(), following() with custom rules");
        logln("testing word iterator - string :- \"" + testString + "\"\n");
        RuleBasedBreakIterator wordIter1 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault());
        wordIter1.setText(testString);
        p = wordIter1.first();
        if (p != 0)
            errln("ERROR: first() returned" + p + "instead of 0");
        q = wordIter1.next(9);
        doTest(testString, p, q, 20, "This is a word break");
        p = q;
        q = wordIter1.next();
        doTest(testString, p, q, 21, ".");
        p = q;
        q = wordIter1.next(3);
        doTest(testString, p, q, 28, " Isn't ");
        p = q;
        q = wordIter1.next(2);
        doTest(testString, p, q, 31, "it?");
        q = wordIter1.following(2);
        doTest(testString, 2, q, 4, "is");
        q = wordIter1.following(22);
        doTest(testString, 22, q, 27, "Isn't");
        wordIter1.last();
        p = wordIter1.next();
        q = wordIter1.following(wordIter1.last());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: next()/following() at last position returned #"
                    + p + " and " + q + " instead of" + testString.length() + "\n"); 
        RuleBasedBreakIterator charIter1 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.getDefault()); 
        testString = "Write hindi here. "; 
        logln("testing char iter - string:- \"" + testString + "\"");
        charIter1.setText(testString);
        p = charIter1.first();
        if (p != 0)
            errln("ERROR: first() returned" + p + "instead of 0");
        q = charIter1.next();
        doTest(testString, p, q, 1, "W");
        p = q;
        q = charIter1.next(4);
        doTest(testString, p, q, 5, "rite");
        p = q;
        q = charIter1.next(12);
        doTest(testString, p, q, 17, " hindi here.");
        p = q;
        q = charIter1.next(-6);
        doTest(testString, p, q, 11, " here.");
        p = q;
        q = charIter1.next(6);
        doTest(testString, p, q, 17, " here.");
        p = charIter1.following(charIter1.last());
        q = charIter1.next(charIter1.last());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: following()/next() at last position returned #"
                    + p + " and " + q + " instead of" + testString.length()); 
        testString = "Hello! how are you? I'am fine. Thankyou. How are you doing? This  costs $20,00,000."; 
        RuleBasedBreakIterator sentIter1 = (RuleBasedBreakIterator) BreakIterator.getSentenceInstance(Locale.getDefault()); 
        logln("testing sentence iter - String:- \"" + testString + "\"");
        sentIter1.setText(testString);
        p = sentIter1.first();
        if (p != 0)
            errln("ERROR: first() returned" + p + "instead of 0");
        q = sentIter1.next();
        doTest(testString, p, q, 7, "Hello! ");
        p = q;
        q = sentIter1.next(2);
        doTest(testString, p, q, 31, "how are you? I'am fine. ");
        p = q;
        q = sentIter1.next(-2);
        doTest(testString, p, q, 7, "how are you? I'am fine. ");
        p = q;
        q = sentIter1.next(4);
        doTest(testString, p, q, 60, "how are you? I'am fine. Thankyou. How are you doing? ");
        p = q;
        q = sentIter1.next();
        doTest(testString, p, q, 83, "This  costs $20,00,000.");
        q = sentIter1.following(1);
        doTest(testString, 1, q, 7, "ello! ");
        q = sentIter1.following(10);
        doTest(testString, 10, q, 20, " are you? ");
        q = sentIter1.following(20);
        doTest(testString, 20, q, 31, "I'am fine. ");
        p = sentIter1.following(sentIter1.last());
        q = sentIter1.next(sentIter1.last());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: following()/next() at last position returned #"
                    + p + " and " + q + " instead of" + testString.length()); 
        testString = "Hello! how\r\n (are)\r you? I'am fine- Thankyou. foo\u00a0bar How, are, you? This, costs $20,00,000."; 
        logln("(UnicodeString)testing line iter - String:- \"" + testString + "\"");
        RuleBasedBreakIterator lineIter1 = (RuleBasedBreakIterator) BreakIterator.getLineInstance(Locale.getDefault()); 
        lineIter1.setText(testString);
        p = lineIter1.first();
        if (p != 0)
            errln("ERROR: first() returned" + p + "instead of 0");
        q = lineIter1.next();
        doTest(testString, p, q, 7, "Hello! ");
        p = q;
        p = q;
        q = lineIter1.next(4);
        doTest(testString, p, q, 20, "how\r\n (are)\r ");
        p = q;
        q = lineIter1.next(-4);
        doTest(testString, p, q, 7, "how\r\n (are)\r ");
        p = q;
        q = lineIter1.next(6);
        doTest(testString, p, q, 30, "how\r\n (are)\r you? I'am ");
        p = q;
        q = lineIter1.next();
        doTest(testString, p, q, 36, "fine- ");
        p = q;
        q = lineIter1.next(2);
        doTest(testString, p, q, 54, "Thankyou. foo\u00a0bar ");
        q = lineIter1.following(60);
        doTest(testString, 60, q, 64, "re, ");
        q = lineIter1.following(1);
        doTest(testString, 1, q, 7, "ello! ");
        q = lineIter1.following(10);
        doTest(testString, 10, q, 12, "\r\n");
        q = lineIter1.following(20);
        doTest(testString, 20, q, 25, "you? ");
        p = lineIter1.following(lineIter1.last());
        q = lineIter1.next(lineIter1.last());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: following()/next() at last position returned #"
                    + p + " and " + q + " instead of" + testString.length()); 
    }

View Full Code Here

    public void TestLastPreviousPreceding() {
        int p, q;
        String testString = "This is a word break. Isn't it? 2.25 dollars";
        logln("Testing last(),previous(), preceding() with custom rules");
        logln("testing word iteration for string \"" + testString + "\"");
        RuleBasedBreakIterator wordIter1 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault()); 
        wordIter1.setText(testString);
        p = wordIter1.last();
        if (p != testString.length()) {
            errln("ERROR: first() returned" + p + "instead of" + testString.length());
        }
        q = wordIter1.previous();
        doTest(testString, p, q, 37, "dollars");
        p = q;
        q = wordIter1.previous();
        doTest(testString, p, q, 36, " ");
        q = wordIter1.preceding(25);
        doTest(testString, 25, q, 22, "Isn");
        p = q;
        q = wordIter1.previous();
        doTest(testString, p, q, 21, " ");
        q = wordIter1.preceding(20);
        doTest(testString, 20, q, 15, "break");
        p = wordIter1.preceding(wordIter1.first());
        if (p != BreakIterator.DONE)
            errln("ERROR: preceding()  at starting position returned #" + p + " instead of 0");
        testString = "Hello! how are you? I'am fine. Thankyou. How are you doing? This  costs $20,00,000."; 
        logln("testing sentence iter - String:- \"" + testString + "\"");
        RuleBasedBreakIterator sentIter1 = (RuleBasedBreakIterator) BreakIterator.getSentenceInstance(Locale.getDefault()); 
        sentIter1.setText(testString);
        p = sentIter1.last();
        if (p != testString.length())
            errln("ERROR: last() returned" + p + "instead of " + testString.length());
        q = sentIter1.previous();
        doTest(testString, p, q, 60, "This  costs $20,00,000.");
        p = q;
        q = sentIter1.previous();
        doTest(testString, p, q, 41, "How are you doing? ");
        q = sentIter1.preceding(40);
        doTest(testString, 40, q, 31, "Thankyou.");
        q = sentIter1.preceding(25);
        doTest(testString, 25, q, 20, "I'am ");
        sentIter1.first();
        p = sentIter1.previous();
        q = sentIter1.preceding(sentIter1.first());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: previous()/preceding() at starting position returned #"
                    + p + " and " + q + " instead of 0\n"); 
        testString = "Hello! how are you? I'am fine. Thankyou. How are you doing? This\n costs $20,00,000."; 
        logln("testing line iter - String:- \"" + testString + "\"");
        RuleBasedBreakIterator lineIter1 = (RuleBasedBreakIterator) BreakIterator.getLineInstance(Locale.getDefault());
        lineIter1.setText(testString);
        p = lineIter1.last();
        if (p != testString.length())
            errln("ERROR: last() returned" + p + "instead of " + testString.length());
        q = lineIter1.previous();
        doTest(testString, p, q, 72, "$20,00,000.");
        p = q;
        q = lineIter1.previous();
        doTest(testString, p, q, 66, "costs ");
        q = lineIter1.preceding(40);
        doTest(testString, 40, q, 31, "Thankyou.");
        q = lineIter1.preceding(25);
        doTest(testString, 25, q, 20, "I'am ");
        lineIter1.first();
        p = lineIter1.previous();
        q = lineIter1.preceding(sentIter1.first());
        if (p != BreakIterator.DONE || q != BreakIterator.DONE)
            errln("ERROR: previous()/preceding() at starting position returned #"
                    + p + " and " + q + " instead of 0\n");
    }

View Full Code Here


  // tests default rules based character iteration.
  // Builds a new iterator from the source rules in the default (prebuilt) iterator.
  //
  public void TestDefaultRuleBasedCharacterIteration(){
      RuleBasedBreakIterator rbbi=(RuleBasedBreakIterator)BreakIterator.getCharacterInstance();
      logln("Testing the RBBI for character iteration by using default rules");


      //fetch the rules used to create the above RuleBasedBreakIterator
      String defaultRules=rbbi.toString();
      
      RuleBasedBreakIterator charIterDefault=null;
      try{
          charIterDefault   = new RuleBasedBreakIterator(defaultRules); 
      }catch(IllegalArgumentException iae){
          errln("ERROR: failed construction in TestDefaultRuleBasedCharacterIteration()"+ iae.toString());
      }


      Vector chardata = new Vector();

View Full Code Here


  }


  public void TestDefaultRuleBasedWordIteration(){
      logln("Testing the RBBI for word iteration using default rules");
      RuleBasedBreakIterator rbbi=(RuleBasedBreakIterator)BreakIterator.getWordInstance();
      //fetch the rules used to create the above RuleBasedBreakIterator
      String defaultRules=rbbi.toString();
      
      RuleBasedBreakIterator wordIterDefault=null;
      try{
      wordIterDefault   = new RuleBasedBreakIterator(defaultRules); 
      }catch(IllegalArgumentException iae){
          errln("ERROR: failed construction in TestDefaultRuleBasedWordIteration() -- custom rules"+ iae.toString());
      }


      Vector worddata = new Vector();

View Full Code Here

//  private static final String kParagraphSeparator = "\u2029";
  private static final String kLineSeparator      = "\u2028";


  public void TestDefaultRuleBasedSentenceIteration(){
      logln("Testing the RBBI for sentence iteration using default rules");
      RuleBasedBreakIterator rbbi=(RuleBasedBreakIterator)BreakIterator.getSentenceInstance();
      
      //fetch the rules used to create the above RuleBasedBreakIterator
      String defaultRules=rbbi.toString();
      RuleBasedBreakIterator sentIterDefault=null;
      try{
          sentIterDefault   = new RuleBasedBreakIterator(defaultRules); 
      }catch(IllegalArgumentException iae){
          errln("ERROR: failed construction in TestDefaultRuleBasedSentenceIteration()" + iae.toString());
      }
      
      Vector sentdata = new Vector();

View Full Code Here

       generalIteratorTest(sentIterDefault, sentdata);
  }
   
  public void TestDefaultRuleBasedLineIteration(){
      logln("Testing the RBBI for line iteration using default rules");
      RuleBasedBreakIterator rbbi=(RuleBasedBreakIterator)RuleBasedBreakIterator.getLineInstance();
      //fetch the rules used to create the above RuleBasedBreakIterator
      String defaultRules=rbbi.toString();
      RuleBasedBreakIterator lineIterDefault=null;
      try{
          lineIterDefault   = new RuleBasedBreakIterator(defaultRules); 
      }catch(IllegalArgumentException iae){
          errln("ERROR: failed construction in TestDefaultRuleBasedLineIteration()" + iae.toString());
      }


     Vector linedata = new Vector();

View Full Code Here

       }
   }
   private void doMultipleSelectionTest(RuleBasedBreakIterator iterator, String testText)
   {
       logln("Multiple selection test...");
       RuleBasedBreakIterator testIterator = (RuleBasedBreakIterator)iterator.clone();
       int offset = iterator.first();
       int testOffset;
       int count = 0;


       do {
           testOffset = testIterator.first();
           testOffset = testIterator.next(count);
           logln("next(" + count + ") -> " + testOffset);
           if (offset != testOffset)
               errln("next(n) and next() not returning consistent results: for step " + count + ", next(n) returned " + testOffset + " and next() had " + offset);


           if (offset != RuleBasedBreakIterator.DONE) {
               count++;
               offset = iterator.next();
           }
       } while (offset != RuleBasedBreakIterator.DONE);


       // now do it backwards...
       offset = iterator.last();
       count = 0;


       do {
           testOffset = testIterator.last();
           testOffset = testIterator.next(count);
           logln("next(" + count + ") -> " + testOffset);
           if (offset != testOffset)
               errln("next(n) and next() not returning consistent results: for step " + count + ", next(n) returned " + testOffset + " and next() had " + offset);


           if (offset != RuleBasedBreakIterator.DONE) {

View Full Code Here

        String dictionaryName = kindNames[kind] + "BreakDictionary";
        
        String[] classNames = bundle.getStringArray("BreakIteratorClasses");
        String rules = bundle.getString(rulesName);
        if (classNames[kind].equals("RuleBasedBreakIterator")) {
            iter = new RuleBasedBreakIterator(rules);
        }
        else if (classNames[kind].equals("DictionaryBasedBreakIterator")) {
            try {
                String dictionaryPath = bundle.getString(dictionaryName);
                InputStream dictionary = bundle.getClass().getResourceAsStream(dictionaryPath);

View Full Code Here

0 1 2

TOP

Related Classes of com.ibm.icu.text.RuleBasedBreakIterator

com.ibm.icu.dev.test.lang.UCharacterCaseTest

com.ibm.icu.dev.test.rbbi.RBBIAPITest

com.ibm.icu.dev.test.rbbi.RBBITest

com.ibm.icu.dev.test.rbbi.RBBITestMonkey

com.ibm.icu.dev.test.rbbi.SimpleBITest

com.ibm.icu.impl.CharTrie

org.apache.lucene.analysis.icu.RBBIRuleCompiler

org.apache.lucene.analysis.icu.segmentation.DefaultICUTokenizerConfig

org.apache.lucene.analysis.icu.segmentation.ICUTokenizerFactory

java.io.InputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.