Package net.sourceforge.segment

Examples of net.sourceforge.segment.TextIterator

85
86
87
88
89
90
91
92
93
94
95
  }

  @Override
  public final List<String> tokenize(final String text) {
    final List<String> segments = new ArrayList<>();
    final TextIterator textIterator = new SrxTextIterator(document, languageCode + parCode, text);
    while (textIterator.hasNext()) {
      segments.add(textIterator.next());
    }
    return segments;
  }
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    this.document = parser.parse(reader);
    this.languageCode = languageCode;
  }
 
  public List<String> split(String string) {
    TextIterator textIterator =
      new SrxTextIterator(document, languageCode, string);
    List<String> segmentList = new ArrayList<String>();
    while(textIterator.hasNext()) {
      String segment = textIterator.next();
      segmentList.add(segment);
    }
    return segmentList;
    }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
  public void testMaxLookbehindConstructLength() {
    Map<String, Object> parameterMap = new HashMap<String, Object>();
    parameterMap.put(SrxTextIterator.MAX_LOOKBEHIND_CONSTRUCT_LENGTH_PARAMETER, 2);
   
    String text = merge(MAX_LOOKBEHIND_CONSTRUCT_LENGTH_RESULT);
    TextIterator textIterator =
      new SrxTextIterator(MAX_LOOKBEHIND_CONSTRUCT_LENGTH_DOCUMENT,
          "", text, parameterMap);
    List<String> segmentList = segment(textIterator);
    String[] segmentArray = segmentList.toArray(new String[segmentList.size()]);
View Full Code Here
610
611
612
613
614
615
616
617
618
619
620
  private void performTest(String[] expectedResult,
      SrxDocument document, String languageCode) {
   
    String text = merge(expectedResult);
   
    TextIterator textIterator;
    List<String> segmentList;
   
    textIterator = getTextIterator(text, document, languageCode);
    segmentList = segment(textIterator);
   
View Full Code Here
432
433
434
435
436
437
438
439
440
441
442
      System.out.println("Segmenting... ");
    }

    long start = System.currentTimeMillis();

    TextIterator textIterator = createTextIterator(commandLine,
        document, reader, profile);

    performSegment(commandLine, textIterator, writer, profile);
   
    if (profile) {
View Full Code Here
446
447
448
449
450
451
452
453
454
455
456
  }
 
  private TextIterator createTextIterator(CommandLine commandLine,
      SrxDocument document, Reader reader, boolean profile) {
   
    TextIterator textIterator;
   
    String languageCode = commandLine.getOptionValue('l');
    if (languageCode == null) {
      languageCode = "";
    }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    }
  }

  static List<String> tokenize(String text, SrxDocument srxDocument, String code) {
    List<String> segments = new ArrayList<>();
    TextIterator textIterator = new SrxTextIterator(srxDocument, code, text);
    while (textIterator.hasNext()) {
      segments.add(textIterator.next());
    }
    return segments;
  }
View Full Code Here
TOP

Related Classes of net.sourceforge.segment.TextIterator

  • net.sourceforge.align.filter.modifier.modify.split.SrxSplitAlgorithm
  • net.sourceforge.segment.srx.AbstractSrxTextIteratorTest
  • net.sourceforge.segment.srx.SrxTextIteratorStringTest
  • net.sourceforge.segment.ui.console.Segment
  • org.languagetool.tokenizers.SRXSentenceTokenizer
  • org.languagetool.tokenizers.SrxTools
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.