Package com.clearnlp.tokenization

Examples of com.clearnlp.tokenization.AbstractTokenizer

127
128
129
130
131
132
133
134
135
136
137
public class Tmp
{
  public Tmp(String[] args) throws Exception
  {
    BufferedReader reader = UTInput.createBufferedFileReader(args[0]);
    AbstractTokenizer tokenizer = NLPGetter.getTokenizer("en");
    Map<String,String> redirects = Maps.newHashMap();
//    Pattern paren = Pattern.compile("\\(.+?\\)$");
    Set<String> titles = Sets.newHashSet();
    String line, title, redirect;
    PTHtml html = new PTHtml();
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    String readerType = reader.getType();
    boolean bTwit = isTwit(eConfig);
    PrintStream fout;
   
    AbstractSegmenter segmenter = readerType.equals(AbstractReader.TYPE_RAW)  ? getSegmenter(eConfig, bTwit) : null;
    AbstractTokenizer tokenizer = readerType.equals(AbstractReader.TYPE_LINE) ? getTokenizer(eConfig, bTwit) : null;
    AbstractComponent[] components = null;
   
    if (modelFile != null && !modelFile.equals(UNConstant.EMPTY))
    {
      if (new File(modelFile).isFile())
View Full Code Here
169
170
171
172
173
174
175
176
177
178
 
//  ===================================== COMPONENT GETTERS =====================================

  protected AbstractSegmenter getSegmenter(Element eConfig, boolean twit) throws IOException
  {
    AbstractTokenizer tokenizer = getTokenizer(eConfig, twit);
    String language = getLanguage(eConfig);
   
    return NLPGetter.getSegmenter(language, tokenizer);
  }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
    return NLPGetter.getSegmenter(language, tokenizer);
  }
 
  protected AbstractTokenizer getTokenizer(Element eConfig, boolean twit) throws IOException
  {
    AbstractTokenizer tokenizer = NLPGetter.getTokenizer(getLanguage(eConfig));
    tokenizer.setTwit(twit);
   
    return tokenizer;
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  {
    initArgs(args);
   
    try
    {
      AbstractTokenizer tokenizer = NLPGetter.getTokenizer(s_language);
      AbstractSegmenter segmenter = i_format.equals(AbstractReader.TYPE_RAW) ? NLPGetter.getSegmenter(s_language, tokenizer) : null;
      List<String[]>    filenames = getFilenames(s_inputPath, s_inputExt, s_outputExt);
      boolean outLine = o_format.equals(AbstractReader.TYPE_LINE);
      tokenizer.setTwit(b_twit);
     
      for (String[] io : filenames)
      {
        System.out.println(io[0]);
        tokenize(tokenizer, segmenter, io[0], io[1], outLine);
View Full Code Here
TOP

Related Classes of com.clearnlp.tokenization.AbstractTokenizer

  • com.clearnlp.experiment.Tmp
  • com.clearnlp.nlp.decode.NLPDecoder
  • com.clearnlp.run.Tokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.