Package com.clearnlp.util.map

Examples of com.clearnlp.util.map.Prob1DMap

56
57
58
59
60
61
62
{
  Prob1DMap g_map;
 
  public CTExtractPhrases()
  {
    g_map = new Prob1DMap();
  }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
  /** Called by {@link #getCollector(JointReader, String, JointFtrXml[], String[], int)}. */
  protected Set<String> getLowerSimplifiedForms(JointReader reader, JointFtrXml xml, String[] trainFiles, int devId)
  {
    Set<String> set = new HashSet<String>();
    int i, j, len, size = trainFiles.length;
    Prob1DMap map = new Prob1DMap();
    DEPTree tree;
   
    LOG.info("Collecting word-forms:\n");
   
    for (i=0; i<size; i++)
    {
      if (devId == i)  continue;
     
      reader.open(UTInput.createBufferedFileReader(trainFiles[i]));
      set.clear();
     
      while ((tree = reader.next()) != null)
      {
        NLPProcess.simplifyForms(tree);
        len = tree.size();
       
        for (j=1; j<len; j++)
          set.add(tree.get(j).lowerSimplifiedForm);
      }
     
      reader.close();
      map.addAll(set);
      LOG.debug(".");
    }  LOG.debug("\n");
   
    return map.toSet(xml.getDocumentFrequencyCutoff());
  }
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
  public void countDEP(String[] args) throws Exception
  {
    DEPReader reader = new DEPReader(0, 1, 2, 3, 4, 6, 7);
    reader.open(UTInput.createBufferedFileReader(args[0]));
    Prob1DMap map = new Prob1DMap();
    int[] count = {0, 0};
    DEPTree tree;
   
    while ((tree = reader.next()) != null)
      countDEP(tree, count, map);
   
    reader.close();
    System.out.printf("%5.2f (%d/%d)\n", 100d*count[0]/count[1], count[0], count[1]);
    for (StringIntPair p : map.toSortedList())
      System.out.println(p.s+" "+p.i);
  }
View Full Code Here
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
 
  private Set<String> getLowerSimplifiedFormsByDocumentFrequencies(JointReader reader, String[] filenames, int devId, int cutoff)
  {
    int i, j, len, size = filenames.length;
    Set<String> set = Sets.newHashSet();
    Prob1DMap map = new Prob1DMap();
    DEPTree tree;
   
    LOG.info(String.format("Collecting simplified-forms: cutoff = %d\n", cutoff));
   
    for (i=0; i<size; i++)
    {
      if (i == devId)  continue;
      reader.open(UTInput.createBufferedFileReader(filenames[i]));
      set.clear();
     
      while ((tree = reader.next()) != null)
      {
        len = tree.size();
       
        for (j=1; j<len; j++)
          set.add(MPLib.getSimplifiedLowercaseWordForm(tree.get(j).form));
      }
     
      map.addAll(set);
      reader.close();
      LOG.info(".");
    }  LOG.info("\n");
   
    return map.toSet(cutoff);
  }
View Full Code Here
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
    }
    }
 
  public void collectPhrases(String[] args) throws Exception
  {
    Prob1DMap map = new Prob1DMap();
    CTReader reader = new CTReader();
    reader.open(UTInput.createBufferedFileReader(args[0]));
    CTTree tree;
   
    while ((tree = reader.nextTree()) != null)
      traverse(tree.getRoot(), map, args[2]);

    PrintStream ugram = UTOutput.createPrintBufferedFileStream(args[1]+".1gram");
    PrintStream ngram = UTOutput.createPrintBufferedFileStream(args[1]+".ngram");
    String s;
   
    for (StringIntPair p : map.toSortedList())
    {
      s = p.s+" "+p.i;
      if (p.s.contains("_"))  ngram.println(s);
      else          ugram.println(s);
    }
View Full Code Here
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
 
  private Set<String> getLowerSimplifiedFormsByDocumentFrequencies(JointReader reader, String[] filenames, int devId, int cutoff, int maxCount)
  {
    int i, j, len, count = 0, size = filenames.length;
    Set<String> set = Sets.newHashSet();
    Prob1DMap map = new Prob1DMap();
    DEPTree tree;
   
    LOG.info(String.format("Collecting simplified-forms: cutoff = %d, max = %d\n", cutoff, maxCount));
   
    for (i=0; i<size; i++)
    {
      if (i == devId)  continue;
      reader.open(UTInput.createBufferedFileReader(filenames[i]));
     
      while ((tree = reader.next()) != null)
      {
        len = tree.size();
       
        for (j=1; j<len; j++)
          set.add(MPLib.getSimplifiedLowercaseWordForm(tree.get(j).form));
       
        if ((count += len) >= maxCount)
        {
          map.addAll(set);
          LOG.info(".");
          set.clear();
          count = 0;
        }
      }
     
      reader.close();
    }  LOG.info("\n");
   
    if (!set.isEmpty()) map.addAll(set);
    return map.toSet(cutoff);
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
 
  /** Constructs a semantic role labeler for collecting lexica. */
  public AbstractSRLabeler(JointFtrXml[] xmls, AbstractFrames frames)
  {
    super(xmls);
    m_down   = new Prob1DMap();
    m_up     = new Prob1DMap();
    m_frames = frames;
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
 
  /** Constructs a semantic role labeler for collecting lexica. */
  public AbstractSRLabeler(JointFtrXml[] xmls, AbstractFrames frames)
  {
    super(xmls);
    m_down   = new Prob1DMap();
    m_up     = new Prob1DMap();
    m_frames = frames;
  }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  /** Called by {@link #getCollector(JointReader, String, JointFtrXml[], String[], int)}. */
  private Set<String> getLowerSimplifiedForms(JointReader reader, JointFtrXml xml, String[] trainFiles, int devId)
  {
    Set<String> set = new HashSet<String>();
    int i, j, len, size = trainFiles.length;
    Prob1DMap map = new Prob1DMap();
    DEPTree tree;
   
    LOG.info("Collecting word-forms:\n");
   
    for (i=0; i<size; i++)
    {
      if (devId == i)  continue;
     
      reader.open(UTInput.createBufferedFileReader(trainFiles[i]));
      set.clear();
     
      while ((tree = reader.next()) != null)
      {
        NLPProcess.simplifyForms(tree);
        len = tree.size();
       
        for (j=1; j<len; j++)
          set.add(tree.get(j).lowerSimplifiedForm);
      }
     
      reader.close();
      map.addAll(set);
      LOG.debug(".");
    }  LOG.debug("\n");
   
    return map.toSet(xml.getDocumentFrequencyCutoff());
  }
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
  }

  protected Set<String> getSimplifiedFormsByDocumentFrequencies(JointReader reader, String[] filenames, int cutoff, int maxCount)
  {
    Set<String> set = Sets.newHashSet();
    Prob1DMap map = new Prob1DMap();
    int j, len, count = 0;
    DEPTree tree;
   
    for (String filename : filenames)
    {
      reader.open(UTInput.createBufferedFileReader(filename));
     
      while ((tree = reader.next()) != null)
      {
        len = tree.size();
       
        for (j=1; j<len; j++)
          set.add(MPLib.getSimplifiedLowercaseWordForm(tree.get(j).form));
       
        if ((count += len) >= maxCount)
        {
          map.addAll(set);
          count = 0;
          set = Sets.newHashSet();
        }
      }
     
      reader.close();
    }
   
    if (!set.isEmpty()) map.addAll(set);
    return map.toSet(cutoff);
  }
View Full Code Here
TOP

Related Classes of com.clearnlp.util.map.Prob1DMap

  • com.clearnlp.bin.NLPDevelop
  • com.clearnlp.component.srl.AbstractSRLabeler
  • com.clearnlp.demo.CTExtractPhrases
  • com.clearnlp.experiment.Tmp
  • com.clearnlp.nlp.NLPDevelop
  • com.clearnlp.nlp.train.POSTrainer
  • com.clearnlp.util.pair.StringIntPair
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.