Package edu.stanford.nlp.ling

Examples of edu.stanford.nlp.ling.CoreLabel$CoreLabelFactory

510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
          System.err.println("Length:\t" + sentence.size());
        }
        boolean printSpace = false;
        for (HasWord word : sentence) {
          if (printOriginalText) {
            CoreLabel cl = (CoreLabel) word;
            if ( ! printSpace) {
              pw.print(cl.get(CoreAnnotations.BeforeAnnotation.class));
              printSpace = true;
            }
            pw.print(cl.get(CoreAnnotations.OriginalTextAnnotation.class));
            pw.print(cl.get(CoreAnnotations.AfterAnnotation.class));
          } else {
            if (printSpace) pw.print(" ");
            printSpace = true;
            pw.print(word.word());
          }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    for (Entry<String, List<CoreLabel>> en : sents.entrySet()) {
      List<CoreLabel> value = en.getValue();
      CoreLabel[] sent = value.toArray(new CoreLabel[value.size()]);

      for (int i = 0; i < sent.length; i++) {
        CoreLabel l = sent[i];

        boolean chooseThis = false;
        boolean ignoreclass = false;
        Boolean datumlabel = false;
        for (Class cl : otherIgnoreClasses.keySet()) {
          if ((Boolean) l.get(cl)) {    // cast is needed for jdk 1.6
            ignoreclass = true;
          }
        }
        if (l.get(answerClass).equals(answerLabel)) {
          datumlabel = true;
          chooseThis = true;
          numpos++;
        }
        if (chooseThis) {
          chosen.add(new Pair<String, Integer>(en.getKey(), i));

          Counter<ScorePhraseMeasures> feat = null;
          if (forLearningPattern) {
            feat = getPhraseFeaturesForPattern(label, l.word());
          } else {
            feat = getFeatures(label, l.word(), wordsPatExtracted.getCounter(l.word()), allSelectedPatterns);
          }
          RVFDatum<String, ScorePhraseMeasures> datum = new RVFDatum<String, ScorePhraseMeasures>(feat, datumlabel.toString());
          dataset.add(datum);
        }
      }

      for (int i = 0; i < sent.length; i++) {
        CoreLabel l = sent[i];
        if (numneg >= numpos)
          break;
        boolean chooseThis = false;
        boolean ignoreclass = false;
        Boolean datumlabel = false;
        if (l.get(answerClass).equals(answerLabel)) {
          continue;
        } else if ((ignoreclass || negativeWords.contains(l.word().toLowerCase())) && getRandomBoolean(rneg, perSelectNeg)) {
          chooseThis = true;
          datumlabel = false;
          numneg++;
        } else if (getRandomBoolean(r, perSelectRand)) {
          chooseThis = true;
          datumlabel = false;
          numneg++;
        } else
          chooseThis = false;
        if (chooseThis) {
          chosen.add(new Pair<String, Integer>(en.getKey(), i));
          Counter<ScorePhraseMeasures> feat = null;
          if (forLearningPattern) {
            feat = getPhraseFeaturesForPattern(label, l.word());
          } else {
            feat = getFeatures(label, l.word(), wordsPatExtracted.getCounter(l.word()), allSelectedPatterns);
          }
          RVFDatum<String, ScorePhraseMeasures> datum = new RVFDatum<String, ScorePhraseMeasures>(feat, datumlabel.toString());
          dataset.add(datum);
        }
      }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
  }


  protected Collection<String> featuresC(PaddedList<? extends CoreLabel> cInfo, int loc) {
    Collection<String> features = new ArrayList<String>();
    CoreLabel c = cInfo.get(loc);
    CoreLabel c2 = cInfo.get(loc + 1);
    CoreLabel c3 = cInfo.get(loc + 2);
    CoreLabel p = cInfo.get(loc - 1);
    CoreLabel p2 = cInfo.get(loc - 2);
    CoreLabel p3 = cInfo.get(loc - 3);
    String charc = c.getString(CoreAnnotations.CharAnnotation.class);
    String charc2 = c2.getString(CoreAnnotations.CharAnnotation.class);
    String charc3 = c3.getString(CoreAnnotations.CharAnnotation.class);
    String charp = p.getString(CoreAnnotations.CharAnnotation.class);
    String charp2 = p2.getString(CoreAnnotations.CharAnnotation.class);
    String charp3 = p3.getString(CoreAnnotations.CharAnnotation.class);
    Integer cI = c.get(CoreAnnotations.UTypeAnnotation.class);
    String uTypec = (cI != null ? cI.toString() : "");
    Integer c2I = c2.get(CoreAnnotations.UTypeAnnotation.class);
    String uTypec2 = (c2I != null ? c2I.toString() : "");
    Integer c3I = c3.get(CoreAnnotations.UTypeAnnotation.class);
View Full Code Here
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
  }


  protected Collection<String> featuresCpC(PaddedList<? extends CoreLabel> cInfo, int loc) {
    Collection<String> features = new ArrayList<String>();
    CoreLabel c = cInfo.get(loc);
    CoreLabel c2 = cInfo.get(loc + 1);
    CoreLabel c3 = cInfo.get(loc + 2);
    CoreLabel p = cInfo.get(loc - 1);
    CoreLabel p2 = cInfo.get(loc - 2);
    CoreLabel p3 = cInfo.get(loc - 3);

    String charc = c.getString(CoreAnnotations.CharAnnotation.class);
    String charc2 = c2.getString(CoreAnnotations.CharAnnotation.class);
    String charc3 = c3.getString(CoreAnnotations.CharAnnotation.class);
    String charp = p.getString(CoreAnnotations.CharAnnotation.class);
    String charp2 = p2.getString(CoreAnnotations.CharAnnotation.class);
    String charp3 = p3.getString(CoreAnnotations.CharAnnotation.class);

    Integer cI = c.get(CoreAnnotations.UTypeAnnotation.class);
    String uTypec = (cI != null ? cI.toString() : "");
    Integer c2I = c2.get(CoreAnnotations.UTypeAnnotation.class);
    String uTypec2 = (c2I != null ? c2I.toString() : "");
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
   *  @return Collection of String features (sparse set of boolean features
   */
  protected Collection<String> featuresCnC(PaddedList<? extends CoreLabel> cInfo, int loc) {
    Collection<String> features = new ArrayList<String>();
    if (flags.useWordn) {
      CoreLabel c = cInfo.get(loc);
      CoreLabel c2 = cInfo.get(loc + 1);
      CoreLabel p = cInfo.get(loc - 1);
      CoreLabel p2 = cInfo.get(loc - 2);
      String charc = c.getString(CoreAnnotations.CharAnnotation.class);
      String charc2 = c2.getString(CoreAnnotations.CharAnnotation.class);
      String charp = p.getString(CoreAnnotations.CharAnnotation.class);
      String charp2 = p2.getString(CoreAnnotations.CharAnnotation.class);

      features.add(charc +"c");
      features.add(charc2+"c2");
      features.add(charp +"p");
      features.add(charp2 + "p2");
View Full Code Here
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
   *  @param loc Position of c in list
   *  @return Collection of String features (sparse set of boolean features
   */
  protected Collection<String> featuresCpCp2C(PaddedList<? extends CoreLabel> cInfo, int loc) {
    Collection<String> features = new ArrayList<String>();
    CoreLabel c = cInfo.get(loc);
    CoreLabel c2 = cInfo.get(loc + 1);
    CoreLabel c3 = cInfo.get(loc + 2);
    CoreLabel p = cInfo.get(loc - 1);
    CoreLabel p2 = cInfo.get(loc - 2);
    CoreLabel p3 = cInfo.get(loc - 3);

    String charc = c.getString(CoreAnnotations.CharAnnotation.class);
    String charc2 = c2.getString(CoreAnnotations.CharAnnotation.class);
    String charc3 = c3.getString(CoreAnnotations.CharAnnotation.class);
    String charp = p.getString(CoreAnnotations.CharAnnotation.class);
    String charp2 = p2.getString(CoreAnnotations.CharAnnotation.class);
    String charp3 = p3.getString(CoreAnnotations.CharAnnotation.class);

    // N-gram features. N is up to 3
    if (flags.useWord3) {
      features.add(charc +"::c");
      features.add(charc2+"::n");
View Full Code Here
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807

  protected Collection<String> featuresCpCp2Cp3C(PaddedList<? extends CoreLabel> cInfo, int loc) {
    Collection<String> features = new ArrayList<String>();
    if (flags.use4Clique && flags.maxLeft >= 3) {
      CoreLabel c = cInfo.get(loc);
      CoreLabel c2 = cInfo.get(loc + 1);
      CoreLabel p = cInfo.get(loc - 1);
      CoreLabel p2 = cInfo.get(loc - 2);
      CoreLabel p3 = cInfo.get(loc - 3);
      String charc = c.getString(CoreAnnotations.CharAnnotation.class);
      String charp = p.getString(CoreAnnotations.CharAnnotation.class);
      String charp2 = p2.getString(CoreAnnotations.CharAnnotation.class);
      String charp3 = p3.getString(CoreAnnotations.CharAnnotation.class);
      Integer cI = c.get(CoreAnnotations.UTypeAnnotation.class);
      String uTypec = (cI != null ? cI.toString() : "");
      Integer c2I = c2.get(CoreAnnotations.UTypeAnnotation.class);
      String uTypec2 = (c2I != null ? c2I.toString() : "");
      Integer pI = p.get(CoreAnnotations.UTypeAnnotation.class);
      String uTypep = (pI != null ? pI.toString() : "");
      Integer p2I = p2.get(CoreAnnotations.UTypeAnnotation.class);
      String uTypep2 = (p2I != null ? p2I.toString() : "");
      Integer p3I = p3.get(CoreAnnotations.UTypeAnnotation.class);
      String uTypep3 = (p3I != null ? p3I.toString() : "");


      if (flags.useLongSequences) {
        features.add(charp3 + charp2 + charp + charc + "p3p2pc");
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
        //to make sure we discard phrases with stopwords in between, but include the ones in which stop words were removed at the ends if removeStopWordsFromSelectedPhrases is true
        boolean[] addedindices = new boolean[e-s];
        Arrays.fill(addedindices, false);
       
        for (int i = s; i < e; i++) {
          CoreLabel l = sent.get(i);
          l.set(PatternsAnnotations.MatchedPattern.class, true);

          if(!l.containsKey(PatternsAnnotations.MatchedPatterns.class))
            l.set(PatternsAnnotations.MatchedPatterns.class, new HashSet<Pattern>());
          l.get(PatternsAnnotations.MatchedPatterns.class).add(matchedPat);

          // if (restrictToMatched) {
          // tokensMatchedPattern.add(sentid, i);
          // }
          for (Entry<Class, Object> ig : constVars.getIgnoreWordswithClassesDuringSelection().get(label).entrySet()) {
            if (l.containsKey(ig.getKey()) && l.get(ig.getKey()).equals(ig.getValue())) {
              doNotUse = true;
            }
          }
          boolean containsStop = containsStopWord(l, constVars.getCommonEngWords(), PatternFactory.ignoreWordRegex);
          if (removePhrasesWithStopWords && containsStop) {
            doNotUse = true;
          } else {
            if (!containsStop || !removeStopWordsFromSelectedPhrases) {
             
              if (label == null || l.get(constVars.getAnswerClass().get(label)) == null || !l.get(constVars.getAnswerClass().get(label)).equals(label.toString())) {
                useWordNotLabeled = true;
              }
              phrase += " " + l.word();
              phraseLemma += " " + l.lemma();
              addedindices[i-s] = true;
            }
          }
        }
       
View Full Code Here
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
  private static final Pattern numberPattern = Pattern.compile("[0-9]+");
  private static final Pattern ordinalEndPattern = Pattern.compile("(?:st|nd|rd|th)", Pattern.CASE_INSENSITIVE);

  private boolean isOrdinal(List<? extends CoreLabel> wordInfos, int pos) {
    CoreLabel c = wordInfos.get(pos);
    String cWord = getWord(c);
    Matcher m = ordinalPattern.matcher(cWord);
    if (m.matches()) { return true; }
    m = numberPattern.matcher(cWord);
    if (m.matches()) {
      if (pos+1 < wordInfos.size()) {
        CoreLabel n = wordInfos.get(pos+1);
        m = ordinalEndPattern.matcher(getWord(n));
        if (m.matches()) { return true; }
      }
      return false;
    }

    m = ordinalEndPattern.matcher(cWord);
    if (m.matches()) {
      if (pos > 0) {
        CoreLabel p = wordInfos.get(pos-1);
        m = numberPattern.matcher(getWord(p));
        if (m.matches()) { return true; }
      }
    }
    if (cWord.equals("-")) {
      if (pos+1 < wordInfos.size() && pos > 0) {
        CoreLabel p = wordInfos.get(pos-1);
        CoreLabel n = wordInfos.get(pos+1);
        m = ordinalPattern.matcher(getWord(p));
        if (m.matches()) {
          m = ordinalPattern.matcher(getWord(n));
          if (m.matches()) {
            return true;
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
  protected HasWord getNext() {
    while (wordIter == null || ! wordIter.hasNext()) {
      if ( ! tok.hasNext()) {
        return null;
      }
      CoreLabel token = tok.next();
      String s = token.word();
      if (s == null) {
        return null;
      }
      if (s.equals(WhitespaceLexer.NEWLINE)) {
        // if newlines were significant, we should make sure to return
View Full Code Here
TOP

Related Classes of edu.stanford.nlp.ling.CoreLabel$CoreLabelFactory

  • edu.arizona.sista.processors.bionlp.BioNLPTokenizer
  • edu.stanford.nlp.dcoref.Mention
  • edu.stanford.nlp.dcoref.MentionExtractor
  • edu.stanford.nlp.dcoref.RuleBasedCorefMentionFinder
  • edu.stanford.nlp.ie.AbstractSequenceClassifierTest
  • edu.stanford.nlp.ie.machinereading.GenericDataSetReader
  • edu.stanford.nlp.ie.ner.CMMClassifier
  • edu.stanford.nlp.ie.ner.CMMClassifier$Scorer
  • edu.stanford.nlp.ie.NERClassifierCombiner
  • edu.stanford.nlp.ie.NERFeatureFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.