Examples of BaseToken

edu.mayo.bmi.uima.core.type.syntax.BaseToken
info.bliki.htmlcleaner.BaseToken

Base token interface. Tokens are individual entities recognized by HTML parser.
Created by: Vladimir Nikic
Date: November, 2006.
org.apache.ctakes.core.fsm.token.BaseToken
org.apache.ctakes.typesystem.type.syntax.BaseToken
org.apache.ws.sandbox.security.trust.message.token.BaseToken
@author Malinda Kaushalye
Base token.

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

          String previousOutcome = index < 0 ? "O" : outcomes.get(index);
          features.add(new Feature("PreviousOutcome_" + i, previousOutcome));
        }
        // features from dominating parse tree
//        for(SimpleFeatureExtractor extractor : this.parseFeatureExtractors){
        BaseToken startToken = token;
        for(int i = tokenIndex-1; i >= 0; --i){
          String outcome = outcomes.get(i);
          if(outcome.equals("O")){
            break;
          }
          startToken = tokens.get(i);
        }
        features.addAll(parseExtractor.extract(jCas, startToken.getBegin(), token.getEnd()));
//        }
        // if training, write to data file
        if (this.isTraining()) {
          String outcome = outcomes.get(tokenIndex);
          this.dataWriter.write(new Instance<String>(outcome, features));

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

      if (words.length > 0) {
        String[] wordTagList = tagger.tag(words);


        try {
          for (int i = 0; i < tokens.size(); i++) {
            BaseToken token = tokens.get(i);
            String posTag = wordTagList[i];
            token.setPartOfSpeech(posTag);
          }
        } catch (IndexOutOfBoundsException e) {
          throw new AnalysisEngineProcessException(
              "sentence being tagged is: '"
                  + sentence.getCoveredText() + "'", null, e);

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

        newGoldSentence.addToIndexes();
      }


      for (BaseToken oldSystemToken : JCasUtil.select(jCas, BaseToken.class))
      {
        BaseToken newGoldToken = null; //new BaseToken(goldView, oldSystemEventMention.getBegin(), oldSystemEventMention.getEnd());


        // TODO the following commented out block is an alternative to having the hard coded if..then..else-if..else block for constructing new BaseToken objects
//        Constructor<? extends BaseToken> constructor = null;
//        try
//        {
//          constructor = oldSystemToken.getClass().getConstructor(JCas.class, int.class, int.class);
//        } catch(NoSuchMethodException| SecurityException e)
//        {
//          logger.error("problem getting constructor for copying BaseToken instance (inside AssertionEvalBasedOnModifier.ReferenceSupportingAnnotationsSystemToGoldcopier.process())");
//          continue;
//        }
//        try
//        {
//          newGoldToken = constructor.newInstance(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
//        } catch (InstantiationException | IllegalAccessException
//            | IllegalArgumentException | InvocationTargetException e)
//        {
//          logger.error("problem invoking constructor to copy BaseToken instance (inside AssertionEvalBasedOnModifier.ReferenceSupportingAnnotationsSystemToGoldcopier.process())");
//          continue;
//        }
        
        String oldSystemTokenClass = oldSystemToken.getClass().getName();
        if (oldSystemTokenClass.equals(WordToken.class.getName()))
        {
          newGoldToken = new WordToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(ContractionToken.class.getName()))
        {
          newGoldToken = new ContractionToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(NewlineToken.class.getName()))
        {
          newGoldToken = new NewlineToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(NumToken.class.getName()))
        {
          newGoldToken = new NumToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(PunctuationToken.class.getName()))
        {
          newGoldToken = new PunctuationToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(SymbolToken.class.getName()))
        {
          newGoldToken = new SymbolToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else if (oldSystemTokenClass.equals(BaseToken.class.getName()))
        {
          newGoldToken = new BaseToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        } else
        {
          newGoldToken = new BaseToken(goldView, oldSystemToken.getBegin(), oldSystemToken.getEnd());
        }
        
        newGoldToken.setPartOfSpeech(oldSystemToken.getPartOfSpeech());
        newGoldToken.setTokenNumber(oldSystemToken.getTokenNumber());
        
        newGoldToken.addToIndexes();
      }


    } // end of method ReferenceSupportingAnnotationsSystemToGoldCopier.process()

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

  public static ArrayList<String> getEntityFeats(JCas jcas, IdentifiedAnnotation ent, Sentence sent){
    ArrayList<String> feats = new ArrayList<String>();


    List<BaseToken> words = JCasUtil.selectCovered(jcas, BaseToken.class, ent);
    if(words != null && words.size() > 0){
      BaseToken lastWord = words.get(words.size()-1);
      List<SemanticArgument> args = JCasUtil.selectCovered(jcas, SemanticArgument.class, lastWord);
      Predicate pred = null;


      if(args.size() > 0){
        SemanticArgument entArg = args.get(0);

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

      List<BaseToken> tokens = JCasUtil.selectCovered(jCas, BaseToken.class, sentence);
      DEPTree tree = new DEPTree();


      // Convert CAS data into structures usable by ClearNLP
      for (int i = 0; i < tokens.size(); i++) {
        BaseToken token = tokens.get(i);
        String lemma = useLemmatizer ? lemmatizer.getLemma(token.getCoveredText(), token.getPartOfSpeech()) : token.getNormalizedForm();
        DEPNode node = new DEPNode(i+1, token.getCoveredText(), lemma, token.getPartOfSpeech(), new DEPFeat());
        tree.add(node);
      }


      // Run parser and convert output back to CAS friendly data types
      parser.process(tree);

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

      // Initialize Token / Sentence info for the ClearNLP Semantic Role Labeler
      // we are filtering out newline tokens
      // use idIter as the non-newline token index counter 
      int idIter = 0;
      for (int i = 0; i < tokens.size(); i++) {
        BaseToken token = tokens.get(i);
        // ignore newline tokens within a sentence - newline = whitespace = non-token
        if(!(token instanceof NewlineToken)) {
          // Determine HeadId
          List<ConllDependencyNode> casDepNodes = JCasUtil.selectCovered(jCas, ConllDependencyNode.class, token);

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

    Map<Predicate, List<SemanticArgument>> predicateArguments = new HashMap<Predicate, List<SemanticArgument>>();


    for (int i = 1; i < tree.size(); i++) {
        // Every ClearNLP parserNode will contain an srlInfo field.
        DEPNode parserNode = tree.get(i);
        BaseToken token = tokens.get(i - 1);
        String rolesetId;
        
        if ((rolesetId = parserNode.getFeat(DEPLib.FEAT_PB)) != null)
        {
          if (!headIdToPredicate.containsKey(parserNode)) {
            // We have not encountered this predicate yet, so create it
            Predicate pred = this.createPredicate(jCas, rolesetId, token);
            headIdToPredicate.put(parserNode, pred);
            pred.setRelations(new EmptyFSList(jCas));
          }
        }
    }
    
    
    // Start at node 1, since node 0 is considered the head of the sentence
    for (int i = 1; i < tree.size(); i++) {
      // Every ClearNLP parserNode will contain an srlInfo field.
      DEPNode parserNode = tree.get(i);
      BaseToken token = tokens.get(i - 1);
     
      for (DEPArc head : parserNode.getSHeads()) {
          Predicate predicate = headIdToPredicate.get(head.getNode());


          // Append this argument to the predicate's list of arguments

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

    tokenBeginEndTreeSet = new TreeSet<Integer>();
    
    AnnotationIndex<Annotation> annotationIndex = jcas.getAnnotationIndex(BaseToken.type);
    for (Annotation current : annotationIndex)
    {
      BaseToken bt = (BaseToken)current;
      // filter out NewlineToken
      if (!(bt instanceof NewlineToken)) {
        int begin = bt.getBegin();
        int end = bt.getEnd();
          tokenBeginEndTreeSet.add(begin);
          tokenBeginEndTreeSet.add(end);
      }
    }
  }

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

      sentenceToTokenNumberMap = new HashMap<Sentence, List<BaseToken>>();
      for (Sentence s : beginTreeMap.values()) {
              FSIterator<Annotation> tokensInSentenceIterator = jcas
                              .getAnnotationIndex(BaseToken.type).subiterator(s);
              List<BaseToken> btList = new ArrayList<BaseToken>();
              BaseToken bt = null;
              while ((bt = this.getNextNonEOLToken(tokensInSentenceIterator)) != null) {
                      btList.add(bt);
              }
              sentenceToTokenNumberMap.put(s, btList);
      }

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

    
    
    FSIterator<Annotation> beginTokenInSentenceIterator = constraintConstructorFindContainedBy
            .createFilteredIterator(characterOffset, characterOffset,
                            baseTokenType);
  BaseToken beginToken = this
              .getNextNonEOLToken(beginTokenInSentenceIterator);
  int beginTokenWordNumber = this.sentenceToTokenNumberMap.get(sentence)
              .indexOf(beginToken);
  LineAndTokenPosition b = new LineAndTokenPosition();
  b.setLine(lineNumber);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.