Examples of BaseToken

edu.mayo.bmi.uima.core.type.syntax.BaseToken
info.bliki.htmlcleaner.BaseToken

Base token interface. Tokens are individual entities recognized by HTML parser.
Created by: Vladimir Nikic
Date: November, 2006.
org.apache.ctakes.core.fsm.token.BaseToken
org.apache.ctakes.typesystem.type.syntax.BaseToken
org.apache.ws.sandbox.security.trust.message.token.BaseToken
@author Malinda Kaushalye
Base token.

Examples of edu.mayo.bmi.uima.core.type.syntax.BaseToken

//        if (baseTokenAnnotationUntyped.getBegin() < sentenceBegin ||
//            baseTokenAnnotationUntyped.getEnd() > sentenceEnd)
//        {
//          continue;
//        }
        BaseToken baseToken = (BaseToken)baseTokenAnnotationUntyped;
        if (!(baseToken instanceof NewlineToken))
        {
          String currentTokenText = baseToken.getCoveredText();
          arrayOfTokens.add(currentTokenText);
        }
      }
      arrayOfLines.add(arrayOfTokens);

View Full Code Here

Examples of info.bliki.htmlcleaner.BaseToken

  }


  @Override
  public void content(AbstractHTMLToWiki w, TagNode node, StringBuilder resultBuffer, boolean showWithoutTag) {
    List children = node.getChildren();
    BaseToken tok = getFirstContent(children, "img");
    if (tok != null) {
      if (tok instanceof TagNode) {
        w.nodeToWiki(tok, resultBuffer);
      } else {
        resultBuffer.append(openStr);

View Full Code Here

Examples of info.bliki.htmlcleaner.BaseToken


  @Override
  public void content(AbstractHTMLToWiki w, TagNode node, StringBuilder resultBuffer, boolean showWithoutTag) {
    List<Object> children = node.getChildren();
    StringBuilder buf = new StringBuilder();
    BaseToken tok = getFirstContent(children, "img");
    if (tok != null && tok instanceof TagNode) {
      w.nodeToWiki(tok, resultBuffer);
    } else {
      resultBuffer.append(openStr);
      // no wiki tags inside wiki links:

View Full Code Here

Examples of info.bliki.htmlcleaner.BaseToken

    }


    @Override
    public void content(AbstractHTMLToWiki w, TagNode node, StringBuilder resultBuffer, boolean showWithoutTag) {
        List<Object> children = node.getChildren();
        BaseToken tok = getFirstContent(children, "img");
        if (tok != null && tok instanceof TagNode) {
            w.nodeToWiki(tok, resultBuffer);
        } else {
            String hrefStr = node.getAttributes().get("href");
            if (hrefStr != null && hrefStr.trim().length() > 0) {

View Full Code Here

Examples of info.bliki.htmlcleaner.BaseToken

  }


  @Override
  public void content(AbstractHTMLToWiki w, TagNode node, StringBuilder resultBuffer, boolean showWithoutTag) {
    List<Object> children = node.getChildren();
    BaseToken tok = getFirstContent(children, "img");
    if (tok != null && tok instanceof TagNode) {
      w.nodeToWiki(tok, resultBuffer);
    } else {
      String hrefStr = node.getAttributes().get("href");
      if (hrefStr != null && hrefStr.trim().length() > 0) {

View Full Code Here

Examples of org.apache.ctakes.core.fsm.token.BaseToken

        // adapt JCas objects into objects expected by the Finite state
        // machines
        List<BaseToken> baseTokenList = new ArrayList<>();
        for(org.apache.ctakes.typesystem.type.syntax.BaseToken bta : tokens){
          // ignore newlines, avoid null tokens
          BaseToken bt = adaptToBaseToken(bta);
          if(bt != null && !(bt instanceof EolToken))
            baseTokenList.add(bt);
        }


        // execute FSM logic

View Full Code Here

Examples of org.apache.ctakes.core.fsm.token.BaseToken

    // maps a fsm to a token start index
    // key = fsm , value = token start index
    Map<Machine, Integer> tokenStartMap = new HashMap<Machine, Integer>();


    for (int i = 0; i < tokens.size(); i++) {
      BaseToken token = tokens.get(i);


      Iterator<Machine> machineItr = iv_machineSet.iterator();
      while (machineItr.hasNext()) {
        Machine fsm = machineItr.next();


        fsm.input(token);


        State currentState = fsm.getCurrentState();
        if (currentState.getStartStateFlag()) {
          tokenStartMap.put(fsm, new Integer(i));
        }
        if (currentState.getEndStateFlag()) {
          Object o = tokenStartMap.get(fsm);
          int tokenStartIndex;
          if (o == null) {
            // By default, all machines start with
            // token zero.
            tokenStartIndex = 0;
          } else {
            tokenStartIndex = ((Integer) o).intValue();
            // skip ahead over single token we don't want
            tokenStartIndex++;
          }
          BaseToken endToken = null;
          if (currentState instanceof NonTerminalEndState) {
            endToken = tokens.get(i - 1);
          } else {
            endToken = token;
          }


          BaseToken startToken = tokens
              .get(tokenStartIndex);
          DateToken dateToken = new DateToken(startToken
              .getStartOffset(), endToken.getEndOffset());
          dateSet.add(dateToken);
          fsm.reset();
        }
      }

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken

        }
      }


      // Initialize Token / Sentence info for the ClearParser Semantic Role Labeler
      for (int i = 0; i < tokens.size(); i++) {
        BaseToken token = tokens.get(i);


        // Determine HeadId
        DepNode node = new DepNode();
        ConllDependencyNode casDepNode = JCasUtil.selectCovered(jCas, ConllDependencyNode.class, token).get(0);
        casDepNode.getDeprel();
        String headRelation = casDepNode.getDeprel();
        ConllDependencyNode head = casDepNode.getHead();
        
        // If there is no head, this is the head node, set node to 0
        int headId = (head == null) ? 0 : depNodeToID.get(head);


        // Populate Dependency Node / Tree information
        node.id = i + 1;
        node.form = token.getCoveredText();
        node.pos = token.getPartOfSpeech();
        node.lemma = useLemmatizer ? "" : token.getNormalizedForm();
        node.setHead(headId, headRelation, 0);
        tree.add(node);
      }
      tree.setPredicates(AbstractReader.LANG_EN);

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken


    // Start at node 1, since node 0 is considered the head of the sentence
    for (int i = 1; i < tree.size(); i++) {
      // Every ClearParser parserNode will contain an srlInfo field.
      DepNode parserNode = tree.get(i);
      BaseToken token = tokens.get(i - 1);
      if (parserNode.srlInfo == null) {
        continue;
      }


      if (parserNode.srlInfo.isPredicate()) {
        int headId = i;
        if (!headIdToPredicate.containsKey(headId)) {
          // We have not encountered this predicate yet, so create it
          Predicate pred = this.createPredicate(jCas, parserNode.srlInfo.rolesetId, token);
          headIdToPredicate.put(headId, pred);
          pred.setRelations(new EmptyFSList(jCas));
        }
      } else {
        for (SRLHead head : parserNode.srlInfo.heads) {
          Predicate predicate;


          // Determine which predicate this argument belongs to
          if (!headIdToPredicate.containsKey(head.headId)) {
            // The predicate hasn't been encountered, so create it
            BaseToken headToken = tokens.get(head.headId - 1);
            predicate = this.createPredicate(jCas, parserNode.srlInfo.rolesetId, headToken);
            headIdToPredicate.put(head.headId, predicate);
          } else {
            predicate = headIdToPredicate.get(head.headId);
          }

View Full Code Here

Examples of org.apache.ctakes.typesystem.type.syntax.BaseToken


            
            
            ListIterator<BaseToken>           itt = tokens.listIterator();
            ListIterator<ConllDependencyNode> itn = nodes.listIterator();
            BaseToken           bt = null;
            ConllDependencyNode dn = null;
            if (tokens.size()>0 && nodes.size()>0) {
                // iterate through the parallel sorted lists
                if (itt.hasNext()) bt                  = itt.next();
                if (itn.hasNext()) dn                  = itn.next();
                if (dn != null) 
                    if (dn.getId()==0 && itn.hasNext()) 
                        dn = itn.next();
                while (itt.hasNext() || itn.hasNext()) {
                    if (bt.getBegin()==dn.getBegin() ) { // Allow ragged right edge //&& bt.getEnd()==dn.getEnd()) { 
                        dn.setLemma( bt.getNormalizedForm() );
                        if (dn.getLemma()==null)
                            dn.setLemma( dn.getForm().toLowerCase() );
                        dn.addToIndexes();
                        if (itt.hasNext()) bt = itt.next();
                        if (itn.hasNext()) dn = itn.next();
                    } else if ( bt.getBegin()<dn.getBegin() ) {
                        if (itt.hasNext()) bt = itt.next(); else break;
                    } else if ( bt.getBegin()>dn.getBegin() ) {
                        // not every node will get a lemma b/c not all tokens are word tokens
                        if (itn.hasNext()) {
                            dn.setLemma( dn.getForm().toLowerCase() );
                            dn.addToIndexes();
                            dn = itn.next(); 
                        } else break;
                    }
                }
                if (bt.getBegin()==dn.getBegin() && bt.getEnd()==dn.getEnd()) {
                    dn.setLemma( bt.getNormalizedForm() );
                    dn.addToIndexes();
                }
            }
            
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.