Examples of com.antlersoft.parser.Token

com.antlersoft.parser.Token
A token is a lexically-recognized atomic element of a statement in some language. A token combines a Symbol which represents the tokens semantic role in the language with a value which represents how the token appeared in the statement in string form. @author Michael A. MacDonald

    if ( size>0)
    {
      Member o=m_contents.get( size-1);
      if ( o instanceof TokenHolder)
      {
        Token t=((TokenHolder)o).m_token;
        if ( t.symbol==Parser._end_ || t.symbol==scope.findReserved(";"))
          m_contents.remove( size-1);
      }
    }
  }

View Full Code Here

    for ( Iterator<Member> it=m_contents.iterator(); it.hasNext();)
    {
      Member o=it.next();
      if ( o instanceof Replacement)
      {
        tokens.add( new Token( scope.getReserved(  "("), "("));
        ((Replacement)o).getReplacer().collectTokens( scope, tokens);
        tokens.add( new Token( scope.getReserved( ")"), ")"));
      }
      else
        tokens.add(((TokenHolder)o).m_token);
    }
  }

View Full Code Here


    /* (non-Javadoc)
     * @see com.antlersoft.query.environment.TokenSequence.Member#collectTokens(com.antlersoft.parser.ReservedScope, java.util.Collection)
     */
    public void collectTokens(ReservedScope scope, Collection<Token> tokens) {
      tokens.add( new Token( scope.getReserved(  "("), "("));
      getReplacer().collectTokens( scope, tokens);
      tokens.add( new Token( scope.getReserved( ")"), ")"));
    }

View Full Code Here

        m_stack.startWithHandler( new TokenSequenceElement( m_replacer).readFromXML( m_stack),
            uri, localName, qName, attributes);
      }
      else
      {
        Token replaced_token=ElementFactory.getInstance().getTokenForTag(qName);
        if (replaced_token != null)
        {
          m_replaced.add( replaced_token);
          m_stack.startWithHandler( ElementFactory.getInstance().getElementForToken( replaced_token).readFromXML( m_stack),
              uri, localName, qName, attributes);

View Full Code Here

      m_sequence_stack.clear();
      m_sequence_stack.add( new TokenSequence());
      boolean errorOut=false;
      for (; ! errorOut && currentIndex<tokens.length; currentIndex++)
      {
        Token token=tokens[currentIndex];
        if (token.symbol == SelectionToken.SELECTION_SYMBOL)
        {
          token = new SelectionToken(
              getSelection() != null && getSelection().getResultClass() != null ?
                  getSelection().getResultClass().getName()

View Full Code Here

    // A saved transform converts, not to itself, but to an empty set expression,
    // represented by a single "list" token here
    if ( value instanceof Transform)
    {
      seq=new TokenSequence();
      seq.addToken( new Token(m_parser.getReservedScope().getReserved( "list"), "list"));
    }
    ((TokenSequence)m_sequence_stack.peek()).replaceTokensBySequence( saved.m_token_count, seq);
  }

View Full Code Here

    String class_name=(String)to_restore.getResult();
    boolean is_transform=false;
    if ( Transform.class.isAssignableFrom( Class.forName( class_name)))
    {
      is_transform=true;
      restore_tokens.add( new Token( m_parser.getReservedScope().getReserved( "set"), "set"));
      restore_tokens.add( new Token( BasicBase.nameSymbol, name));
      restore_tokens.add( new Token( m_parser.getReservedScope().getReserved( "to"), "to"));
    }
    to_restore.collectTokens( m_parser.getReservedScope(), restore_tokens);
    for (Token t : restore_tokens)
    {
      if (t instanceof SelectionToken)
      {
        SelectionToken slt = (SelectionToken)t;
        if (slt.m_className != null)
        {
          Class<?> slc = Class.forName(slt.m_className);
          if (m_selection.getResultClass() == null || ! m_selection.getResultClass().isAssignableFrom(slc))
          {
            m_selection = new SelectionSetExpression(slc);
          }
        }
      }
    }
      restore_tokens.add( new Token( Parser._end_, ""));
    m_parser.reset();
    tokens=restore_tokens.toArray( new Token[restore_tokens.size()]);
    SetExpression exp=getExpression();
    if ( is_transform)
    {

View Full Code Here

          String cs=currentString.toString();
          currentString.setLength(0);
          Symbol rw=m_parser.getReservedScope().findReserved( cs);
          if ( rw==null)
          {
              adder.addToken( new Token( BasicBase.nameSymbol, cs));
          }
          else
          {
              adder.addToken( new Token( rw, cs));
          }
      }    
  }

View Full Code Here

        break;
      case IN_FIRST_SLASH:
        addPunctuationCharacter( '\\');
        break;
      case IN_NUMBER :
        adder.addToken( new Token( BasicBase.number, currentString.toString()));
        break;
      case IN_WORD :
      case IN_PUNC :
        addCurrentString();
        break;
      // INITIAL state, do nothing
      }
      adder.addToken( new Token( Parser._end_, ""));
      return this;
  }

View Full Code Here

          {
            currentString.append( c);
          }
          else
          {
            adder.addToken( new Token( BasicBase.number, currentString.toString()));
            currentString.setLength(0);
            lex_state=INITIAL;
            return nextCharacter(c);
          }
          break;

View Full Code Here

0 1

TOP

Related Classes of com.antlersoft.parser.Token

com.antlersoft.query.environment.ElementFactory

com.antlersoft.query.environment.Lexer

com.antlersoft.query.environment.QueryLanguageEnvironment

com.antlersoft.query.environment.TokenSequence

com.antlersoft.query.environment.TokenSequence$Replacement

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.