Package org.auraframework.impl.util

Examples of org.auraframework.impl.util.TextTokenizer$Token


      }

      StringMultiset tagAttributes = tag.getAttributes();

      while (index < size) {
        Token token = tokens.get(index);
        index++;
        TokenType tokenType = token.getType();
        if (ATTRIBUTE == tokenType) {
          String attribute = token.getValue();
          attribute = ignoreCase ? attribute.toLowerCase() : attribute;
          attributes.add(attribute);
          tagAttributes.add(attribute);
        } else if (TAG == tokenType) {
          // Tag closing token
          tag.setDirty(false);
          inTag = false;
          break;
        }
      }
      if (newAttributes && attributes.size() != 0) {
        line.putTag(TAG_ATTRIBUTES, attributes);
      } else if (!newAttributes && attributes.size() == 0) {
        line.putTag(TAG_ATTRIBUTES, null);
      }
    } else {
      line.putTag(TAG_ATTRIBUTES, null);
    }

    while (index < size) {
      Token token = tokens.get(index);
      index++;
      TokenType tokenType = token.getType();
      if (TAG == tokenType) {
        if (inTag) {
          if (">".equals(token.getValue()) || "/>".equals(token.getValue())) {
            // If type is "tag" and content is ">", this is HTML token.
            inTag = false;
          }
        } else {
          // Check that we are in html mode.
          if (CodeMirror2.HTML.equals(token.getMode())) {
            lastTagTokenIndex = index - 1;
            inTag = true;
          }
        }
      }
    }

    if (inTag) {
      if (lastTagTokenIndex != -1) {
        index = lastTagTokenIndex;
        Token token = tokens.get(index);
        index++;
        String tagName = token.getValue().substring(1).trim();
        tag = new HtmlTagWithAttributes(tagName);
        StringMultiset tagAttributes = tag.getAttributes();
        while (index < size) {
          token = tokens.get(index);
          index++;
          TokenType tokenType = token.getType();
          if (ATTRIBUTE == tokenType) {
            String attribute = token.getValue();
            tagAttributes.add(ignoreCase ? attribute.toLowerCase() : attribute);
          }
        }
      }
View Full Code Here


   */
  static boolean buildInvocationSequenceContext(
      JsonArray<Token> tokens, boolean expectingPeriod, JsonArray<String> contextParts) {
    // right-to-left tokens processing loop.
    while (!tokens.isEmpty()) {
      Token lastToken = tokens.pop();
      TokenType lastTokenType = lastToken.getType();
      String lastTokenValue = lastToken.getValue();

      // Omit whitespaces.
      if (lastTokenType == WHITESPACE) {
        continue;
      }
View Full Code Here

    if (result == null) {
      return new ExtendedParseResult<T>(null, Context.NOT_PARSED);
    }

    JsonArray<Token> tokens = result.getTokens();
    Token lastToken = tokens.peek();
    Preconditions.checkNotNull(lastToken,
        "Last token expected to be non-null; text='%s', position=%s", text, position);
    TokenType lastTokenType = lastToken.getType();
    String lastTokenValue = lastToken.getValue();
    if (!addSpace) {
      if (lastTokenType == STRING || lastTokenType == REGEXP) {
        return new ExtendedParseResult<T>(result, Context.IN_STRING);
      } else if (lastTokenType == TokenType.COMMENT) {
        return new ExtendedParseResult<T>(result, Context.IN_COMMENT);
      }

      // Python parser, for a purpose of simplicity, parses period and variable
      // name as a single token. If period is not followed by identifier, parser
      // states that this is and error, which is, generally, not truth.
      if ((lastTokenType == TokenType.ERROR) && LITERAL_PERIOD.equals(lastTokenValue)) {
        tokens.pop();
        tokens.add(new Token(lastToken.getMode(), TokenType.NULL, LITERAL_PERIOD));
      }

      return new ExtendedParseResult<T>(result, Context.IN_CODE);
    }

    // Remove / shorten last token to omit added whitespace.
    tokens.pop();
    if (lastTokenType == STRING || lastTokenType == REGEXP || lastTokenType == TokenType.COMMENT) {
      // Whitespace stuck to token - strip it.
      lastTokenValue = lastTokenValue.substring(0, lastTokenValue.length() - 1);
      tokens.add(new Token(lastToken.getMode(), lastTokenType, lastTokenValue));
      if (lastTokenType == STRING || lastTokenType == REGEXP) {
        return new ExtendedParseResult<T>(result, Context.IN_STRING);
      } else {
        return new ExtendedParseResult<T>(result, Context.IN_COMMENT);
      }
View Full Code Here

      }
      JsonArray<Token> tokens = parseResult.getTokens();
      if (tokens.isEmpty()) {
        return null;
      }
      Token lastToken = tokens.peek();
      return lastToken.getValue();
    }
View Full Code Here

    int indent = 0;
    if (TokenType.WHITESPACE == tokens.get(0).getType()) {
      indent = tokens.get(0).getValue().length();
    }

    Token lastToken = tokens.pop();
    TokenType lastTokenType = lastToken.getType();

    if (lastTokenType == WHITESPACE) {
      return new CompletionContext<T>("", "", false, CompletionType.GLOBAL, parseResult, indent);
    }

    String lastTokenValue = lastToken.getValue();

    if (lastTokenType == KEYWORD) {
      return new CompletionContext<T>(
          "", lastTokenValue, false, CompletionType.GLOBAL, parseResult, indent);
    }
View Full Code Here

    }

    private Token buildTokenModel(TokenInfo token) throws CloudException {
        Preconditions.checkNotNull(token);

        Token response = new Token();

        response.expires = TokenAuth.getExpiration(token);

        UserData user = authRepository.getUsers().find(token.getUserId());
        if (user == null) {
View Full Code Here

    return new JAVACCParser(reader, new SPARQLParser(reader));
  }

  @Override
  public TOKEN getNextToken() {
    final Token token = this.parser.getNextToken();
    if(token==null){
      return null;
    } else {
      return new SPARQLToken(token);
    }
View Full Code Here

    int type = adaptor.getType(t);
    buf.append("\t");
    buf.append(ID);
    buf.append("\t");
    buf.append(type);
    Token token = adaptor.getToken(t);
    int line = -1;
    int pos = -1;
    if ( token!=null ) {
      line = token.getLine();
      pos = token.getCharPositionInLine();
    }
    buf.append("\t");
    buf.append(line);
    buf.append("\t");
    buf.append(pos);
View Full Code Here

  /** Get num hidden tokens between i..j inclusive */
  public int getNumberOfHiddenTokens(int i, int j) {
    int n = 0;
    TokenStream input = parser.getTokenStream();
    for (int ti = i; ti<input.size() && ti <= j; ti++) {
      Token t = input.get(ti);
      if ( t.getChannel()!=Token.DEFAULT_CHANNEL ) {
        n++;
      }
    }
    return n;
  }
View Full Code Here

      }
      previousTokenIndex = t.getTokenIndex();
      listener.consumeHiddenToken(t);
    }
    else if ( elements[0].equals("LT") ) {
      Token t = deserializeToken(elements, 2);
      listener.LT(Integer.parseInt(elements[1]), t);
    }
    else if ( elements[0].equals("mark") ) {
      listener.mark(Integer.parseInt(elements[1]));
    }
View Full Code Here

TOP

Related Classes of org.auraframework.impl.util.TextTokenizer$Token

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.