Package com.google.caja.lexer.TokenQueue

Examples of com.google.caja.lexer.TokenQueue.Mark


  }

  /** Parses and returns a Statement. */
  public Statement parseStatement() throws ParseException {
    // look for any labels preceding statement
    Mark m = tq.mark();
    Token<JsTokenType> t = tq.peek();
    if (JsTokenType.WORD == t.type) {
      String label = parseIdentifier(false);
      FilePosition labelPos = t.pos;
      if (tq.checkToken(Punctuation.COLON)) {
View Full Code Here


    return parseProgramOrFunctionBody(false);
  }

  private Block parseProgramOrFunctionBody(boolean requireBrackets)
      throws ParseException {
    Mark m = tq.mark();
    if (requireBrackets) { tq.expectToken(Punctuation.LCURLY); }
    List<Statement> stmts = Lists.newArrayList();
    DirectivePrologue prologue = parseOptionalDirectivePrologue();
    if (prologue != null) { stmts.add(prologue); }
    while (!tq.isEmpty() && !tq.lookaheadToken(Punctuation.RCURLY)) {
View Full Code Here

  private DirectivePrologue parseOptionalDirectivePrologue()
      throws ParseException {
    // Quick return if we are sure we will not accumulate anything
    if (tq.isEmpty() || tq.peek().type != JsTokenType.STRING) { return null; }

    Mark startOfPrologue = tq.mark();
    List<Directive> directives = Lists.newArrayList();

    while (!tq.isEmpty() && tq.peek().type == JsTokenType.STRING) {
      Mark startOfDirective = tq.mark();
      Token<JsTokenType> quotedString = tq.pop();

      if (!tq.checkToken(Punctuation.SEMI)) {
        Token<JsTokenType> t = !tq.isEmpty() ? tq.peek() : null;
        if ((t == null || !continuesExpr(t.text)) && semicolonInserted()) {
View Full Code Here

              s = new ForEachLoop(
                  posFrom(start), label, lvalue, iterable, body);
            }

          } else {
            Mark m = tq.mark();
            tq.expectToken(Punctuation.SEMI);
            Expression condition = parseExpressionOrNoop(
                new BooleanLiteral(posFrom(m), true), true);
            Statement increment;

            if (!tq.checkToken(Punctuation.RPAREN)) {
              increment = parseExpressionStmt(true);
              tq.expectToken(Punctuation.RPAREN);
            } else {
              increment = noop(tq.lastPosition());
            }
            Statement body = parseBody(true);
            s = new ForLoop(
                posFrom(start), label, initializer, condition, increment, body);
          }
        }
        break;
      }
      case WHILE:
      {
        tq.advance();
        tq.expectToken(Punctuation.LPAREN);
        Expression cond = parseExpressionInt(true);
        tq.expectToken(Punctuation.RPAREN);
        Statement body = parseBody(true);
        s = new WhileLoop(posFrom(start), label, cond, body);
        break;
      }
      case DO:
      {
        tq.advance();
        Statement body = parseBody(false);
        tq.expectToken(Keyword.WHILE);
        tq.expectToken(Punctuation.LPAREN);
        Expression cond = parseExpressionInt(true);
        tq.expectToken(Punctuation.RPAREN);
        s = new DoWhileLoop(posFrom(start), label, body, cond);
        // http://code.google.com/p/google-caja/issues/detail?id=1316
        // ES[35] requires ; after do-while, but browsers are ok without it.
        // Here we either eat a ; or warn if it's missing.
        if (!tq.checkToken(Punctuation.SEMI)) {
          FilePosition pos = FilePosition.endOf(tq.lastPosition());
          mq.addMessage(
              MessageType.SEMICOLON_INSERTED, MessageLevel.LINT, pos);
        }
        break;
      }
      case SWITCH:
      {
        tq.advance();
        tq.expectToken(Punctuation.LPAREN);
        Expression switchValue = parseExpressionInt(true);
        tq.expectToken(Punctuation.RPAREN);
        tq.expectToken(Punctuation.LCURLY);
        List<SwitchCase> cases = Lists.newArrayList();
        while (!tq.checkToken(Punctuation.RCURLY)) {
          Mark caseMark = tq.mark();
          Expression caseValue;
          if (tq.checkToken(Keyword.DEFAULT)) {
            caseValue = null;
          } else {
            tq.expectToken(Keyword.CASE);
            caseValue = parseExpressionInt(false);
          }
          tq.expectToken(Punctuation.COLON);
          FilePosition colonPos = tq.lastPosition();
          Mark caseBodyStart = tq.mark();
          List<Statement> caseBodyContents = Lists.newArrayList();
          while (!(tq.lookaheadToken(Keyword.DEFAULT)
                   || tq.lookaheadToken(Keyword.CASE)
                   || tq.lookaheadToken(Punctuation.RCURLY))) {
            caseBodyContents.add(parseTerminatedStatement());
View Full Code Here

    }
    return s;
  }

  private AbstractStatement parseStatementWithoutLabel() throws ParseException {
    Mark m = tq.mark();

    Token<JsTokenType> t = tq.peek();

    if (JsTokenType.KEYWORD == t.type) {
      AbstractStatement s;
      switch (Keyword.fromString(t.text)) {
        case FOR: case DO: case WHILE: case SWITCH:
          s = parseLoopOrSwitch(t.pos, "");
          break;
        case IF:
        {
          tq.advance();
          List<Pair<Expression, Statement>> clauses = Lists.newArrayList();
          Statement elseClause = null;
          boolean sawElse;
          do {
            tq.expectToken(Punctuation.LPAREN);
            Expression cond = parseExpressionInt(true);
            tq.expectToken(Punctuation.RPAREN);
            Statement body = parseBody(false);
            sawElse = tq.checkToken(Keyword.ELSE);
            clauses.add(new Pair<Expression, Statement>(cond, body));
          } while (sawElse && tq.checkToken(Keyword.IF));
          if (sawElse) {
            elseClause = parseBody(true);
          }
          s = new Conditional(posFrom(m), clauses, elseClause);
          break;
        }
        case VAR:
        case CONST:
          return parseDeclarationsOrExpression(false);
        case FUNCTION:
        {
          Mark fs = tq.mark();
          tq.advance();
          if (tq.lookaheadToken(Punctuation.LPAREN)) {
            // If no name, then treat it as an expression
            tq.rewind(fs);
            return parseExpressionStmt(false);
          } else // a function declaration
            Identifier identifier = parseIdentifierNode(false);
            tq.expectToken(Punctuation.LPAREN);
            FormalParamList params = parseFormalParams();
            tq.expectToken(Punctuation.RPAREN);
            Block body = parseFunctionBody();
            FunctionConstructor fc = new FunctionConstructor(
                posFrom(m), identifier, params.params, body);
            finish(fc, m);
            s = new FunctionDeclaration(fc);
            finish(s, m);
          }
          break;
        }
        case RETURN:
        {
          tq.advance();
          AbstractExpression value;
          // Check for semicolon insertion without lookahead since return is a
          // restricted production. See the grammar above and ES3 or ES5 S7.9.1
          if (semicolonInserted() || tq.lookaheadToken(Punctuation.SEMI)) {
            value = null;
          } else {
            value = parseExpressionInt(false);
          }
          s = new ReturnStmt(posFrom(m), value);
          break;
        }
        case BREAK:
        {
          tq.advance();
          String targetLabel = "";
          if (!semicolonInserted() && JsTokenType.WORD == tq.peek().type) {
            targetLabel = parseIdentifier(false);
          }
          s = new BreakStmt(posFrom(m), targetLabel);
          break;
        }
        case CONTINUE:
        {
          tq.advance();
          String targetLabel = "";
          if (!semicolonInserted() && JsTokenType.WORD == tq.peek().type) {
            targetLabel = parseIdentifier(false);
          }
          s = new ContinueStmt(posFrom(m), targetLabel);
          break;
        }
        case DEBUGGER:
        {
          tq.advance();
          s = new DebuggerStmt(posFrom(m));
          break;
        }
        case THROW:
        {
          tq.advance();
          if (semicolonInserted()) {
            throw new ParseException(new Message(
                MessageType.EXPECTED_TOKEN,
                FilePosition.endOf(tq.lastPosition()),
                MessagePart.Factory.valueOf("<expression>"),
                MessagePart.Factory.valueOf("<newline>")));
          }
          Expression ex = parseExpressionInt(false);
          s = new ThrowStmt(posFrom(m), ex);
          break;
        }
        case TRY:
        {
          tq.advance();
          Block body = parseBodyBlock();
          CatchStmt handler;
          FinallyStmt finallyBlock;
          Mark m2 = tq.mark();
          boolean sawFinally = tq.checkToken(Keyword.FINALLY);
          if (sawFinally) {
            handler = null;
          } else {
            tq.expectToken(Keyword.CATCH);
View Full Code Here

    return parseExpressionInt(insertionProtected);
  }

  private AbstractExpression parseExpressionInt(boolean insertionProtected)
      throws ParseException {
    Mark m = tq.mark();
    AbstractExpression e = parseOp(Integer.MAX_VALUE, insertionProtected);
    // Handle comma operator
    while (tq.checkToken(Punctuation.COMMA)) {
      // The comma operator is left-associative so parse expression part in loop
      // instead of recursing
View Full Code Here

    // Handle prefix operations
    {
      Token<JsTokenType> t = tq.peek();
      Operator op = Operator.lookupOperation(t.text, OperatorType.PREFIX);
      if (null != op) {
        Mark m = tq.mark();
        tq.advance();
        int opprec = op.getPrecedence();
        if (opprec < precedence) {
          // The opprec + 1 may look a bit odd but it allows binary operators
          // to associate when they have the same precedence as the prefix
          // op preceding them.  This is the desired behavior:
          // new Foo[4] should parenthesize as new (Foo[4]) as verified by
          // the fact that new Object['toString'] fails to parse in FF.
          // It introduces no problem since there are no right-associative
          // binary operators with precedence 2 or 5.
          left = parseOp(opprec + 1, insertionProtected);
        } else {
          throw new ParseException(
              new Message(MessageType.UNEXPECTED_TOKEN, t.pos,
                          MessagePart.Factory.valueOf(t.text)));
        }
        if (op == Operator.CONSTRUCTOR && tq.checkToken(Punctuation.LPAREN)) {
          List<Expression> operands = Lists.newArrayList();
          operands.add(left);
          if (!tq.checkToken(Punctuation.RPAREN)) {
            do {
              operands.add(parseExpressionPart(true));
            } while (tq.checkToken(Punctuation.COMMA));
            tq.expectToken(Punctuation.RPAREN);
          }
          left = new SpecialOperation(posFrom(m), op, operands);
        } else {
          try {
            left = Operation.create(posFrom(m), op, left);
          } catch (IllegalArgumentException e) {
            throw new ParseException(
                new Message(MessageType.ASSIGN_TO_NON_LVALUE, t.pos,
                    MessagePart.Factory.valueOf(t.text)));
          }
        }
        finish(left, m);
        // Not pulling multiple operators off the stack means that
        // some prefix operator nestings are impossible.  This is intended.
        // This prevents such things as (new (++i)).
        // This only affects the new operator though since it is the only
        // prefix operator with a precedence != 4.
      }
      if (null == left) {
        left = parseExpressionAtom();
      }
    }

    // Parse binary operators, except comma.
    while (!tq.isEmpty()) {
      Token<JsTokenType> t = tq.peek();
      // If it is a binary op then we should consider using it
      Operator op = Operator.lookupOperation(t.text, OperatorType.INFIX);
      if (null == op) {
        op = Operator.lookupOperation(t.text, OperatorType.BRACKET);
        if (null == op) {
          op = Operator.lookupOperation(t.text, OperatorType.TERNARY);
          // Check for semicolon insertion since postfix operators are
          // "restricted productions" according to ES3 or ES5 S7.9.1.
          if (null == op) {
            if (!semicolonInserted()) {
              op = Operator.lookupOperation(t.text, OperatorType.POSTFIX);
            }
            if (null == op) { break; }
          }
        }
      } else if (Operator.COMMA == op) {
        break;
      }
      int opprec = op.getPrecedence();
      if (!(opprec < precedence
            || (opprec == precedence
                && Associativity.RIGHT == op.getAssociativity()))) {
        break;
      }

      if (op.getType() == OperatorType.BRACKET) {
        checkForMissingSemicolon();
      }

      Mark opStart = tq.mark();
      int nMessages = mq.getMessages().size();
      tq.advance()// Consume the operator token

      Expression right;
      try {
        // Recurse to parse operator arguments.
        if (OperatorType.BRACKET == op.getType()) {
          if (Operator.FUNCTION_CALL == op) {
            List<Expression> actuals;
            if (tq.checkToken(op.getClosingSymbol())) {
              actuals = Collections.<Expression>emptyList();
            } else {
              actuals = Lists.newArrayList();
              do {
                actuals.add(parseExpressionPart(true));
              } while (tq.checkToken(Punctuation.COMMA));
              tq.expectToken(op.getClosingSymbol());
            }

            right = new ActualList(actuals);
          } else {
            right = parseExpressionInt(true);
            tq.expectToken(op.getClosingSymbol());
          }
        } else if (OperatorType.POSTFIX == op.getType()) {
          right = null;
        } else if (OperatorType.TERNARY == op.getType()) {
          right = parseExpressionPart(insertionProtected);
        } else if (Operator.MEMBER_ACCESS != op) {
          right = parseOp(opprec, insertionProtected);
        } else {
          // The . operator only accepts a reference on the right.
          // No a.b.4 or a.b.(c.d)
          right = parseReference(true);
        }
      } catch (ParseException ex) {
        // According to
        // http://www.mozilla.org/js/language/js20/rationale/syntax.html
        // semicolon insertion requires that we reconsider the decision to
        // treat op as a binary op if it could be a prefix op.

        // Line-Break Semicolon Insertion
        // If the first through the nth tokens of a JavaScript program form
        // are grammatically valid but the first through the n+1st tokens
        // are not and there is a line break between the nth tokens and the
        // n+1st tokens, then the parser tries to parse the program again
        // after inserting a VirtualSemicolon token between the nth and the
        // n+1st tokens.
        if ((Operator.FUNCTION_CALL == op
             || null != Operator.lookupOperation(
                 op.getOpeningSymbol(), OperatorType.PREFIX))
            && !insertionProtected) {
          Mark m3 = tq.mark();
          tq.rewind(opStart);
          if (semicolonInserted()) {
            List<Message> messages = mq.getMessages();
            if (nMessages < messages.size()) {
              messages.subList(nMessages, messages.size()).clear();
View Full Code Here

  }

  @SuppressWarnings("fallthrough")
  private AbstractExpression parseExpressionAtom() throws ParseException {
    AbstractExpression e;
    Mark m = tq.mark();

    Token<JsTokenType> t = tq.pop();
    typeswitch: switch (t.type) {
      case STRING:
        issueLintWarningsForProblematicEscapes(t, mq);
        e = new StringLiteral(t.pos, t.text);
        break;
      case INTEGER:
        if (integerPartIsOctal(t.text)) {
          mq.addMessage(
              MessageType.OCTAL_LITERAL, MessageLevel.LINT,
              t.pos, MessagePart.Factory.valueOf(t.text));
        }
        e = toIntegerLiteral(t);
        break;
      case FLOAT:
        if (integerPartIsOctal(t.text)) {
          mq.addMessage(
              MessageType.OCTAL_LITERAL, MessageLevel.ERROR,
              t.pos, MessagePart.Factory.valueOf(t.text));
        }
        e = toNumberLiteral(t);
        break;
      case REGEXP:
      {
        e = new RegexpLiteral(t.pos, t.text);
        // Check letters.  Warn on s suffix character as non-FF.
        String modifiers = t.text.substring(t.text.lastIndexOf("/") + 1);
        if (!RegexpLiteral.areRegexpModifiersValid(modifiers)) {
          mq.addMessage(
                  MessageType.UNRECOGNIZED_REGEX_MODIFIERS, t.pos,
                  MessagePart.Factory.valueOf(modifiers));
        }
        break;
      }
      case KEYWORD:
      {
        Keyword k = Keyword.fromString(t.text);
        if (null != k) {
          switch (k) {
            case NULL:
              e = new NullLiteral(t.pos);
              break typeswitch;
            case TRUE:
              e = new BooleanLiteral(t.pos, true);
              break typeswitch;
            case FALSE:
              e = new BooleanLiteral(t.pos, false);
              break typeswitch;
            case FUNCTION:
            {
              Identifier identifier = null;
              if (!tq.isEmpty() && JsTokenType.WORD == tq.peek().type) {
                identifier = parseIdentifierNode(false);
              } else {
                identifier = new Identifier(
                    FilePosition.endOf(tq.lastPosition()), null);
              }
              tq.expectToken(Punctuation.LPAREN);
              FormalParamList params = parseFormalParams();
              tq.expectToken(Punctuation.RPAREN);
              Block body = parseFunctionBody();
              e = new FunctionConstructor(
                  posFrom(m), identifier, params.params, body);
              break typeswitch;
            }
            default:
              break// Will be handled by the word handler below
          }
        }
        // fall through
      }
      case WORD:
      {
        String identifier;
        if (Keyword.THIS.toString().equals(t.text)) {
          // this is allowed, but not t\u0068is as per the grammar
          identifier = Keyword.THIS.toString();
        } else {
          tq.rewind(m);
          identifier = parseIdentifier(false);
        }
        Identifier idNode = new Identifier(posFrom(m), identifier);
        finish(idNode, m);
        e = new Reference(idNode);
        break;
      }
      case PUNCTUATION:
        switch (Punctuation.fromString(t.text)) {
          case LPAREN:
            e = parseExpressionInt(true);
            tq.expectToken(Punctuation.RPAREN);
            return e;  // Don't pull comments outside parens inside
          case LSQUARE:
          {
            List<Expression> elements = Lists.newArrayList();

            if (!tq.checkToken(Punctuation.RSQUARE)) {
              Mark comma = null;
              do {
                // Handle adjacent commas that specify undefined values.
                // E.g. [1,,2]
                for (Mark cm = tq.mark(); tq.checkToken(Punctuation.COMMA);
                     cm = tq.mark()) {
                  comma = cm;
                  Elision vl = new Elision(posFrom(cm));
                  finish(vl, cm);
                  elements.add(vl);
                }
                if (tq.lookaheadToken(Punctuation.RSQUARE)) { break; }
                comma = null;
                elements.add(parseExpressionPart(true));
              } while (tq.checkToken(Punctuation.COMMA));
              if (comma != null) {
                // On IE, [1,] has length 2 unlike on other browsers.
                mq.addMessage(MessageType.NOT_IE, comma.getFilePosition());
              }
              tq.expectToken(Punctuation.RSQUARE);
            }

            e = new ArrayConstructor(posFrom(m), elements);
            break;
          }
          case LCURLY:
          {
            List<ObjProperty> properties = Lists.newArrayList();
            if (!tq.checkToken(Punctuation.RCURLY)) {
              boolean sawComma;
              do {
                Mark km = tq.mark();
                Token<JsTokenType> keyToken = tq.peek();
                String propertyType = null;
                Mark beforeProperty = km;
                if (keyToken.type == JsTokenType.WORD) {
                  if ("get".equals(keyToken.text)
                      || "set".equals(keyToken.text)) {
                    tq.advance();
                    Mark afterWord = tq.mark();
                    if (!tq.checkToken(Punctuation.COLON)) {
                      propertyType = keyToken.text;
                      beforeProperty = afterWord;
                      keyToken = tq.peek();
                    }
                    tq.rewind(beforeProperty);
                  }
                }
                StringLiteral key;
                switch (keyToken.type) {
                  case STRING:
                    tq.advance();
                    key = new StringLiteral(posFrom(km), keyToken.text);
                    break;
                  case FLOAT:
                    tq.advance();
                    key = StringLiteral.valueOf(
                        posFrom(km), floatToString(keyToken));
                    break;
                  case INTEGER:
                    tq.advance();
                    key = StringLiteral.valueOf(
                        posFrom(km), "" + toInteger(keyToken));
                    break;
                  default:
                    String ident = parseIdentifier(true);
                    key = new StringLiteral(posFrom(km), ident);
                    break;
                }
                finish(key, beforeProperty);
                ObjProperty prop;
                if (propertyType == null) {
                  tq.expectToken(Punctuation.COLON);
                  Expression value = parseExpressionPart(true);
                  prop = new ValueProperty(posFrom(km), key, value);
                } else {
                  Mark beforeFormals = tq.mark();
                  Identifier ident = new Identifier(tq.currentPosition(), null);
                  tq.expectToken(Punctuation.LPAREN);
                  FormalParamList params = parseFormalParams();
                  tq.expectToken(Punctuation.RPAREN);
                  Block body = parseFunctionBody();
                  FunctionConstructor fn = new FunctionConstructor(
                      posFrom(beforeFormals), ident, params.params, body);
                  if ("get".equals(propertyType)) {
                    prop = new GetterProperty(posFrom(km), key, fn);
                  } else {
                    assert "set".equals(propertyType);
                    prop = new SetterProperty(posFrom(km), key, fn);
                  }
                }
                finish(prop, km);
                properties.add(prop);
                Mark cm = tq.mark();
                sawComma = tq.checkToken(Punctuation.COMMA);
                if (sawComma && tq.lookaheadToken(Punctuation.RCURLY)) {
                  tq.rewind(cm);
                  mq.addMessage(MessageType.NOT_IE, tq.currentPosition());
                  tq.advance();
View Full Code Here

    return e;
  }

  private Reference parseReference(boolean allowReservedWords)
      throws ParseException {
    Mark m = tq.mark();
    Identifier idNode = parseIdentifierNode(allowReservedWords);
    Reference r = new Reference(idNode);
    finish(r, m);
    return r;
  }
View Full Code Here

    return r;
  }

  private Identifier parseIdentifierNode(boolean allowReservedWords)
      throws ParseException {
    Mark m = tq.mark();
    String identifierName = parseIdentifier(allowReservedWords);
    Identifier ident = new Identifier(posFrom(m), identifierName);
    finish(ident, m);
    return ident;
  }
View Full Code Here

TOP

Related Classes of com.google.caja.lexer.TokenQueue.Mark

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.