Package com.sonar.sslr.api

Examples of com.sonar.sslr.api.Token


  }

  @Override
  public boolean consume(CodeReader code, Lexer output) {
    if (code.peek() == '/') {
      Token lastToken = getLastToken(output);
      if (lastToken == null || guessNextIsRegexp(lastToken.getValue())) {
        return delegate.consume(code, output);
      }
    }
    return false;
  }
View Full Code Here


import java.util.List;

public class JoinStringsPreprocessor extends Preprocessor {
  @Override
  public PreprocessorAction process(List<Token> tokens) {
    Token token = tokens.get(0);

    if (token.getType() == CxxTokenType.STRING) {

      // Joining string literals (C++ Standard, "2.2 Phases of translation, Phase 6")

      int numberOfStrings = 1;
      StringBuilder sb = new StringBuilder();

      for (;;) {
        Token nextToken = tokens.get(numberOfStrings);
        if (nextToken.getType() != CxxTokenType.STRING) {
          break;
        }
        sb.append(stripQuotes(nextToken.getValue()));
        numberOfStrings++;
      }

      if (numberOfStrings > 1) {
        List<Token> tokensToInject = Lists.newArrayList();
View Full Code Here

        }
      }

      // set standard macros
      for (Map.Entry<String, String> entry : StandardDefinitions.macros().entrySet()) {
        Token bodyToken;
        try {
          bodyToken = Token.builder()
            .setLine(1)
            .setColumn(0)
            .setURI(new java.net.URI(""))
View Full Code Here

    return missingIncludeFiles.get(file.getPath());
  }

  @Override
  public PreprocessorAction process(List<Token> tokens) {
    Token token = tokens.get(0);
    TokenType ttype = token.getType();
    File file = getFileUnderAnalysis();
    String filePath = file == null ? token.getURI().toString() : file.getAbsolutePath();

    if (ttype == PREPROCESSOR) {

      AstNode lineAst = null;
      try {
        lineAst = pplineParser.parse(token.getValue()).getFirstChild();
      } catch (com.sonar.sslr.api.RecognitionException re) {
        LOG.warn("Cannot parse '{}', ignoring...", token.getValue());
        return new PreprocessorAction(1, Lists.newArrayList(Trivia.createSkippedText(token)), new ArrayList<Token>());
      }

      String lineKind = lineAst.getName();
View Full Code Here

    Macro macro = macros.get(macroName);
    if (macro != null && macro.checkArgumentsCount(arguments.size())) {
      if (arguments.size() > macro.params.size()) {
        //Group all arguments into the last one
        List<Token> vaargs = arguments.subList(macro.params.size() - 1, arguments.size());
        Token firstToken = vaargs.get(0);
        arguments = arguments.subList(0, macro.params.size() - 1);
        arguments.add(Token.builder()
            .setLine(firstToken.getLine())
            .setColumn(firstToken.getColumn())
            .setURI(firstToken.getURI())
            .setValueAndOriginalValue(serialize(vaargs, ","))
            .setType(STRING)
            .build());
      }
      replTokens = replaceParams(macro.body, macro.params, arguments);
View Full Code Here

  private List<Token> matchArgument(List<Token> tokens, List<Token> arguments) throws MismatchException {
    int nestingLevel = 0;
    int tokensConsumed = 0;
    int noTokens = tokens.size();
    Token firstToken = tokens.get(0);
    Token currToken = firstToken;
    String curr = currToken.getValue();
    List<Token> matchedTokens = new LinkedList<Token>();

    while (true) {
      if (nestingLevel == 0 && (",".equals(curr) || ")".equals(curr))) {
        if (tokensConsumed > 0) {
          arguments.add(Token.builder()
              .setLine(firstToken.getLine())
              .setColumn(firstToken.getColumn())
              .setURI(firstToken.getURI())
              .setValueAndOriginalValue(serialize(matchedTokens))
              .setType(STRING)
              .build());
        }
        return tokens.subList(tokensConsumed, noTokens);
      }

      if (curr.equals("(")) {
        nestingLevel++;
      } else if (curr.equals(")")) {
        nestingLevel--;
      }

      tokensConsumed++;
      if (tokensConsumed == noTokens) {
        throw new MismatchException("reached the end of the stream while matching a macro argument");
      }

      matchedTokens.add(currToken);
      currToken = tokens.get(tokensConsumed);
      curr = currToken.getValue();
    }
  }
View Full Code Here

      for (Token t : parameters) {
        defParamValues.add(t.getValue());
      }

      for (int i = 0; i < body.size(); ++i) {
        Token curr = body.get(i);
        int index = defParamValues.indexOf(curr.getValue());
        if (index == -1) {
          newTokens.add(curr);
        }
        else if (index == arguments.size()) {
          // EXTENSION: GCC's special meaning of token paste operator
          // If variable argument is left out then the comma before the paste operator will be deleted
          int j = i;
          while(j > 0 && body.get(j - 1).getType() == WS)
            j--;
          if (j == 0 || !body.get(--j).getValue().equals("##"))
            continue;
          int k = j;
          while(j > 0 && body.get(j - 1).getType() == WS)
            j--;
          if (j > 0 && body.get(j - 1).getValue().equals(",")) {
            newTokens.remove(newTokens.size() - 1 + j - i); //remove the comma
            newTokens.remove(newTokens.size() - 1 + k - i); //remove the paste operator
          }
        }
        else if (index < arguments.size()) {
          Token replacement = arguments.get(index);

          // The arguments have to be fully expanded before expanding the body of the macro
          String newValue = serialize(expandMacro("", replacement.getValue()));

          if (i > 0 && body.get(i - 1).getValue().equals("#")) {
            newTokens.remove(newTokens.size() - 1);
            newValue = encloseWithQuotes(quote(newValue));
          }
          newTokens.add(Token.builder()
              .setLine(replacement.getLine())
              .setColumn(replacement.getColumn())
              .setURI(replacement.getURI())
              .setValueAndOriginalValue(newValue)
              .setType(replacement.getType())
              .setGeneratedCode(true)
              .build());
        }
      }
    }
View Full Code Here

  private List<Token> evaluateHashhashOperators(List<Token> tokens) {
    List<Token> newTokens = new ArrayList<Token>();

    Iterator<Token> it = tokens.iterator();
    while (it.hasNext()) {
      Token curr = it.next();
      if (curr.getValue().equals("##")) {
        Token pred = predConcatToken(newTokens);
        Token succ = succConcatToken(it);
        newTokens.add(Token.builder()
            .setLine(pred.getLine())
            .setColumn(pred.getColumn())
            .setURI(pred.getURI())
            .setValueAndOriginalValue(pred.getValue() + succ.getValue())
            .setType(pred.getType())
            .setGeneratedCode(true)
            .build());
      } else {
        newTokens.add(curr);
View Full Code Here

    return newTokens;
  }

  private Token predConcatToken(List<Token> tokens) {
    while (!tokens.isEmpty()) {
      Token last = tokens.remove(tokens.size() - 1);
      if (last.getType() != WS) {
        return last;
      }
    }
    return null;
  }
View Full Code Here

    }
    return null;
  }

  private Token succConcatToken(Iterator<Token> it) {
    Token succ = null;
    while (it.hasNext()) {
      succ = it.next();
      if (!succ.getValue().equals("##") && succ.getType() != WS) {
        break;
      }
    }
    return succ;
  }
View Full Code Here

TOP

Related Classes of com.sonar.sslr.api.Token

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.