***** BEGIN LICENSE BLOCK *****

* Version: EPL 1.0/GPL 2.0/LGPL 2.1

*

* The contents of this file are subject to the Eclipse Public

* License Version 1.0 (the "License"); you may not use this file

* except in compliance with the License. You may obtain a copy of

* the License at http://www.eclipse.org/legal/epl-v10.html

*

* Software distributed under the License is distributed on an "AS

* IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or

* implied. See the License for the specific language governing

* rights and limitations under the License.

*

* Copyright (C) 2002 Benoit Cerrina <b.cerrina@wanadoo.fr>

* Copyright (C) 2002-2004 Anders Bengtsson <ndrsbngtssn@yahoo.se>

* Copyright (C) 2002-2004 Jan Arne Petersen <jpetersen@uni-bonn.de>

* Copyright (C) 2004-2006 Thomas E Enebo <enebo@acm.org>

* Copyright (C) 2004 Stefan Matthias Aust <sma@3plus4.de>

* Copyright (C) 2004-2005 David Corbin <dcorbin@users.sourceforge.net>

* Copyright (C) 2005 Zach Dennis <zdennis@mktec.com>

* Copyright (C) 2006 Thomas Corbat <tcorbat@hsr.ch>

*

* Alternatively, the contents of this file may be used under the terms of

* either of the GNU General Public License Version 2 or later (the "GPL"),

* or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),

* in which case the provisions of the GPL or the LGPL are applicable instead

* of those above. If you wish to allow use of your version of this file only

* under the terms of either the GPL or the LGPL, and not to allow others to

* use your version of this file under the terms of the EPL, indicate your

* decision by deleting the provisions above and replace them with the notice

* and other provisions required by the GPL or the LGPL. If you do not delete

* the provisions above, a recipient may use your version of this file under

* the terms of any one of the EPL, the GPL or the LGPL.

***** END LICENSE BLOCK *****/

* This is a port of the MRI lexer to Java.

*/

    public static final Encoding UTF8_ENCODING = UTF8Encoding.INSTANCE;

    public static final Encoding USASCII_ENCODING = USASCIIEncoding.INSTANCE;

    public static final Encoding ASCII8BIT_ENCODING = ASCIIEncoding.INSTANCE;

   

    private static final ByteList END_MARKER = new ByteList(new byte[] {'_', 'E', 'N', 'D', '_', '_'});

    private static final ByteList BEGIN_DOC_MARKER = new ByteList(new byte[] {'b', 'e', 'g', 'i', 'n'});

    private static final ByteList END_DOC_MARKER = new ByteList(new byte[] {'e', 'n', 'd'});

    private static final HashMap<String, Keyword> map;

   

    private static final int SUFFIX_R = 1<<0;

    private static final int SUFFIX_I = 1<<1;

    private static final int SUFFIX_ALL = 3;

    static {

        map = new HashMap<String, Keyword>();

        map.put("end", Keyword.END);

        map.put("else", Keyword.ELSE);

        map.put("case", Keyword.CASE);

        map.put("ensure", Keyword.ENSURE);

        map.put("module", Keyword.MODULE);

        map.put("elsif", Keyword.ELSIF);

        map.put("def", Keyword.DEF);

        map.put("rescue", Keyword.RESCUE);

        map.put("not", Keyword.NOT);

        map.put("then", Keyword.THEN);

        map.put("yield", Keyword.YIELD);

        map.put("for", Keyword.FOR);

        map.put("self", Keyword.SELF);

        map.put("false", Keyword.FALSE);

        map.put("retry", Keyword.RETRY);

        map.put("return", Keyword.RETURN);

        map.put("true", Keyword.TRUE);

        map.put("if", Keyword.IF);

        map.put("defined?", Keyword.DEFINED_P);

        map.put("super", Keyword.SUPER);

        map.put("undef", Keyword.UNDEF);

        map.put("break", Keyword.BREAK);

        map.put("in", Keyword.IN);

        map.put("do", Keyword.DO);

        map.put("nil", Keyword.NIL);

        map.put("until", Keyword.UNTIL);

        map.put("unless", Keyword.UNLESS);

        map.put("or", Keyword.OR);

        map.put("next", Keyword.NEXT);

        map.put("when", Keyword.WHEN);

        map.put("redo", Keyword.REDO);

        map.put("and", Keyword.AND);

        map.put("begin", Keyword.BEGIN);

        map.put("__LINE__", Keyword.__LINE__);

        map.put("class", Keyword.CLASS);

        map.put("__FILE__", Keyword.__FILE__);

        map.put("END", Keyword.LEND);

        map.put("BEGIN", Keyword.LBEGIN);

        map.put("while", Keyword.WHILE);

        map.put("alias", Keyword.ALIAS);

        map.put("__ENCODING__", Keyword.__ENCODING__);

    }

    private Encoding encoding;

    public Encoding getEncoding() {

        return encoding;

    }

    private int getFloatToken(String number, int suffix) {

        if ((suffix & SUFFIX_R) != 0) {

            BigDecimal bd = new BigDecimal(number);

            BigDecimal denominator = BigDecimal.ONE.scaleByPowerOfTen(bd.scale());

            BigDecimal numerator = bd.multiply(denominator);

            try {

                yaccValue = new RationalNode(getPosition(), numerator.longValueExact(), denominator.longValueExact());

            } catch (ArithmeticException ae) {

                // FIXME: Rational supports Bignum numerator and denominator

                throw new SyntaxException(PID.RATIONAL_OUT_OF_RANGE, getPosition(), getCurrentLine(), "Rational (" + numerator + "/" + denominator + ") out of range.");

            }

            return Tokens.tRATIONAL;

        }

        double d;

        try {

            d = SafeDoubleParser.parseDouble(number);

        } catch (NumberFormatException e) {

            warnings.warn(ID.FLOAT_OUT_OF_RANGE, getPosition(), "Float " + number + " out of range.");

            d = number.startsWith("-") ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;

        }

        yaccValue = new FloatNode(getPosition(), d);

        return Tokens.tFLOAT;

    }

    private BignumNode newBignumNode(String value, int radix) {

        return new BignumNode(getPosition(), new BigInteger(value, radix));

    }

    private FixnumNode newFixnumNode(String value, int radix) throws NumberFormatException {

        return new FixnumNode(getPosition(), Long.parseLong(value, radix));

    }

   

    private RationalNode newRationalNode(String value, int radix) throws NumberFormatException {

        return new RationalNode(getPosition(), Long.parseLong(value, radix), 1);

    }

   

    private ComplexNode newComplexNode(Node number) {

        return new ComplexNode(getPosition(), number);

    }

   

    private void ambiguousOperator(String op, String syn) {

        warnings.warn(ID.AMBIGUOUS_ARGUMENT, "`" + op + "' after local variable is interpreted as binary operator\nevent though it seems like \"" + syn + "\"");

    }

   

    private void warn_balanced(int c, boolean spaceSeen, String op, String syn) {

        if (false && last_state != LexState.EXPR_CLASS && last_state != LexState.EXPR_DOT &&

                last_state != LexState.EXPR_FNAME && last_state != LexState.EXPR_ENDFN &&

                last_state != LexState.EXPR_ENDARG && spaceSeen && !Character.isWhitespace(c)) {

            ambiguousOperator(op, syn);

        }

    }

    // FIXME: Also sucks that matchMarker will strip off valuable bytes and not work for this (could be a one-liner)

    private void detectUTF8BOM() throws IOException {

        int b1 = src.read();

        if (b1 == 0xef) {

            int b2 = src.read();

            if (b2 == 0xbb) {

                int b3 = src.read();

                if (b3 == 0xbf) {

                    setEncoding(UTF8_ENCODING);

                } else {

                    src.unread(b3);

                    src.unread(b2);

                    src.unread(b1);

                }

            } else {

                src.unread(b2);

                src.unread(b1);

            }

        } else {

            src.unread(b1);

        }

    }

    private int numberLiteralSuffix(int mask) throws IOException {

        int c = src.read();

       

        if (c == 'i') return (mask & SUFFIX_I) != 0 ?  mask & SUFFIX_I : 0;

       

        if (c == 'r') {

            int result = 0;

            if ((mask & SUFFIX_R) != 0) result |= (mask & SUFFIX_R);

           

            if (src.peek('i') && (mask & SUFFIX_I) != 0) {

                c = src.read();

                result |= (mask & SUFFIX_I);

            }

           

            return result;

        }

        src.unread(c);

        return 0;

    }

  

    public enum Keyword {

        END ("end", Tokens.kEND, Tokens.kEND, LexState.EXPR_END),

        ELSE ("else", Tokens.kELSE, Tokens.kELSE, LexState.EXPR_BEG),

        CASE ("case", Tokens.kCASE, Tokens.kCASE, LexState.EXPR_BEG),

        ENSURE ("ensure", Tokens.kENSURE, Tokens.kENSURE, LexState.EXPR_BEG),

        MODULE ("module", Tokens.kMODULE, Tokens.kMODULE, LexState.EXPR_BEG),

        ELSIF ("elsif", Tokens.kELSIF, Tokens.kELSIF, LexState.EXPR_BEG),

        DEF ("def", Tokens.kDEF, Tokens.kDEF, LexState.EXPR_FNAME),

        RESCUE ("rescue", Tokens.kRESCUE, Tokens.kRESCUE_MOD, LexState.EXPR_MID),

        NOT ("not", Tokens.kNOT, Tokens.kNOT, LexState.EXPR_BEG),

        THEN ("then", Tokens.kTHEN, Tokens.kTHEN, LexState.EXPR_BEG),

        YIELD ("yield", Tokens.kYIELD, Tokens.kYIELD, LexState.EXPR_ARG),

        FOR ("for", Tokens.kFOR, Tokens.kFOR, LexState.EXPR_BEG),

        SELF ("self", Tokens.kSELF, Tokens.kSELF, LexState.EXPR_END),

        FALSE ("false", Tokens.kFALSE, Tokens.kFALSE, LexState.EXPR_END),

        RETRY ("retry", Tokens.kRETRY, Tokens.kRETRY, LexState.EXPR_END),

        RETURN ("return", Tokens.kRETURN, Tokens.kRETURN, LexState.EXPR_MID),

        TRUE ("true", Tokens.kTRUE, Tokens.kTRUE, LexState.EXPR_END),

        IF ("if", Tokens.kIF, Tokens.kIF_MOD, LexState.EXPR_BEG),

        DEFINED_P ("defined?", Tokens.kDEFINED, Tokens.kDEFINED, LexState.EXPR_ARG),

        SUPER ("super", Tokens.kSUPER, Tokens.kSUPER, LexState.EXPR_ARG),

        UNDEF ("undef", Tokens.kUNDEF, Tokens.kUNDEF, LexState.EXPR_FNAME),

        BREAK ("break", Tokens.kBREAK, Tokens.kBREAK, LexState.EXPR_MID),

        IN ("in", Tokens.kIN, Tokens.kIN, LexState.EXPR_BEG),

        DO ("do", Tokens.kDO, Tokens.kDO, LexState.EXPR_BEG),

        NIL ("nil", Tokens.kNIL, Tokens.kNIL, LexState.EXPR_END),

        UNTIL ("until", Tokens.kUNTIL, Tokens.kUNTIL_MOD, LexState.EXPR_BEG),

        UNLESS ("unless", Tokens.kUNLESS, Tokens.kUNLESS_MOD, LexState.EXPR_BEG),

        OR ("or", Tokens.kOR, Tokens.kOR, LexState.EXPR_BEG),

        NEXT ("next", Tokens.kNEXT, Tokens.kNEXT, LexState.EXPR_MID),

        WHEN ("when", Tokens.kWHEN, Tokens.kWHEN, LexState.EXPR_BEG),

        REDO ("redo", Tokens.kREDO, Tokens.kREDO, LexState.EXPR_END),

        AND ("and", Tokens.kAND, Tokens.kAND, LexState.EXPR_BEG),

        BEGIN ("begin", Tokens.kBEGIN, Tokens.kBEGIN, LexState.EXPR_BEG),

        __LINE__ ("__LINE__", Tokens.k__LINE__, Tokens.k__LINE__, LexState.EXPR_END),

        CLASS ("class", Tokens.kCLASS, Tokens.kCLASS, LexState.EXPR_CLASS),

        __FILE__("__FILE__", Tokens.k__FILE__, Tokens.k__FILE__, LexState.EXPR_END),

        LEND ("END", Tokens.klEND, Tokens.klEND, LexState.EXPR_END),

        LBEGIN ("BEGIN", Tokens.klBEGIN, Tokens.klBEGIN, LexState.EXPR_END),

        WHILE ("while", Tokens.kWHILE, Tokens.kWHILE_MOD, LexState.EXPR_BEG),

        ALIAS ("alias", Tokens.kALIAS, Tokens.kALIAS, LexState.EXPR_FNAME),

        __ENCODING__("__ENCODING__", Tokens.k__ENCODING__, Tokens.k__ENCODING__, LexState.EXPR_END);

       

        public final String name;

        public final int id0;

        public final int id1;

        public final LexState state;

       

        Keyword(String name, int id0, int id1, LexState state) {

            this.name = name;

            this.id0 = id0;

            this.id1 = id1;

            this.state = state;

        }

    }

   

    public enum LexState {

        EXPR_BEG, EXPR_END, EXPR_ARG, EXPR_CMDARG, EXPR_ENDARG, EXPR_MID,

        EXPR_FNAME, EXPR_DOT, EXPR_CLASS, EXPR_VALUE, EXPR_ENDFN

    }

   

    public static Keyword getKeyword(String str) {

        return (Keyword) map.get(str);

    }

    // Last token read via yylex().

    private int token;

   

    // Value of last token which had a value associated with it.

    Object yaccValue;

    // Stream of data that yylex() examines.

    private LexerSource src;

   

    // Used for tiny smidgen of grammar in lexer (see setParserSupport())

    private ParserSupport parserSupport = null;

    // What handles warnings

    private IRubyWarnings warnings;

    // Additional context surrounding tokens that both the lexer and

    // grammar use.

    private LexState lex_state;

    private LexState last_state;

    public ISourcePosition tokline;

    public void startOfToken() {

        src.startOfToken();

    }

    public void newtok() {

        tokline = getPosition();

    }

    // Tempory buffer to build up a potential token.  Consumer takes responsibility to reset

    // this before use.

    private StringBuilder tokenBuffer = new StringBuilder(60);

    private StackState conditionState = new StackState();

    private StackState cmdArgumentState = new StackState();

    private StrTerm lex_strterm;

    public boolean commandStart;

    // Give a name to a value.  Enebo: This should be used more.

    static final int EOF = -1;

    // ruby constants for strings (should this be moved somewhere else?)

    static final int STR_FUNC_ESCAPE=0x01;

    static final int STR_FUNC_EXPAND=0x02;

    static final int STR_FUNC_REGEXP=0x04;

    static final int STR_FUNC_QWORDS=0x08;

    static final int STR_FUNC_SYMBOL=0x10;

    // When the heredoc identifier specifies <<-EOF that indents before ident. are ok (the '-').

    static final int STR_FUNC_INDENT=0x20;

    private static final int str_squote = 0;

    private static final int str_dquote = STR_FUNC_EXPAND;

    private static final int str_xquote = STR_FUNC_EXPAND;

    private static final int str_regexp = STR_FUNC_REGEXP | STR_FUNC_ESCAPE | STR_FUNC_EXPAND;

    private static final int str_ssym   = STR_FUNC_SYMBOL;

    private static final int str_dsym   = STR_FUNC_SYMBOL | STR_FUNC_EXPAND;

    // Count of nested parentheses

    private int parenNest = 0;

    private int leftParenBegin = 0;

    public int incrementParenNest() {

        parenNest++;

        return parenNest;

    }

    public int getLeftParenBegin() {

        return leftParenBegin;

    }

    public void setLeftParenBegin(int value) {

        leftParenBegin = value;

    }

    public RubyLexer() {

        reset();

    }

   

    public final void reset() {

      token = 0;

        tokline = null;

      yaccValue = null;

      src = null;

        setState(null);

        resetStacks();

        lex_strterm = null;

        commandStart = true;

    }

    public int nextToken() throws IOException {

        token = yylex();

        return token == EOF ? 0 : token;

    }

   

    /**

     * Last token read from the lexer at the end of a call to yylex()

     *

     * @return last token read

     */

    public int token() {

        return token;

    }

    public StringBuilder getTokenBuffer() {

        return tokenBuffer;

    }

   

    /**

     * Value of last token (if it is a token which has a value).

     *

     * @return value of last value-laden token

     */

    public Object value() {

        return yaccValue;

    }

   

    /**

     * Get position information for Token/Node that follows node represented by startPosition

     * and current lexer location.

     *

     * @param startPosition previous node/token

     * @return a new position

     */

    public ISourcePosition getPosition(ISourcePosition startPosition) {

      return src.getPosition(startPosition);

    }

    public ISourcePosition getPosition() {

        return src.getPosition(null);

    }

    public String getCurrentLine() {

        return src.getCurrentLine();

    }

    /**

     * Parse must pass its support object for some check at bottom of

     * yylex().  Ruby does it this way as well (i.e. a little parsing

     * logic in the lexer).

     *

     * @param parserSupport

     */

    public void setParserSupport(ParserSupport parserSupport) {

        this.parserSupport = parserSupport;

    }

    private void setEncoding(ByteList name) {

        Encoding newEncoding = parserSupport.getConfiguration().getEncodingService().loadEncoding(name);

        if (newEncoding == null) {

            throw new SyntaxException(PID.UNKNOWN_ENCODING, getPosition(),

                    null, "unknown encoding name: " + name.toString());

        }

        if (!newEncoding.isAsciiCompatible()) {

            throw new SyntaxException(PID.NOT_ASCII_COMPATIBLE, getPosition(),

                    null, name.toString() + " is not ASCII compatible");

        }

        setEncoding(newEncoding);

    }

    public void setEncoding(Encoding encoding) {

        this.encoding = encoding;

    }

    /**

     * Allow the parser to set the source for its lexer.

     *

     * @param source where the lexer gets raw data

     */

    public void setSource(LexerSource source) {

        this.src = source;

    }

    public StrTerm getStrTerm() {

        return lex_strterm;

    }

   

    public void setStrTerm(StrTerm strterm) {

        this.lex_strterm = strterm;

    }

    public void resetStacks() {

        conditionState.reset();

        cmdArgumentState.reset();

    }

   

    public void setWarnings(IRubyWarnings warnings) {

        this.warnings = warnings;

    }

    private void printState() {

        if (lex_state == null) {

            System.out.println("NULL");

        } else {

            System.out.println(lex_state);

        }

    }

    public void setState(LexState state) {

        this.lex_state = state;

    }

    public StackState getCmdArgumentState() {

        return cmdArgumentState;

    }

    public StackState getConditionState() {

        return conditionState;

    }

   

    public void setValue(Object yaccValue) {

        this.yaccValue = yaccValue;

    }

    private boolean isNext_identchar() throws IOException {

        int c = src.read();

        src.unread(c);

        return c != EOF && (Character.isLetterOrDigit(c) || c == '_');

    }

    private boolean isBEG() {

        return lex_state == LexState.EXPR_BEG || lex_state == LexState.EXPR_MID ||

                lex_state == LexState.EXPR_CLASS || (lex_state == LexState.EXPR_VALUE);

    }

   

    private boolean isEND() {

        return lex_state == LexState.EXPR_END || lex_state == LexState.EXPR_ENDARG ||

                (lex_state == LexState.EXPR_ENDFN);

    }

    private boolean isARG() {

        return lex_state == LexState.EXPR_ARG || lex_state == LexState.EXPR_CMDARG;

    }

   

    private boolean isLabelPossible(boolean commandState) {

        return ((lex_state == LexState.EXPR_BEG || lex_state == LexState.EXPR_ENDFN) && !commandState) || isARG();

    }

   

    private boolean isSpaceArg(int c, boolean spaceSeen) {

        return isARG() && spaceSeen && !Character.isWhitespace(c);

    }

    private void determineExpressionState() {

        switch (lex_state) {

        case EXPR_FNAME: case EXPR_DOT:

            setState(LexState.EXPR_ARG);

            break;

        default:

            setState(LexState.EXPR_BEG);

            break;

        }

    }

   

    private Object getInteger(String value, int radix, int suffix) {

        Node literalValue = null;

        if ((suffix & SUFFIX_R) != 0) {

            literalValue = newRationalNode(value, radix);

        } else {

            try {

                literalValue = newFixnumNode(value, radix);

            } catch (NumberFormatException e) {

                literalValue = newBignumNode(value, radix);

            }

        }

       

        return (suffix & SUFFIX_I) != 0 ? newComplexNode(literalValue) : literalValue;

    }

  /**

   * @param c the character to test

   * @return true if character is a hex value (0-9a-f)

   */

    static boolean isHexChar(int c) {

        return Character.isDigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F');

    }

    /**

   * @param c the character to test

     * @return true if character is an octal value (0-7)

   */

    static boolean isOctChar(int c) {

        return '0' <= c && c <= '7';

    }

   

    /**

     * This is a valid character for an identifier?

     *

     * @param c is character to be compared

     * @return whether c is an identifier or not

     *

     * mri: is_identchar

     */

    public boolean isIdentifierChar(int c) {

        return Character.isLetterOrDigit(c) || c == '_' || isMultiByteChar(c);

    }

    public boolean isASCII(int c) {

        return !isMultiByteChar(c);

    }

   

    /**

     * Is this a multibyte character from a multibyte encoding?

     *

     * @param c

     * @return whether c is an multibyte char or not

     */

    protected boolean isMultiByteChar(int c) {

        return encoding.codeToMbcLength(c) != 1;

    }

    // STR_NEW3/parser_str_new

    public StrNode createStrNode(ISourcePosition position, ByteList buffer, int flags) {