// Author: sreeni@google.com (Sreeni Viswanadha)

* All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions are met:

*

*     * Redistributions of source code must retain the above copyright notice,

*       this list of conditions and the following disclaimer.

*     * Redistributions in binary form must reproduce the above copyright

*       notice, this list of conditions and the following disclaimer in the

*       documentation and/or other materials provided with the distribution.

*     * Neither the name of the Sun Microsystems, Inc. nor the names of its

*       contributors may be used to endorse or promote products derived from

*       this software without specific prior written permission.

*

* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE

* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF

* THE POSSIBILITY OF SUCH DAMAGE.

*/

* Generate lexer.

*/

  public static String staticString;

  public static String tokMgrClassName;

  // Hashtable of vectors

  static Hashtable allTpsForState = new Hashtable();

  public static int lexStateIndex = 0;

  static int[] kinds;

  public static int maxOrdinal = 1;

  public static String lexStateSuffix;

  static String[] newLexState;

  public static int[] lexStates;

  public static boolean[] ignoreCase;

  public static Action[] actions;

  public static Hashtable initStates = new Hashtable();

  public static int stateSetSize;

  public static int maxLexStates;

  public static String[] lexStateName;

  static NfaState[] singlesToSkip;

  public static long[] toSkip;

  public static long[] toSpecial;

  public static long[] toMore;

  public static long[] toToken;

  public static int defaultLexState;

  public static RegularExpression[] rexprs;

  public static int[] maxLongsReqd;

  public static int[] initMatch;

  public static int[] canMatchAnyChar;

  public static boolean hasEmptyMatch;

  public static boolean[] canLoop;

  public static boolean[] stateHasActions;

  public static boolean hasLoop = false;

  public static boolean[] canReachOnMore;

  public static boolean[] hasNfa;

  public static boolean[] mixed;

  public static NfaState initialState;

  public static int curKind;

  static boolean hasSkipActions = false;

  static boolean hasMoreActions = false;

  static boolean hasTokenActions = false;

  static boolean hasSpecial = false;

  static boolean hasSkip = false;

  static boolean hasMore = false;

  public static RegularExpression curRE;

  public static boolean keepLineCol;

  void PrintClassHead()

  {

    int i, j;

    List tn = new ArrayList(toolNames);

    tn.add(toolName);

    genCodeLine("/* " + getIdString(tn, tokMgrClassName + getFileExtension(Options.getOutputLanguage())) + " */");

   

    int l = 0, kind;

    i = 1;

    for (;;)

    {

      if (cu_to_insertion_point_1.size() <= l)

        break;

      kind = ((Token)cu_to_insertion_point_1.get(l)).kind;

      if(kind == PACKAGE || kind == IMPORT) {

        for (; i < cu_to_insertion_point_1.size(); i++) {

          kind = ((Token)cu_to_insertion_point_1.get(i)).kind;

          if (kind == SEMICOLON ||

              kind == ABSTRACT ||

              kind == FINAL ||

              kind == PUBLIC ||

              kind == CLASS ||

              kind == INTERFACE)

          {

            cline = ((Token)(cu_to_insertion_point_1.get(l))).beginLine;

            ccol = ((Token)(cu_to_insertion_point_1.get(l))).beginColumn;

            for (j = l; j < i; j++) {

              printToken((Token)(cu_to_insertion_point_1.get(j)));

            }

            if (kind == SEMICOLON)

              printToken((Token)(cu_to_insertion_point_1.get(j)));

            genCodeLine("");

            break;

          }

        }

        l = ++i;

      }

      else if (((Token)cu_to_insertion_point_1.get(l)).specialToken != null)

      {

        getSpecialTokenToPrint(((Token)cu_to_insertion_point_1.get(l)));

      }

        break;

    }

    genCodeLine("");

    genCodeLine("/** Token Manager. */");

    //genCodeLine("@SuppressWarnings(\"unused\")");

    genAnnotation("SuppressWarnings(\"unused\")\r\n");

    if(Options.getSupportClassVisibilityPublic()) {

      //genModifier("public ");

      genModifier("public ");

    }

    //genCodeLine("class " + tokMgrClassName + " implements " +

        //cu_name + "Constants");

    //String superClass = Options.stringValue("TOKEN_MANAGER_SUPER_CLASS");

    genClassStart(null, tokMgrClassName, new String[]{}, new String[]{cu_name + "Constants"});

    //genCodeLine("{"); // }

    if (token_mgr_decls != null && token_mgr_decls.size() > 0)

    {

      Token t = (Token)token_mgr_decls.get(0);

      boolean commonTokenActionSeen = false;

      boolean commonTokenActionNeeded = Options.getCommonTokenAction();

      printTokenSetup((Token)token_mgr_decls.get(0));

      ccol = 1;

      for (j = 0; j < token_mgr_decls.size(); j++)

      {

        t = (Token)token_mgr_decls.get(j);

        if (t.kind == IDENTIFIER &&

            commonTokenActionNeeded &&

            !commonTokenActionSeen)

          commonTokenActionSeen = t.image.equals("CommonTokenAction");

        printToken(t);

      }

      genCodeLine("");

      if (commonTokenActionNeeded && !commonTokenActionSeen)

        JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +

            "But it appears you have not defined the method :\n"+

            "      " + staticString + "void CommonTokenAction(Token t)\n" +

        "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");

    }

    else if (Options.getCommonTokenAction())

    {

      JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +

          "But you have not defined the method :\n"+

          "      " + staticString + "void CommonTokenAction(Token t)\n" +

      "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");

    }

    genCodeLine("");

    genCodeLine("  /** Debug output. */");

    genCodeLine("  public " + staticString + " java.io.PrintStream debugStream = System.out;");

    genCodeLine("  /** Set debug output. */");

    genCodeLine("  public " + staticString + " void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }");

    if(Options.getTokenManagerUsesParser() && !Options.getStatic()){

      genCodeLine("");

      genCodeLine("  /** The parser. */");

      genCodeLine("  public " + cu_name + " parser = null;");

    }

  }

 

  @SuppressWarnings("unchecked")

  protected void writeTemplate(String name, Object... additionalOptions) throws IOException

  {

    Map<String, Object> options = new HashMap<String, Object>(Options.getOptions());

    options.put("maxOrdinal", Integer.valueOf(maxOrdinal));

    options.put("maxLexStates", Integer.valueOf(maxLexStates));

    options.put("hasEmptyMatch", Boolean.valueOf(hasEmptyMatch));

    options.put("hasSkip", Boolean.valueOf(hasSkip));

    options.put("hasMore", Boolean.valueOf(hasMore));

    options.put("hasSpecial", Boolean.valueOf(hasSpecial));

    options.put("hasMoreActions", Boolean.valueOf(hasMoreActions));

    options.put("hasSkipActions", Boolean.valueOf(hasSkipActions));

    options.put("hasTokenActions", Boolean.valueOf(hasTokenActions));

    options.put("stateSetSize", stateSetSize);

    options.put("hasActions", hasMoreActions || hasSkipActions || hasTokenActions);

    options.put("tokMgrClassName", tokMgrClassName);

    options.put("cu_name", cu_name);

    // options.put("", .valueOf(maxOrdinal));

   

   

    for (int i = 0; i < additionalOptions.length; i++)

    {

      Object o = additionalOptions[i];

   

      if (o instanceof Map<?,?>)

      {

        options.putAll((Map<String,Object>) o);

      }

      else

      {

        if (i == additionalOptions.length - 1)

          throw new IllegalArgumentException("Must supply pairs of [name value] args");

       

        options.put((String) o, additionalOptions[i+1]);

        i++;

      }

    }

   

    JavaFileGenerator gen = new JavaFileGenerator(name, options);

    StringWriter sw = new StringWriter();

    gen.generate(new PrintWriter(sw));

    sw.close();

    genCode(sw.toString());

  }

  void DumpDebugMethods() throws IOException

  {

    writeTemplate("/templates/DumpDebugMethods.template");

  }

  static void BuildLexStatesTable()

  {

    Iterator it = rexprlist.iterator();

    TokenProduction tp;

    int i;

    String[] tmpLexStateName = new String[lexstate_I2S.size()];

    while (it.hasNext())

    {

      tp = (TokenProduction)it.next();

      List respecs = tp.respecs;

      List tps;

      for (i = 0; i < tp.lexStates.length; i++)

      {

        if ((tps = (List)allTpsForState.get(tp.lexStates[i])) == null)

        {

          tmpLexStateName[maxLexStates++] = tp.lexStates[i];

          allTpsForState.put(tp.lexStates[i], tps = new ArrayList());

        }

        tps.add(tp);

      }

      if (respecs == null || respecs.size() == 0)

        continue;

      RegularExpression re;

      for (i = 0; i < respecs.size(); i++)

        if (maxOrdinal <= (re = ((RegExprSpec)respecs.get(i)).rexp).ordinal)

          maxOrdinal = re.ordinal + 1;

    }

    kinds = new int[maxOrdinal];

    toSkip = new long[maxOrdinal / 64 + 1];

    toSpecial = new long[maxOrdinal / 64 + 1];

    toMore = new long[maxOrdinal / 64 + 1];

    toToken = new long[maxOrdinal / 64 + 1];

    toToken[0] = 1L;

    actions = new Action[maxOrdinal];

    actions[0] = actForEof;

    hasTokenActions = actForEof != null;

    initStates = new Hashtable();

    canMatchAnyChar = new int[maxLexStates];

    canLoop = new boolean[maxLexStates];

    stateHasActions = new boolean[maxLexStates];

    lexStateName = new String[maxLexStates];

    singlesToSkip = new NfaState[maxLexStates];

    System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);

    for (i = 0; i < maxLexStates; i++)

      canMatchAnyChar[i] = -1;

    hasNfa = new boolean[maxLexStates];

    mixed = new boolean[maxLexStates];

    maxLongsReqd = new int[maxLexStates];

    initMatch = new int[maxLexStates];

    newLexState = new String[maxOrdinal];

    newLexState[0] = nextStateForEof;

    hasEmptyMatch = false;

    lexStates = new int[maxOrdinal];

    ignoreCase = new boolean[maxOrdinal];

    rexprs = new RegularExpression[maxOrdinal];

    RStringLiteral.allImages = new String[maxOrdinal];

    canReachOnMore = new boolean[maxLexStates];

  }

  static int GetIndex(String name)

  {

    for (int i = 0; i < lexStateName.length; i++)

      if (lexStateName[i] != null && lexStateName[i].equals(name))

        return i;

    throw new Error(); // Should never come here

  }

  public static void AddCharToSkip(char c, int kind)

  {

    singlesToSkip[lexStateIndex].AddChar(c);

    singlesToSkip[lexStateIndex].kind = kind;

  }

  public void start() throws IOException

  {

    if (!Options.getBuildTokenManager() ||

        Options.getUserTokenManager() ||

        JavaCCErrors.get_error_count() > 0)

      return;

    keepLineCol = Options.getKeepLineColumn();

    List choices = new ArrayList();

    Enumeration e;

    TokenProduction tp;

    int i, j;

    staticString = (Options.getStatic() ? "static " : "");

    tokMgrClassName = cu_name + "TokenManager";

    PrintClassHead();

    BuildLexStatesTable();

    e = allTpsForState.keys();

    boolean ignoring = false;

    while (e.hasMoreElements())

    {

      NfaState.ReInit();

      RStringLiteral.ReInit();

      String key = (String)e.nextElement();

      lexStateIndex = GetIndex(key);

      lexStateSuffix = "_" + lexStateIndex;

      List allTps = (List)allTpsForState.get(key);

      initStates.put(key, initialState = new NfaState());

      ignoring = false;

      singlesToSkip[lexStateIndex] = new NfaState();

      singlesToSkip[lexStateIndex].dummy = true;

      if (key.equals("DEFAULT"))

        defaultLexState = lexStateIndex;

      for (i = 0; i < allTps.size(); i++)

      {

        tp = (TokenProduction)allTps.get(i);

        int kind = tp.kind;

        boolean ignore = tp.ignoreCase;

        List rexps = tp.respecs;

        if (i == 0)

          ignoring = ignore;

        for (j = 0; j < rexps.size(); j++)

        {

          RegExprSpec respec = (RegExprSpec)rexps.get(j);

          curRE = respec.rexp;

          rexprs[curKind = curRE.ordinal] = curRE;

          lexStates[curRE.ordinal] = lexStateIndex;

          ignoreCase[curRE.ordinal] = ignore;

          if (curRE.private_rexp)

          {

            kinds[curRE.ordinal] = -1;

            continue;

          }

          if (curRE instanceof RStringLiteral &&

              !((RStringLiteral)curRE).image.equals(""))

          {

            ((RStringLiteral)curRE).GenerateDfa(this, curRE.ordinal);

            if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)

              mixed[lexStateIndex] = true;

          }

          else if (curRE.CanMatchAnyChar())

          {

            if (canMatchAnyChar[lexStateIndex] == -1 ||

                canMatchAnyChar[lexStateIndex] > curRE.ordinal)

              canMatchAnyChar[lexStateIndex] = curRE.ordinal;

          }

          else

          {

            Nfa temp;

            if (curRE instanceof RChoice)

              choices.add(curRE);

            temp = curRE.GenerateNfa(ignore);

            temp.end.isFinal = true;

            temp.end.kind = curRE.ordinal;

            initialState.AddMove(temp.start);

          }

          if (kinds.length < curRE.ordinal)

          {

            int[] tmp = new int[curRE.ordinal + 1];

            System.arraycopy(kinds, 0, tmp, 0, kinds.length);

            kinds = tmp;

          }

          //System.out.println("   ordina : " + curRE.ordinal);

          kinds[curRE.ordinal] = kind;

          if (respec.nextState != null &&

              !respec.nextState.equals(lexStateName[lexStateIndex]))

            newLexState[curRE.ordinal] = respec.nextState;

          if (respec.act != null && respec.act.getActionTokens() != null &&

              respec.act.getActionTokens().size() > 0)

            actions[curRE.ordinal] = respec.act;

          switch(kind)

          {

          case TokenProduction.SPECIAL :

            hasSkipActions |= (actions[curRE.ordinal] != null) ||

            (newLexState[curRE.ordinal] != null);

            hasSpecial = true;

            toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            break;

          case TokenProduction.SKIP :

            hasSkipActions |= (actions[curRE.ordinal] != null);

            hasSkip = true;

            toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            break;

          case TokenProduction.MORE :

            hasMoreActions |= (actions[curRE.ordinal] != null);

            hasMore = true;

            toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            if (newLexState[curRE.ordinal] != null)

              canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;

            else

              canReachOnMore[lexStateIndex] = true;

            break;

          case TokenProduction.TOKEN :

            hasTokenActions |= (actions[curRE.ordinal] != null);

            toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            break;

          }

        }

      }

      // Generate a static block for initializing the nfa transitions

      NfaState.ComputeClosures();

      for (i = 0; i < initialState.epsilonMoves.size(); i++)

        ((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();

      if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))

      {

        initialState.GenerateCode();

        initialState.GenerateInitMoves(this);

      }

      if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)

      {

        if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||

            (toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)

          hasSkipActions = true;

        else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)

          hasMoreActions = true;

        else

          hasTokenActions = true;

        if (initMatch[lexStateIndex] == 0 ||

            initMatch[lexStateIndex] > initialState.kind)

        {

          initMatch[lexStateIndex] = initialState.kind;

          hasEmptyMatch = true;

        }

      }

      else if (initMatch[lexStateIndex] == 0)

        initMatch[lexStateIndex] = Integer.MAX_VALUE;

      RStringLiteral.FillSubString();

      if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])

        RStringLiteral.GenerateNfaStartStates(this, initialState);

      RStringLiteral.DumpDfaCode(this);

      if (hasNfa[lexStateIndex])

        NfaState.DumpMoveNfa(this);

      if (stateSetSize < NfaState.generatedStates)

        stateSetSize = NfaState.generatedStates;

    }

    for (i = 0; i < choices.size(); i++)

      ((RChoice)choices.get(i)).CheckUnmatchability();

    NfaState.DumpStateSets(this);

    CheckEmptyStringMatch();

    NfaState.DumpNonAsciiMoveMethods(this);

    RStringLiteral.DumpStrLiteralImages(this);

    DumpFillToken();

    DumpGetNextToken();

    if (Options.getDebugTokenManager())

    {

      NfaState.DumpStatesForKind(this);

      DumpDebugMethods();

    }

    if (hasLoop)

    {

      genCodeLine(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");

      genCodeLine(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");

      genCodeLine(staticString + "" + Options.getBooleanType() + "[] jjbeenHere = new " + Options.getBooleanType() + "[" + maxLexStates + "];");

    }

    if (hasSkipActions)

      DumpSkipActions();

    if (hasMoreActions)

      DumpMoreActions();

    if (hasTokenActions)

      DumpTokenActions();

    NfaState.PrintBoilerPlate(this);

    String charStreamName;

    if (Options.getUserCharStream())

      charStreamName = "CharStream";

    else

    {

      if (Options.getJavaUnicodeEscape())

        charStreamName = "JavaCharStream";

      else

        charStreamName = "SimpleCharStream";

    }

    writeTemplate("/templates/TokenManagerBoilerPlateMethods.template",

      "charStreamName", charStreamName,

      "lexStateNameLength", lexStateName.length);

    DumpStaticVarDeclarations(charStreamName);

    genCodeLine(/*{*/ "}");

    String fileName = Options.getOutputDirectory() + File.separator +

                      tokMgrClassName +

                      getFileExtension(Options.getOutputLanguage());

    saveOutput(fileName);

  }

  static void CheckEmptyStringMatch()

  {

    int i, j, k, len;

    boolean[] seen = new boolean[maxLexStates];

    boolean[] done = new boolean[maxLexStates];

    String cycle;

    String reList;

    Outer:

      for (i = 0; i < maxLexStates; i++)

      {

        if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||

            canMatchAnyChar[i] != -1)

          continue;

        done[i] = true;

        len = 0;

        cycle = "";

        reList = "";

        for (k = 0; k < maxLexStates; k++)

          seen[k] = false;

        j = i;

        seen[i] = true;

        cycle += lexStateName[j] + "-->";

        while (newLexState[initMatch[j]] != null)

        {

          cycle += newLexState[initMatch[j]];

          if (seen[j = GetIndex(newLexState[initMatch[j]])])

            break;

          cycle += "-->";

          done[j] = true;

          seen[j] = true;

          if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||

              canMatchAnyChar[j] != -1)

            continue Outer;

          if (len != 0)

            reList += "; ";

          reList += "line " + rexprs[initMatch[j]].getLine() + ", column " +

          rexprs[initMatch[j]].getColumn();

          len++;

        }

        if (newLexState[initMatch[j]] == null)

          cycle += lexStateName[lexStates[initMatch[j]]];

        for (k = 0; k < maxLexStates; k++)

          canLoop[k] |= seen[k];

        hasLoop = true;

        if (len == 0)

          JavaCCErrors.warning(rexprs[initMatch[i]],

              "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))

                  ? "" : (" for " + rexprs[initMatch[i]].label)) +

                  " can be matched by the empty string (\"\") in lexical state " +

                  lexStateName[i] + ". This can result in an endless loop of " +

          "empty string matches.");

        else

        {

          JavaCCErrors.warning(rexprs[initMatch[i]],

              "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))

                  ? "" : (" for " + rexprs[initMatch[i]].label)) +

                  " can be matched by the empty string (\"\") in lexical state " +

                  lexStateName[i] + ". This regular expression along with the " +

                  "regular expressions at " + reList + " forms the cycle \n   " +

                  cycle + "\ncontaining regular expressions with empty matches." +

          " This can result in an endless loop of empty string matches.");

        }

      }

  }

  private void DumpStaticVarDeclarations(String charStreamName) throws IOException

  {

    int i;

    genCodeLine("");

    genCodeLine("/** Lexer state names. */");

    genCodeLine("public static final String[] lexStateNames = {");

    for (i = 0; i < maxLexStates; i++)

      genCodeLine("   \"" + lexStateName[i] + "\",");

    genCodeLine("};");

    if (maxLexStates > 1)

    {

      genCodeLine("");

      genCodeLine("/** Lex State array. */");

      genCode("public static final int[] jjnewLexState = {");

      for (i = 0; i < maxOrdinal; i++)

      {

        if (i % 25 == 0)

          genCode("\n   ");

        if (newLexState[i] == null)

          genCode("-1, ");

        else

          genCode(GetIndex(newLexState[i]) + ", ");

      }

      genCodeLine("\n};");

    }

    if (hasSkip || hasMore || hasSpecial)

    {

      // Bit vector for TOKEN

      genCode("static final long[] jjtoToken = {");

      for (i = 0; i < maxOrdinal / 64 + 1; i++)

      {

        if (i % 4 == 0)

          genCode("\n   ");

        genCode("0x" + Long.toHexString(toToken[i]) + "L, ");

      }

      genCodeLine("\n};");

    }

    if (hasSkip || hasSpecial)

    {

      // Bit vector for SKIP

      genCode("static final long[] jjtoSkip = {");

      for (i = 0; i < maxOrdinal / 64 + 1; i++)

      {

        if (i % 4 == 0)

          genCode("\n   ");

        genCode("0x" + Long.toHexString(toSkip[i]) + "L, ");

      }

      genCodeLine("\n};");

    }

    if (hasSpecial)

    {

      // Bit vector for SPECIAL

      genCode("static final long[] jjtoSpecial = {");

      for (i = 0; i < maxOrdinal / 64 + 1; i++)

      {

        if (i % 4 == 0)

          genCode("\n   ");

        genCode("0x" + Long.toHexString(toSpecial[i]) + "L, ");

      }

      genCodeLine("\n};");

    }

    if (hasMore)

    {

      // Bit vector for MORE

      genCode("static final long[] jjtoMore = {");

      for (i = 0; i < maxOrdinal / 64 + 1; i++)

      {

        if (i % 4 == 0)

          genCode("\n   ");

        genCode("0x" + Long.toHexString(toMore[i]) + "L, ");

      }

      genCodeLine("\n};");

    }

    writeTemplate("/templates/DumpStaticVarDeclarations.template",

      "charStreamName", charStreamName,

      "protected", isJavaLanguage() ? "protected" : "",

      "private", isJavaLanguage() ? "private" : "",

      "final", isJavaLanguage() ? "final" : "",

      "lexStateNameLength", lexStateName.length);

  }

  // Assumes l != 0L

  static char MaxChar(long l)

  {

    for (int i = 64; i-- > 0; )

      if ((l & (1L << i)) != 0L)

        return (char)i;

    return 0xffff;

  }

  void DumpFillToken()

  {

    final double tokenVersion = JavaFiles.getVersion("Token.java");

    final boolean hasBinaryNewToken = tokenVersion > 4.09;

    genCodeLine(staticString + "protected Token jjFillToken()");

    genCodeLine("{");

    genCodeLine("   final Token t;");

    genCodeLine("   final String curTokenImage;");

    if (keepLineCol)

    {

      genCodeLine("   final int beginLine;");

      genCodeLine("   final int endLine;");

      genCodeLine("   final int beginColumn;");

      genCodeLine("   final int endColumn;");

    }

    if (hasEmptyMatch)

    {

      genCodeLine("   if (jjmatchedPos < 0)");

      genCodeLine("   {");

      genCodeLine("      if (image == null)");

      genCodeLine("         curTokenImage = \"\";");

      genCodeLine("      else");

      genCodeLine("         curTokenImage = image.toString();");

      if (keepLineCol)

      {

        genCodeLine("      beginLine = endLine = input_stream.getBeginLine();");

        genCodeLine("      beginColumn = endColumn = input_stream.getBeginColumn();");

      }

      genCodeLine("   }");

      genCodeLine("   else");

      genCodeLine("   {");

      genCodeLine("      String im = jjstrLiteralImages[jjmatchedKind];");

      genCodeLine("      curTokenImage = (im == null) ? input_stream.GetImage() : im;");

      if (keepLineCol)

      {

        genCodeLine("      beginLine = input_stream.getBeginLine();");

        genCodeLine("      beginColumn = input_stream.getBeginColumn();");

        genCodeLine("      endLine = input_stream.getEndLine();");

        genCodeLine("      endColumn = input_stream.getEndColumn();");

      }

      genCodeLine("   }");

    }

    else

    {

      genCodeLine("   String im = jjstrLiteralImages[jjmatchedKind];");

      genCodeLine("   curTokenImage = (im == null) ? input_stream.GetImage() : im;");

      if (keepLineCol)

      {

        genCodeLine("   beginLine = input_stream.getBeginLine();");

        genCodeLine("   beginColumn = input_stream.getBeginColumn();");

        genCodeLine("   endLine = input_stream.getEndLine();");

        genCodeLine("   endColumn = input_stream.getEndColumn();");

      }

    }

    if (Options.getTokenFactory().length() > 0) {

      genCodeLine("   t = " + Options.getTokenFactory() + ".newToken(jjmatchedKind, curTokenImage);");

    } else if (hasBinaryNewToken)

    {

      genCodeLine("   t = Token.newToken(jjmatchedKind, curTokenImage);");

    }

    else

    {

      genCodeLine("   t = Token.newToken(jjmatchedKind);");

      genCodeLine("   t.kind = jjmatchedKind;");

      genCodeLine("   t.image = curTokenImage;");

    }

    if (keepLineCol) {

      genCodeLine("");

      genCodeLine("   t.beginLine = beginLine;");

      genCodeLine("   t.endLine = endLine;");

      genCodeLine("   t.beginColumn = beginColumn;");

      genCodeLine("   t.endColumn = endColumn;");

    }

    genCodeLine("");

    genCodeLine("   return t;");

    genCodeLine("}");

  }

  void DumpGetNextToken()

  {

    int i;

    genCodeLine("");

    genCodeLine(staticString + "int curLexState = " + defaultLexState + ";");

    genCodeLine(staticString + "int defaultLexState = " + defaultLexState + ";");

    genCodeLine(staticString + "int jjnewStateCnt;");

    genCodeLine(staticString + "int jjround;");

    genCodeLine(staticString + "int jjmatchedPos;");

    genCodeLine(staticString + "int jjmatchedKind;");

    genCodeLine("");

    genCodeLine("/** Get the next Token. */");

    genCodeLine("public " + staticString + "Token getNextToken()" +

    " ");

    genCodeLine("{");

    if (hasSpecial) {

      genCodeLine("  Token specialToken = null;");

    }

    genCodeLine("  Token matchedToken;");

    genCodeLine("  int curPos = 0;");

    genCodeLine("");

    genCodeLine("  EOFLoop :\n  for (;;)");

    genCodeLine("  {");

    genCodeLine("   try");

    genCodeLine("   {");

    genCodeLine("      curChar = input_stream.BeginToken();");

    genCodeLine("   }");

    genCodeLine("   catch(java.io.IOException e)");

    genCodeLine("   {");

    if (Options.getDebugTokenManager())

      genCodeLine("      debugStream.println(\"Returning the <EOF> token.\\n\");");

    genCodeLine("      jjmatchedKind = 0;");

    genCodeLine("      jjmatchedPos = -1;");

    genCodeLine("      matchedToken = jjFillToken();");

    if (hasSpecial)

      genCodeLine("      matchedToken.specialToken = specialToken;");

    if (nextStateForEof != null || actForEof != null)

      genCodeLine("      TokenLexicalActions(matchedToken);");

    if (Options.getCommonTokenAction())

      genCodeLine("      CommonTokenAction(matchedToken);");

    genCodeLine("      return matchedToken;");

    genCodeLine("   }");

    if (hasMoreActions || hasSkipActions || hasTokenActions)

    {

      genCodeLine("   image = jjimage;");

      genCodeLine("   image.setLength(0);");

      genCodeLine("   jjimageLen = 0;");

    }

    genCodeLine("");

    String prefix = "";

    if (hasMore)

    {

      genCodeLine("   for (;;)");

      genCodeLine("   {");

      prefix = "  ";

    }

    String endSwitch = "";

    String caseStr = "";

    // this also sets up the start state of the nfa

    if (maxLexStates > 1)

    {

      genCodeLine(prefix + "   switch(curLexState)");

      genCodeLine(prefix + "   {");

      endSwitch = prefix + "   }";

      caseStr = prefix + "     case ";

      prefix += "    ";

    }

    prefix += "   ";

    for(i = 0; i < maxLexStates; i++)

    {

      if (maxLexStates > 1)

        genCodeLine(caseStr + i + ":");

      if (singlesToSkip[i].HasTransitions())

      {

        // added the backup(0) to make JIT happy

        genCodeLine(prefix + "try { input_stream.backup(0);");

        if (singlesToSkip[i].asciiMoves[0] != 0L &&

            singlesToSkip[i].asciiMoves[1] != 0L)

        {

          genCodeLine(prefix + "   while ((curChar < 64" + " && (0x" +

              Long.toHexString(singlesToSkip[i].asciiMoves[0]) +

              "L & (1L << curChar)) != 0L) || \n" +

              prefix + "          (curChar >> 6) == 1" +

              " && (0x" +

              Long.toHexString(singlesToSkip[i].asciiMoves[1]) +

          "L & (1L << (curChar & 077))) != 0L)");

        }

        else if (singlesToSkip[i].asciiMoves[1] == 0L)

        {

          genCodeLine(prefix + "   while (curChar <= " +

              (int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +

              Long.toHexString(singlesToSkip[i].asciiMoves[0]) +

          "L & (1L << curChar)) != 0L)");

        }

        else if (singlesToSkip[i].asciiMoves[0] == 0L)

        {

          genCodeLine(prefix + "   while (curChar > 63 && curChar <= " +

              ((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +

              " && (0x" +

              Long.toHexString(singlesToSkip[i].asciiMoves[1]) +

          "L & (1L << (curChar & 077))) != 0L)");

        }

        if (Options.getDebugTokenManager())

        {

          genCodeLine(prefix + "{");

          genCodeLine("      debugStream.println(" +

              (maxLexStates > 1 ?

                  "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +

                  "\"Skipping character : \" + " +

          "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \")\");");

        }

        genCodeLine(prefix + "      curChar = input_stream.BeginToken();");

        if (Options.getDebugTokenManager())

          genCodeLine(prefix + "}");

        genCodeLine(prefix + "}");

        genCodeLine(prefix + "catch (java.io.IOException e1) { continue EOFLoop; }");

      }

      if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)

      {

        if (Options.getDebugTokenManager())

          genCodeLine("      debugStream.println(\"   Matched the empty string as \" + tokenImage[" +

              initMatch[i] + "] + \" token.\");");

        genCodeLine(prefix + "jjmatchedKind = " + initMatch[i] + ";");

        genCodeLine(prefix + "jjmatchedPos = -1;");

        genCodeLine(prefix + "curPos = 0;");

      }

      else

      {

        genCodeLine(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");

        genCodeLine(prefix + "jjmatchedPos = 0;");

      }

      if (Options.getDebugTokenManager())

        genCodeLine("      debugStream.println(" +

            (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +

            "\"Current character : \" + " +

            "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") " +

        "at line \" + input_stream.getEndLine() + \" column \" + input_stream.getEndColumn());");

      genCodeLine(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");

      if (canMatchAnyChar[i] != -1)

      {

        if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)

          genCodeLine(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +

              canMatchAnyChar[i] + "))");

        else

          genCodeLine(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +

              canMatchAnyChar[i] + ")");

        genCodeLine(prefix + "{");

        if (Options.getDebugTokenManager())

          genCodeLine("           debugStream.println(\"   Current character matched as a \" + tokenImage[" +

              canMatchAnyChar[i] + "] + \" token.\");");

        genCodeLine(prefix + "   jjmatchedKind = " + canMatchAnyChar[i] + ";");

        if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)

          genCodeLine(prefix + "   jjmatchedPos = 0;");

        genCodeLine(prefix + "}");

      }

      if (maxLexStates > 1)

        genCodeLine(prefix + "break;");

    }

    if (maxLexStates > 1)

      genCodeLine(endSwitch);

    else if (maxLexStates == 0)

      genCodeLine("       jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");

    if (maxLexStates > 1)

      prefix = "  ";

    else

      prefix = "";

    if (maxLexStates > 0)

    {

      genCodeLine(prefix + "   if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");

      genCodeLine(prefix + "   {");

      genCodeLine(prefix + "      if (jjmatchedPos + 1 < curPos)");

      if (Options.getDebugTokenManager())

      {

        genCodeLine(prefix + "      {");

        genCodeLine(prefix + "         debugStream.println(" +

        "\"   Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");

      }

      genCodeLine(prefix + "         input_stream.backup(curPos - jjmatchedPos - 1);");

      if (Options.getDebugTokenManager())

        genCodeLine(prefix + "      }");

      if (Options.getDebugTokenManager())

      {

        if (Options.getJavaUnicodeEscape() ||

            Options.getUserCharStream())

          genCodeLine("    debugStream.println(" +

              "\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +

              "(\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +

          "\") ******\\n\");");

        else

          genCodeLine("    debugStream.println(" +

              "\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +

              "(\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +

          "\") ******\\n\");");

      }

      if (hasSkip || hasMore || hasSpecial)

      {

        genCodeLine(prefix + "      if ((jjtoToken[jjmatchedKind >> 6] & " +