Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

     * @tests java.io.StreamTokenizer#slashStarComments(boolean)
     */
    public void test_slashStarComments_withSTClosed() throws IOException {
        Reader reader = new CharArrayReader( "t /* t */ t".toCharArray());


        StreamTokenizer st = new StreamTokenizer(reader);
        st.slashStarComments(false);


        assertEquals(StreamTokenizer.TT_WORD,st.nextToken());
        assertEquals(StreamTokenizer.TT_EOF,st.nextToken());
    }

View Full Code Here

  }


  private void setTest(String s) {
    testString = s;
    r = new Support_StringReader(testString);
    st = new StreamTokenizer(r);
  }

View Full Code Here

   */
  private void processIncludeFile()
    throws IOException, DRDAProtocolException
  {
    String fileName = getString();
    StreamTokenizer saveTkn = tkn;
    processFile(fileName);
    tkn = saveTkn;
  }

View Full Code Here

      // the location is different, try it
      String userdir =  System.getProperty("user.dir");
      String sep =  System.getProperty("file.separator");
      fr = new BufferedReader (new InputStreamReader(new FileInputStream(userdir + sep + filename),"UTF-8"));
    }
    tkn = new StreamTokenizer(fr);
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF)
    {
      switch(val)
      {

View Full Code Here

            /* Check for use of '/WEB-INF/tld/portlet.tld' instead of 'http://java.sun.com/portlet' in taglib declaration*/
            String fileName = entry.getName();
            if( !entry.isDirectory() && entry.getName().endsWith(".jsp")) {
            InputStream is = jarFile.getInputStream(entry);
            Reader r = new BufferedReader(new InputStreamReader(is));
            StreamTokenizer st = new StreamTokenizer(r);
            st.quoteChar('\'');
            st.quoteChar('"');
            while(st.nextToken()!=StreamTokenizer.TT_EOF) {
              if(st.ttype=='\'' || st.ttype=='"'){
                String sval = st.sval;
                String sqc = Character.toString((char)st.ttype);
                if(sval.equals("/WEB-INF/tld/portlet.tld")){
                  System.out.println("Warning: " + sqc+st.sval+sqc + " has been found in file " + fileName + ". Use instead " +sqc+"http://java.sun.com/portlet"+sqc+" with your portlet taglib declaration!\n");

View Full Code Here

    {
        RuleSet ruleSet = super.load();
        
        try
        {
            _st = new StreamTokenizer(new BufferedReader(new FileReader(getFile())));
            _st.resetSyntax(); // setup the tokenizer
                
            _st.commentChar(COMMENT); // single line comments
            _st.eolIsSignificant(true); // return EOL as a token
            _st.ordinaryChar('='); // equals is a token

View Full Code Here

  private static class Lexer {


    private StreamTokenizer tok;


    Lexer(String s) {
      tok = new StreamTokenizer(new CharArrayReader(s.toCharArray()));
      tok.quoteChar('"');
      tok.parseNumbers();
      tok.ordinaryChar(',');
      tok.ordinaryChar('(');
      tok.ordinaryChar(')');

View Full Code Here

            {
                // ignore
            }
        }


        tokenizer = new StreamTokenizer(new BufferedReader(in));
        initTokenizer(tokenizer);
    }

View Full Code Here


    File fullFile = tool.getImportedVocabFile(vocabName);
    try {
      FileReader fr = new FileReader(fullFile);
      BufferedReader br = new BufferedReader(fr);
      StreamTokenizer tokenizer = new StreamTokenizer(br);
      tokenizer.parseNumbers();
      tokenizer.wordChars('_', '_');
      tokenizer.eolIsSignificant(true);
      tokenizer.slashSlashComments(true);
      tokenizer.slashStarComments(true);
      tokenizer.ordinaryChar('=');
      tokenizer.quoteChar('\'');
      tokenizer.whitespaceChars(' ',' ');
      tokenizer.whitespaceChars('\t','\t');
      int lineNum = 1;
      int token = tokenizer.nextToken();
      while (token != StreamTokenizer.TT_EOF) {
        String tokenID;
        if ( token == StreamTokenizer.TT_WORD ) {
          tokenID = tokenizer.sval;
        }
        else if ( token == '\'' ) {
          tokenID = "'"+tokenizer.sval+"'";
        }
        else {
          ErrorManager.error(ErrorManager.MSG_TOKENS_FILE_SYNTAX_ERROR,
                     vocabName+CodeGenerator.VOCAB_FILE_EXTENSION,
                     Utils.integer(lineNum));
          while ( tokenizer.nextToken() != StreamTokenizer.TT_EOL ) {}
          token = tokenizer.nextToken();
          continue;
        }
        token = tokenizer.nextToken();
        if ( token != '=' ) {
          ErrorManager.error(ErrorManager.MSG_TOKENS_FILE_SYNTAX_ERROR,
                     vocabName+CodeGenerator.VOCAB_FILE_EXTENSION,
                     Utils.integer(lineNum));
          while ( tokenizer.nextToken() != StreamTokenizer.TT_EOL ) {}
          token = tokenizer.nextToken();
          continue;
        }
        token = tokenizer.nextToken(); // skip '='
        if ( token != StreamTokenizer.TT_NUMBER ) {
          ErrorManager.error(ErrorManager.MSG_TOKENS_FILE_SYNTAX_ERROR,
                     vocabName+CodeGenerator.VOCAB_FILE_EXTENSION,
                     Utils.integer(lineNum));
          while ( tokenizer.nextToken() != StreamTokenizer.TT_EOL ) {}
          token = tokenizer.nextToken();
          continue;
        }
        int tokenType = (int)tokenizer.nval;
        token = tokenizer.nextToken();
        //System.out.println("import "+tokenID+"="+tokenType);
        composite.maxTokenType = Math.max(composite.maxTokenType,tokenType);
        defineToken(tokenID, tokenType);
        lineNum++;
        if ( token != StreamTokenizer.TT_EOL ) {
          ErrorManager.error(ErrorManager.MSG_TOKENS_FILE_SYNTAX_ERROR,
                     vocabName+CodeGenerator.VOCAB_FILE_EXTENSION,
                     Utils.integer(lineNum));
          while ( tokenizer.nextToken() != StreamTokenizer.TT_EOL ) {}
          token = tokenizer.nextToken();
          continue;
        }
        token = tokenizer.nextToken(); // skip newline
      }
      br.close();
    }
    catch (FileNotFoundException fnfe) {
      ErrorManager.error(ErrorManager.MSG_CANNOT_FIND_TOKENS_FILE,

View Full Code Here

    return new ElementMatcher(tokenizer.sval);
  }


  public static NodeMatcher parse(final String s) throws IOException
  {
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(s));
    tokenizer.wordChars('0', '9');
    tokenizer.ordinaryChar('.');
    tokenizer.ordinaryChar(',');
    tokenizer.ordinaryChars(0, ' ');


    ElementMatcher elementMatcher = null;
    NodeMatcher n = null;
    Type selectorType = Type.Start;
    int token;
    while ((token = tokenizer.nextToken()) != StreamTokenizer.TT_EOF)
    {
      if (token == StreamTokenizer.TT_WORD || token == '*')
      {
        NodeMatcher matcher = null;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.