Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

   */
  private static Map<String, Appearance> parseMaterialStream(Reader reader, 
                                                             URL baseUrl) throws IOException {
    Map<String, Appearance> appearances = new HashMap<String, Appearance>();
    Appearance              currentAppearance = null;    
    StreamTokenizer tokenizer = createTokenizer(reader);
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
      switch (tokenizer.ttype) {
        case StreamTokenizer.TT_WORD :
          currentAppearance = parseMaterialLine(tokenizer, 
              appearances, currentAppearance, baseUrl);
          break;
        case StreamTokenizer.TT_EOL:
          break;
        default :
          throw new IncorrectFormatException("Unexpected token " + tokenizer.sval 
              + " at row " + tokenizer.lineno());
      }
    }
    return appearances;
  }

View Full Code Here

     * @param in
     *            Input reader
     * @return Configured StreamTokenizer
     */
    static StreamTokenizer setupTokenizer(Reader in) {
        StreamTokenizer st = new StreamTokenizer(in);
        st.resetSyntax();
        st.whitespaceChars(0, ' ');
        // '!' token
        st.quoteChar('"');
        st.commentChar('#');
        // '$' token
        st.wordChars('%', '%');
        // '&' token
        st.quoteChar('\'');
        st.wordChars('(', ':');
        // '<' token
        st.wordChars('=', '=');
        // '>' token
        st.wordChars('?', '{');
        // '|' token
        st.wordChars('}', 255);
        st.eolIsSignificant(true);
        return st;
    }

View Full Code Here

    this.currentGroup = new Group("default");
    this.groups.put("default", this.currentGroup);
    this.currentMaterial = "default";
    this.appearances = new HashMap<String, Appearance>(DEFAULT_APPEARANCES);


    StreamTokenizer tokenizer = createTokenizer(reader);
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
      switch (tokenizer.ttype) {
        case StreamTokenizer.TT_WORD :
          parseObjectLine(tokenizer, baseUrl);
          break;
        case StreamTokenizer.TT_EOL:
          break;
        default :
          throw new IncorrectFormatException("Unexpected token " + tokenizer.sval 
              + " at row " + tokenizer.lineno());
      }
    }
    
    try {
      return createScene();

View Full Code Here

  
  /**
   * Returns a new tokenizer for an OBJ or MTL stream.
   */
  private static StreamTokenizer createTokenizer(Reader reader) {
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.eolIsSignificant(true);


    // All printable ASCII characters
    tokenizer.wordChars('!', '~');
    // Let's tolerate other ISO-8859-1 characters
    tokenizer.wordChars(0x80, 0xFF);


    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars('\n', '\n');
    tokenizer.whitespaceChars('\r', '\r');
    tokenizer.whitespaceChars('\t', '\t');
    return tokenizer;
  }

View Full Code Here

   *          is used.
   * @param classLoader
   *          The class loader used to instantiate the Java objects.
   */
  public JsonParser(Reader reader, Class rootType, ClassLoader classLoader) {
    m_tokenizer = new StreamTokenizer(reader);


    setSyntax();


    _rootType = rootType;
    _classLoader = classLoader;

View Full Code Here

   * @param classLoader
   *          The class loader used to instantiate the Java objects.
   */
  public JsonParser(Reader reader, Class rootType, Map propertyMap,
      ClassLoader classLoader) {
    m_tokenizer = new StreamTokenizer(reader);


    setSyntax();


    _rootType = rootType;
    _propertyMap = propertyMap;

View Full Code Here


    private void importData(LineNumberReader reader) throws Exception {
        Progress.start(progressTicket);


        ArrayList list;
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        tokenizer.ordinaryChar('[');
        tokenizer.ordinaryChar(']');
        tokenizer.wordChars('_', '_');
        list = parseList(tokenizer);


        boolean ret = false;
        for (int i = 0; i < list.size(); i++) {
            if ("graph".equals(list.get(i)) && list.size() >= i + 2 && list.get(i + 1) instanceof ArrayList) {

View Full Code Here


    private void importData(LineNumberReader reader) throws Exception {


        Progress.start(progressTicket);
        initColorTable();
        StreamTokenizer streamTokenizer = new StreamTokenizer(reader);
        setSyntax(streamTokenizer);


        graph(streamTokenizer);
    }

View Full Code Here

                // try and get the characterEncoding from the reader
                this.characterEncoding = "US-ASCII";
            }
        }


        tokenizer = new StreamTokenizer(new BufferedReader(in));
        initTokenizer(tokenizer);
    }

View Full Code Here

   */
  private void processIncludeFile()
    throws IOException, DRDAProtocolException
  {
    String fileName = getString();
    StreamTokenizer saveTkn = tkn;
    processFile(fileName);
    tkn = saveTkn;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.