Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

    m_UpperBoundIsOpen = false;


    if (rangeString == null) return;


    // set up a tokenzier to parse the string
    StreamTokenizer tokenizer =
      new StreamTokenizer(new StringReader(rangeString));
    tokenizer.resetSyntax();         
    tokenizer.whitespaceChars(0, ' ');    
    tokenizer.wordChars(' '+1,'\u00FF');
    tokenizer.ordinaryChar('[');
    tokenizer.ordinaryChar('(');
    tokenizer.ordinaryChar(',');
    tokenizer.ordinaryChar(']');
    tokenizer.ordinaryChar(')');


    try {


      // get opening brace
      tokenizer.nextToken();
    
      if (tokenizer.ttype == '[') m_LowerBoundIsOpen = false;
      else if (tokenizer.ttype == '(') m_LowerBoundIsOpen = true;
      else throw new IllegalArgumentException("Expected opening brace on range,"
                + " found: "
                + tokenizer.toString());


      // get lower bound
      tokenizer.nextToken();
      if (tokenizer.ttype != tokenizer.TT_WORD)
  throw new IllegalArgumentException("Expected lower bound in range,"
             + " found: "
             + tokenizer.toString());
      if (tokenizer.sval.compareToIgnoreCase("-inf") == 0)
  m_LowerBound = Double.NEGATIVE_INFINITY;
      else if (tokenizer.sval.compareToIgnoreCase("+inf") == 0)
  m_LowerBound = Double.POSITIVE_INFINITY;
      else if (tokenizer.sval.compareToIgnoreCase("inf") == 0)
  m_LowerBound = Double.NEGATIVE_INFINITY;
      else try {
  m_LowerBound = Double.valueOf(tokenizer.sval).doubleValue();
      } catch (NumberFormatException e) {
  throw new IllegalArgumentException("Expected lower bound in range,"
             + " found: '" + tokenizer.sval + "'");
      }


      // get separating comma
      if (tokenizer.nextToken() != ',')
  throw new IllegalArgumentException("Expected comma in range,"
             + " found: "
             + tokenizer.toString());


      // get upper bound
      tokenizer.nextToken();
      if (tokenizer.ttype != tokenizer.TT_WORD)
  throw new IllegalArgumentException("Expected upper bound in range,"
             + " found: "
             + tokenizer.toString());
      if (tokenizer.sval.compareToIgnoreCase("-inf") == 0)
  m_UpperBound = Double.NEGATIVE_INFINITY;
      else if (tokenizer.sval.compareToIgnoreCase("+inf") == 0)
  m_UpperBound = Double.POSITIVE_INFINITY;
      else if (tokenizer.sval.compareToIgnoreCase("inf") == 0)
  m_UpperBound = Double.POSITIVE_INFINITY;
      else try {
  m_UpperBound = Double.valueOf(tokenizer.sval).doubleValue();
      } catch (NumberFormatException e) {
  throw new IllegalArgumentException("Expected upper bound in range,"
             + " found: '" + tokenizer.sval + "'");
      }


      // get closing brace
      tokenizer.nextToken();
    
      if (tokenizer.ttype == ']') m_UpperBoundIsOpen = false;
      else if (tokenizer.ttype == ')') m_UpperBoundIsOpen = true;
      else throw new IllegalArgumentException("Expected closing brace on range,"
                + " found: "
                + tokenizer.toString());


      // check for rubbish on end
      if (tokenizer.nextToken() != tokenizer.TT_EOF)
  throw new IllegalArgumentException("Expected end of range string,"
             + " found: "
             + tokenizer.toString());


    } catch (IOException e) {
      throw new IllegalArgumentException("IOException reading attribute range"
           + " string: " + e.getMessage());
    }

View Full Code Here

      throw new IOException("No source has been specified");
    }


    if (m_structure == null) {
      try {
  m_st = new StreamTokenizer(m_sourceReader);
  initTokenizer(m_st);
  readStructure(m_st);
      } catch (FileNotFoundException ex) {
      }
    }

View Full Code Here

    if (m_structure == null) {
      getStructure();
    }
    
    if (m_st == null) {
      m_st = new StreamTokenizer(m_sourceReader);
      initTokenizer(m_st);      
    }
        
    m_st.ordinaryChar(m_FieldSeparator.charAt(0));

View Full Code Here

     * @param reader    the reader to use
     * @throws IOException  if something goes wrong
     * @see      #getData()
     */
    public ArffReader(Reader reader) throws IOException {
      m_Tokenizer = new StreamTokenizer(reader);
      initTokenizer();


      readHeader(1000);
      initBuffers();

View Full Code Here

     */
    public ArffReader(Reader reader, int capacity) throws IOException {
      if (capacity < 0)
  throw new IllegalArgumentException("Capacity has to be positive!");


      m_Tokenizer = new StreamTokenizer(reader);
      initTokenizer();


      readHeader(capacity);
      initBuffers();
    }

View Full Code Here

     * @throws IOException  if something goes wrong
     * @see      #getData()
     */
    public ArffReader(Reader reader, Instances template, int lines, int capacity) throws IOException {
      m_Lines     = lines;
      m_Tokenizer = new StreamTokenizer(reader);
      initTokenizer();


      m_Data = new Instances(template, capacity);
      initBuffers();
    }

View Full Code Here

     * @param setting the fulltext settings
     * @param set the hash set
     * @param reader the reader
     */
    protected static void addWords(FullTextSettings setting, HashSet<String> set, Reader reader) {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        tokenizer.resetSyntax();
        tokenizer.wordChars(' ' + 1, 255);
        char[] whitespaceChars = setting.getWhitespaceChars().toCharArray();
        for (char ch : whitespaceChars) {
            tokenizer.whitespaceChars(ch, ch);
        }
        try {
            while (true) {
                int token = tokenizer.nextToken();
                if (token == StreamTokenizer.TT_EOF) {
                    break;
                } else if (token == StreamTokenizer.TT_WORD) {
                    String word = tokenizer.sval;
                    word = setting.convertWord(word);

View Full Code Here

      return null;
    }


    try
    {
      final StreamTokenizer strtok = new StreamTokenizer(new StringReader(value));
      strtok.parseNumbers();
      final int firstToken = strtok.nextToken();
      if (firstToken != StreamTokenizer.TT_NUMBER)
      {
        return null;
      }
      final double nval = strtok.nval;
      final int nextToken = strtok.nextToken();
      if (nextToken != StreamTokenizer.TT_WORD)
      {
        // yeah, this is against the standard, but we are dealing with deadly ugly non-standard documents here
        // maybe we will be able to integrate a real HTML processor at some point. 
        return new Float(nval);

View Full Code Here


    /**
     * Sets up the stream tokenizer
     */
    private void setup() {
        st = new StreamTokenizer(this);
        st.resetSyntax();
        st.eolIsSignificant(false);
        st.lowerCaseMode(true);


        // Parse numbers as words

View Full Code Here

  // openFile does the real work of opening the MIX file.
  // It is a helper for use by both constructors.
  
    private void openFile(File file) {
        try {
            st = new StreamTokenizer(           
                fr = new FileReader(file));
            st.eolIsSignificant(true);
            st.parseNumbers();
            st.whitespaceChars(' ', ' ');
            st.whitespaceChars('\t', '\t');

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.