Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

      // the location is different, try it
      String userdir =  System.getProperty("user.dir");
      String sep =  System.getProperty("file.separator");
      fr = new BufferedReader (new InputStreamReader(new FileInputStream(userdir + sep + filename),"UTF-8"));
    }
    tkn = new StreamTokenizer(fr);
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF)
    {
      switch(val)
      {

View Full Code Here

  public Algorithm (PerfRunData runData) throws Exception {
    String algTxt = runData.getConfig().getAlgorithmText();
    sequence = new TaskSequence(runData,null,null,false);
    TaskSequence currSequence = sequence;
    PerfTask prevTask = null;
    StreamTokenizer stok = new StreamTokenizer(new StringReader(algTxt));
    stok.commentChar('#');
    stok.eolIsSignificant(false);
    stok.ordinaryChar('"');
    stok.ordinaryChar('/');
    stok.ordinaryChar('(');
    stok.ordinaryChar(')');
    stok.ordinaryChar('-');
    boolean colonOk = false; 
    boolean isDisableCountNextTask = false; // only for primitive tasks
    currSequence.setDepth(0);
    String taskPackage = PerfTask.class.getPackage().getName() + ".";
    
    Class paramClass[] = {PerfRunData.class};
    Object paramObj[] = {runData};
    
    while (stok.nextToken() != StreamTokenizer.TT_EOF) { 
      switch(stok.ttype) {
  
        case StreamTokenizer.TT_WORD:
          String s = stok.sval;
          Constructor cnstr = Class.forName(taskPackage+s+"Task").getConstructor(paramClass);
          PerfTask task = (PerfTask) cnstr.newInstance(paramObj);
          task.setDisableCounting(isDisableCountNextTask);
          isDisableCountNextTask = false;
          currSequence.addTask(task);
          if (task instanceof RepSumByPrefTask) {
            stok.nextToken();
            String prefix = stok.sval;
            if (prefix==null || prefix.length()==0) { 
              throw new Exception("named report prefix problem - "+stok.toString()); 
            }
            ((RepSumByPrefTask) task).setPrefix(prefix);
          }
          // check for task param: '(' someParam ')'
          stok.nextToken();
          if (stok.ttype!='(') {
            stok.pushBack();
          } else {
            // get params, for tasks that supports them, - anything until next ')'
            StringBuffer params = new StringBuffer();
            stok.nextToken();
            while (stok.ttype!=')') { 
              switch (stok.ttype) {
                case StreamTokenizer.TT_NUMBER:  
                  params.append(stok.nval);
                  break;
                case StreamTokenizer.TT_WORD:    
                  params.append(stok.sval);             
                  break;
                case StreamTokenizer.TT_EOF:     
                  throw new Exception("unexpexted EOF: - "+stok.toString());
                default:
                  params.append((char)stok.ttype);
              }
              stok.nextToken();
            }
            String prm = params.toString().trim();
            if (prm.length()>0) {
              task.setParams(prm);
            }
          }


          // ---------------------------------------
          colonOk = false; prevTask = task;
          break;
  
        default:
          char c = (char)stok.ttype;
          
          switch(c) {
          
            case ':' :
              if (!colonOk) throw new Exception("colon unexpexted: - "+stok.toString());
              colonOk = false;
              // get repetitions number
              stok.nextToken();
              if ((char)stok.ttype == '*') {
                ((TaskSequence)prevTask).setRepetitions(TaskSequence.REPEAT_EXHAUST);
              } else {
                if (stok.ttype!=StreamTokenizer.TT_NUMBER)  {
                  throw new Exception("expected repetitions number or XXXs: - "+stok.toString());
                } else {
                  double num = stok.nval;
                  stok.nextToken();
                  if (stok.ttype == StreamTokenizer.TT_WORD && stok.sval.equals("s")) {
                    ((TaskSequence) prevTask).setRunTime(num);
                  } else {
                    stok.pushBack();
                    ((TaskSequence) prevTask).setRepetitions((int) num);
                  }
                }
              }
              // check for rate specification (ops/min)
              stok.nextToken();
              if (stok.ttype!=':') {
                stok.pushBack();
              } else {
                // get rate number
                stok.nextToken();
                if (stok.ttype!=StreamTokenizer.TT_NUMBER) throw new Exception("expected rate number: - "+stok.toString());
                // check for unit - min or sec, sec is default
                stok.nextToken();
                if (stok.ttype!='/') {
                  stok.pushBack();
                  ((TaskSequence)prevTask).setRate((int)stok.nval,false); // set rate per sec
                } else {
                  stok.nextToken();
                  if (stok.ttype!=StreamTokenizer.TT_WORD) throw new Exception("expected rate unit: 'min' or 'sec' - "+stok.toString());
                  String unit = stok.sval.toLowerCase();
                  if ("min".equals(unit)) {
                    ((TaskSequence)prevTask).setRate((int)stok.nval,true); // set rate per min
                  } else if ("sec".equals(unit)) {
                    ((TaskSequence)prevTask).setRate((int)stok.nval,false); // set rate per sec
                  } else {
                    throw new Exception("expected rate unit: 'min' or 'sec' - "+stok.toString());
                  }
                }
              }
              colonOk = false;
              break;
    
            case '{' : 
            case '[' :  
              // a sequence
              // check for sequence name
              String name = null;
              stok.nextToken();
              if (stok.ttype!='"') {
                stok.pushBack();
              } else {
                stok.nextToken();
                name = stok.sval;
                stok.nextToken();
                if (stok.ttype!='"' || name==null || name.length()==0) { 
                  throw new Exception("sequence name problem - "+stok.toString()); 
                }
              }
              // start the sequence
              TaskSequence seq2 = new TaskSequence(runData, name, currSequence, c=='[');
              currSequence.addTask(seq2);

View Full Code Here


  // Set up streamtokenizer
  BufferedReader reader =
      new BufferedReader(new InputStreamReader(is));


  st = new StreamTokenizer(reader);


  st.commentChar('#');
  st.eolIsSignificant(true);
  st.slashSlashComments(true);
  st.slashStarComments(true);

View Full Code Here

     * the last row is followed by a blank line.
   @param input the input stream.
   */


   public static Matrix read (BufferedReader input) throws java.io.IOException {
      StreamTokenizer tokenizer= new StreamTokenizer(input);


      // Although StreamTokenizer will parse numbers, it doesn't recognize
      // scientific notation (E or D); however, Double.valueOf does.
      // The strategy here is to disable StreamTokenizer's number parsing.
      // We'll only get whitespace delimited words, EOL's and EOF's.
      // These words should all be numbers, for Double.valueOf to parse.


      tokenizer.resetSyntax();
      tokenizer.wordChars(0,255);
      tokenizer.whitespaceChars(0, ' ');
      tokenizer.eolIsSignificant(true);
      java.util.Vector v = new java.util.Vector();


      // Ignore initial empty lines
      while (tokenizer.nextToken() == StreamTokenizer.TT_EOL);
      if (tokenizer.ttype == StreamTokenizer.TT_EOF)
  throw new java.io.IOException("Unexpected EOF on matrix read.");
      do {
         v.addElement(Double.valueOf(tokenizer.sval)); // Read & store 1st row.
      } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);


      int n = v.size();  // Now we've got the number of columns!
      double row[] = new double[n];
      for (int j=0; j<n; j++)  // extract the elements of the 1st row.
         row[j]=((Double)v.elementAt(j)).doubleValue();
      v.removeAllElements();
      v.addElement(row);  // Start storing rows instead of columns.
      while (tokenizer.nextToken() == StreamTokenizer.TT_WORD) {
         // While non-empty lines
         v.addElement(row = new double[n]);
         int j = 0;
         do {
            if (j >= n) throw new java.io.IOException
               ("Row " + v.size() + " is too long.");
            row[j++] = Double.valueOf(tokenizer.sval).doubleValue();
         } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
         if (j < n) throw new java.io.IOException
            ("Row " + v.size() + " is too short.");
      }
      int m = v.size();  // Now we've got the number of rows.
      double[][] A = new double[m][];

View Full Code Here

   *      string (see the OpenGIS Simple Features Specification)
   *@return                  a <code>Geometry</code> read from <code>reader</code>
   *@throws  ParseException  if a parsing problem occurs
   */
  public Geometry read(Reader reader) throws ParseException {
    tokenizer = new StreamTokenizer(reader);
    // set tokenizer to NOT parse numbers
    tokenizer.resetSyntax();
    tokenizer.wordChars('a', 'z');
    tokenizer.wordChars('A', 'Z');
    tokenizer.wordChars(128 + 32, 255);

View Full Code Here

        error("number of muds: " + nfe);
      }
      System.out.println("MudConnector: expecting " + mudCount + " mud entries");
      progress.setMax(mudCount);


      StreamTokenizer ts = new StreamTokenizer(r);
      ts.resetSyntax();
      ts.whitespaceChars(0, 9);
      ts.ordinaryChars(32, 255);
      ts.wordChars(32, 255);


      String name, host;
      Integer port;
      int token, counter = 0, idx = 0;


      while ((token = ts.nextToken()) != ts.TT_EOF) {
        name = ts.sval;


        if ((token = ts.nextToken()) != ts.TT_EOF) {
          if (token == ts.TT_EOL)
            error(name + ": unexpected end of line"
                  + ", missing host and port");
          host = ts.sval;
          port = new Integer(23);
          if ((token = ts.nextToken()) != ts.TT_EOF)
            try {
              if (token == ts.TT_EOL)
                error(name + ": default port 23");
              port = new Integer(ts.sval);
            } catch (NumberFormatException nfe) {
              error("port for " + name + ": " + nfe);
            }


          if (debug > 0)
            error(name + " [" + host + "," + port + "]");
          mudList.put(name, new Object[]{host, port, new Integer(idx++)});
          progress.adjust(++counter, name);
          mudListPanel.repaint();


    String key = (""+name.charAt(0)).toUpperCase();
          JMenu subMenu = (JMenu) menuList.get(key);
          if (subMenu == null) {
            subMenu = new JMenu(key);
            MCMenu.add(subMenu);
            menuList.put(key, subMenu);
          }
          JMenuItem item = new JMenuItem(name);
          item.addActionListener(MudConnector.this);
          subMenu.add(item);
        }
        while (token != ts.TT_EOF && token != ts.TT_EOL)
          token = ts.nextToken();
      }
      List list = new ArrayList(mudList.keySet());
      Collections.sort(list);
      mudListSelector.setListData(list.toArray());
      System.out.println("MudConnector: found " + mudList.size() + " entries");

View Full Code Here

        if (file == null) {
          ButtonBar.this.error("no setup file");
          return;
        }


        StreamTokenizer setup = null;
        InputStream is = null;


        try {
          is = getClass().getResourceAsStream(file);
        } catch (Exception e) {
          // ignore any errors here
        }


        // if the resource access fails, try URL
        if (is == null)
          try {
            is = new URL(file).openStream();
          } catch (Exception ue) {
            ButtonBar.this.error("could not find: " + file);
            return;
          }


        // create a new stream tokenizer to read the file
        try {
          InputStreamReader ir = new InputStreamReader(is);
          setup = new StreamTokenizer(new BufferedReader(ir));
        } catch (Exception e) {
          ButtonBar.this.error("cannot load " + file + ": " + e);
          return;
        }


        setup.commentChar('#');
        setup.quoteChar('"');


        fields = new HashMap();
        buttons = new HashMap();
        choices = new HashMap();


        GridBagLayout l = new GridBagLayout();
        GridBagConstraints c = new GridBagConstraints();
        panel.setLayout(l);
        c.fill = GridBagConstraints.BOTH;


        int token;
        int ChoiceCount = 0;
        JList list;
        // parse the setup file
        try {
          while ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
            switch (token) {
              case StreamTokenizer.TT_WORD:
                // reset the constraints
                c.gridwidth = 1;
                c.weightx = 0.0;
                c.weighty = 0.0;
                // keyword found, parse arguments
                if (setup.sval.equals("button")) {
                  if ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
                    String descr = setup.sval;
                    if ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
                      JButton b = new JButton(descr);
                      buttons.put(b, setup.sval);
                      b.addActionListener(ButtonBar.this);
                      l.setConstraints(b, constraints(c, setup));
                      panel.add(b);
                    } else
                      ButtonBar.this.error(descr + ": missing button command");
                  } else
                    ButtonBar.this.error("unexpected end of file");
                } else if (setup.sval.equals("label")) {
                  if ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
                    String descr = setup.sval;
                    JLabel b = new JLabel(descr);
                    l.setConstraints(b, constraints(c, setup));
                    panel.add(b);
                  } else
                    ButtonBar.this.error("unexpected end of file");
                  /* choice - new stuff added APS 07-dec-2001 for Choice
                   * buttons
                   * Choice info is held in the choices hash. There are two
                   * sorts of hash entry:
                   * 1) for each choices button on the terminal, key=choice
                   *    object, value=ID ("C1.", "C2.", etc)
                   * 2) for each item, key=ID+user's text (eg "C1.opt1"),
                   *    value=user's command
                   */


                } else if (setup.sval.equals("choice")) {
                  ChoiceCount++;
                  String ident = "C" + ChoiceCount + ".";
                  list = new JList();
                  choices.put(list, ident);
                  list.addListSelectionListener(ButtonBar.this);// Choices use ItemListener, not Action
                  l.setConstraints(list, constraints(c, setup));
                  panel.add(list);
                  while ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
                    if (isKeyword(setup.sval)) {// Got command ... Back off.
                      setup.pushBack();
                      break;
                    }
                    String descr = setup.sval;  // This is the hash key.
                    token = setup.nextToken();
                    if (token == StreamTokenizer.TT_EOF) {
                      ButtonBar.this.error("unexpected end of file");
                    } else {
                      String value = setup.sval;
                      if (isKeyword(value)) {   // Missing command - complain but continue
                        System.err.println(descr + ": missing choice command");
                        setup.pushBack();
                        break;
                      }
                      System.out.println("choice: name='" + descr + "', value='" + value);
                      list.add(descr, new JLabel(descr));
                      choices.put(ident + descr, value);
                    }
                  }
                  ButtonBar.this.error("choices hash: " + choices);
                } else if (setup.sval.equals("input")) {
                  if ((token = setup.nextToken()) != StreamTokenizer.TT_EOF) {
                    String descr = setup.sval;
                    if ((token = setup.nextToken()) ==
                            StreamTokenizer.TT_NUMBER) {
                      int size = (int) setup.nval;
                      String init = "", command = "";
                      token = setup.nextToken();
                      if (isKeyword(setup.sval))
                        setup.pushBack();
                      else
                        command = setup.sval;
                      token = setup.nextToken();
                      if (isKeyword(setup.sval)) {
                        setup.pushBack();
                        init = command;
                      } else
                        init = setup.sval;
                      JTextField t = new JTextField(init, size);
                      if (!init.equals(command)) {
                        buttons.put(t, command);
                        t.addActionListener(ButtonBar.this);
                      }
                      fields.put(descr, t);
                      l.setConstraints(t, constraints(c, setup));
                      panel.add(t);
                    } else
                      ButtonBar.this.error(descr + ": missing field size");
                  } else
                    ButtonBar.this.error("unexpected end of file");
                }
                break;
              default:
                ButtonBar.this.error("syntax error at line " + setup.lineno());
            }
          }
        } catch (IOException e) {
          ButtonBar.this.error("unexpected error while reading setup: " + e);
        }

View Full Code Here


  private Vector urlCache = new Vector();


  public void run() {
    try {
      StreamTokenizer st =
              new StreamTokenizer(new BufferedReader(new InputStreamReader(pin)));
      st.eolIsSignificant(true);
      st.slashSlashComments(false);
      st.slashStarComments(false);
      st.whitespaceChars(0, 31);
      st.ordinaryChar('"');
      st.ordinaryChar('<');
      st.ordinaryChar('>');
      st.ordinaryChar('/');
      st.ordinaryChar(':');


      int token;
      while ((token = st.nextToken()) != StreamTokenizer.TT_EOF) {
        if (token == StreamTokenizer.TT_WORD) {
          String word = st.sval.toLowerCase();


          // see if we have found a protocol
          if (protocols.contains(word)) {
            // check that the next chars are ":/"
            if (st.nextToken() == ':' && st.nextToken() == '/') {
              String url = word + ":/";
              // collect the test of the url
              while ((token = st.nextToken()) == StreamTokenizer.TT_WORD ||
                      token == '/')
                if (token == StreamTokenizer.TT_WORD)
                  url += st.sval;
                else
                  url += (char) token;

View Full Code Here

        // perform the parsing process
        if (sql.indexOf("?1") == -1)
            return sql;


        List paramOrder = new ArrayList();
        StreamTokenizer tok = new StreamTokenizer(new StringReader(sql));
        tok.resetSyntax();
        tok.quoteChar('\'');
        tok.wordChars('0', '9');
        tok.wordChars('?', '?');


        StringBuffer buf = new StringBuffer(sql.length());
        for (int ttype; (ttype = tok.nextToken()) != StreamTokenizer.TT_EOF;) {
            switch (ttype) {
                case StreamTokenizer.TT_WORD:
                    // a token is a positional parameter if it starts with
                    // a "?" and the rest of the token are all numbers
                    if (tok.sval.startsWith("?") && tok.sval.length() > 1 &&

View Full Code Here


    private void readFileListProperty(Utf8Properties props, String settingName, List<String> list) throws IOException {
        if (props.containsKey(settingName)) {
            list.clear();
            String s = props.getProperty(settingName);
            StreamTokenizer st = new StreamTokenizer(new StringReader(s.replaceAll("\\\\", "\\\\\\\\")));
            st.resetSyntax();
            st.wordChars('\u0000', '\uFFFF');
            st.whitespaceChars(',', ',');
            st.quoteChar('\"');
            st.quoteChar('\'');
            st.eolIsSignificant(false);
            while (st.nextToken() != StreamTokenizer.TT_EOF)
                list.add(st.sval);
        }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.