Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

      return Collections.emptyList();
    }
    try {
      remote.sessionBegin();
      InputStreamReader is = new InputStreamReader(remote.heads(), "US-ASCII");
      StreamTokenizer st = new StreamTokenizer(is);
      st.ordinaryChars('0', '9'); // wordChars performs |, hence need to 0 first
      st.wordChars('0', '9');
      st.eolIsSignificant(false);
      LinkedList<Nodeid> parseResult = new LinkedList<Nodeid>();
      while (st.nextToken() != StreamTokenizer.TT_EOF) {
        parseResult.add(Nodeid.fromAscii(st.sval));
      }
      return parseResult;
    } catch (IOException ex) {
      throw new HgRemoteConnectionException("Communication failure", ex).setRemoteCommand(CMD_HEADS).setServerInfo(getLocation());

View Full Code Here

    }
    LinkedHashMap<Range, List<Nodeid>> rv = new LinkedHashMap<HgRemoteRepository.Range, List<Nodeid>>(ranges.size() * 4 / 3);
    try {
      remote.sessionBegin();
      InputStreamReader is = new InputStreamReader(remote.between(ranges), "US-ASCII");
      StreamTokenizer st = new StreamTokenizer(is);
      st.ordinaryChars('0', '9');
      st.wordChars('0', '9');
      st.eolIsSignificant(true);
      Iterator<Range> rangeItr = ranges.iterator();
      LinkedList<Nodeid> currRangeList = null;
      Range currRange = null;
      boolean possiblyEmptyNextLine = true;
      while (st.nextToken() != StreamTokenizer.TT_EOF) {
        if (st.ttype == StreamTokenizer.TT_EOL) {
          if (possiblyEmptyNextLine) {
            // newline follows newline;
            assert currRange == null;
            assert currRangeList == null;

View Full Code Here

    static Map<String, String> parseHeader(String fullHeader) {
        
        Map<String, String> map = new HashMap<String, String>();
        fullHeader = fullHeader.substring(7);
        try {
            StreamTokenizer tok = new StreamTokenizer(new StringReader(fullHeader));
            tok.quoteChar('"');
            tok.quoteChar('\'');
            tok.whitespaceChars('=', '=');
            tok.whitespaceChars(',', ',');
            
            while (tok.nextToken() != StreamTokenizer.TT_EOF) {
                String key = tok.sval;
                if (tok.nextToken() == StreamTokenizer.TT_EOF) {
                    map.put(key, null);
                    return map;
                }
                String value = tok.sval;
                if (value.charAt(0) == '"') {

View Full Code Here

   */
  private void processIncludeFile()
    throws IOException, DRDAProtocolException
  {
    String fileName = getString();
    StreamTokenizer saveTkn = tkn;
    processFile(fileName);
    tkn = saveTkn;
  }

View Full Code Here

      // the location is different, try it
      String userdir =  System.getProperty("user.dir");
      String sep =  System.getProperty("file.separator");
      fr = new BufferedReader (new InputStreamReader(new FileInputStream(userdir + sep + ".." + sep + filename),"UTF-8"));
    }
    tkn = new StreamTokenizer(fr);
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF)
    {
      switch(val)
      {

View Full Code Here

            {
                _logger.debug("About to load ACL file " + file);
            }


            fileReader = new FileReader(file);
            _st = new StreamTokenizer(new BufferedReader(fileReader));
            _st.resetSyntax(); // setup the tokenizer


            _st.commentChar(COMMENT); // single line comments
            _st.eolIsSignificant(true); // return EOL as a token
            _st.ordinaryChar('='); // equals is a token

View Full Code Here

     * @param setting the fulltext settings
     * @param set the hash set
     * @param reader the reader
     */
    protected static void addWords(FullTextSettings setting, HashSet<String> set, Reader reader) {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        tokenizer.resetSyntax();
        tokenizer.wordChars(' ' + 1, 255);
        for (char ch : " \t\n\r\f+\"*%&/()=?'!,.;:-_#@|^~`{}[]".toCharArray()) {
            tokenizer.whitespaceChars(ch, ch);
        }
        try {
            while (true) {
                int token = tokenizer.nextToken();
                if (token == StreamTokenizer.TT_EOF) {
                    break;
                } else if (token == StreamTokenizer.TT_WORD) {
                    String word = tokenizer.sval;
                    word = setting.convertWord(word);

View Full Code Here

     * @param lineTimeout number of milliseconds to wait for each subsequent line
     * @return array of CLI output lines
     */
    public CLIOpResult readAllAsOpResult(long timeout, long lineTimeout) throws Exception {
        String output = readAllUnformated(timeout, lineTimeout);
        StreamTokenizer st = new StreamTokenizer(new StringReader(output));
        st.resetSyntax();
        st.whitespaceChars(' ', ' ');
        st.wordChars('#', '+');
        st.wordChars('-', 'Z');
        st.wordChars('a', 'z');
        st.quoteChar('"');


        int token = st.nextToken();
        if (token != '{') {
            throw new CLIException("Parse error. '{' expected, received: '" + token + "'.");
        }
        Map compound = parseCompound(st);
        CLIOpResult res = new CLIOpResult();

View Full Code Here

    }
  }




  private static Node parseModel(final Reader r) {
    final StreamTokenizer st = new StreamTokenizer(r);
    try {
      final Split root = new Split();
      parseSplit(st, root);
      return root.getChildren().get(0);
    } catch (final Exception e) {

View Full Code Here

            try {
                int[] buf = MultiRectAreaOp.createBuf(0);
                int count = 1;


                FileReader f = new FileReader(fileName);
                StreamTokenizer t = new StreamTokenizer(f);
                while(t.nextToken() != StreamTokenizer.TT_EOF) {
                    if (t.ttype == StreamTokenizer.TT_NUMBER) {
                        buf = MultiRectAreaOp.checkBufSize(buf, 1);
                        buf[count++] = (int)t.nval;
                    }
                }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.