Examples of java.io.StreamTokenizer.resetSyntax()

java.io.StreamTokenizer.resetSyntax()
Resets this tokenizer's syntax table so that all characters are "ordinary." See the ordinaryChar method for more information on a character being ordinary. @see java.io.StreamTokenizer#ordinaryChar(int)

     * @param set the hash set
     * @param reader the reader
     */
    protected static void addWords(FullTextSettings setting, HashSet<String> set, Reader reader) {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        tokenizer.resetSyntax();
        tokenizer.wordChars(' ' + 1, 255);
        char[] whitespaceChars = setting.getWhitespaceChars().toCharArray();
        for (char ch : whitespaceChars) {
            tokenizer.whitespaceChars(ch, ch);
        }

View Full Code Here

     * similarly to that used by sun.security.provider.PolicyParser, except
     * that comments are disabled and no quote character is set (yet).
     */
    private static StreamTokenizer createTokenizer(String s) {
  StreamTokenizer st = new StreamTokenizer(new StringReader(s));
  st.resetSyntax();
  st.wordChars('a', 'z');
  st.wordChars('A', 'Z');
  st.wordChars('.', '.');
  st.wordChars('0', '9');
  st.wordChars('_', '_');

View Full Code Here


        private static String readUnicodeData(int ch, String filename) {
            String s = null;
            try {
                StreamTokenizer st = new StreamTokenizer(new FileReader(filename));
                st.resetSyntax();
                st.eolIsSignificant(true);
                for (int i=0; i<ch; ++i) {
                    s = st.toString();
                    st.nextToken(); // eol token
                    st.nextToken(); // next line

View Full Code Here

    // scientific notation (E or D); however, Double.valueOf does.
    // The strategy here is to disable StreamTokenizer's number parsing.
    // We'll only get whitespace delimited words, EOL's and EOF's.
    // These words should all be numbers, for Double.valueOf to parse.


    tokenizer.resetSyntax();
    tokenizer.wordChars(0,255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.eolIsSignificant(true);
    java.util.Vector v = new java.util.Vector();

View Full Code Here

            StringReader sr = new StringReader(cookieHeader);
            StreamTokenizer st = new StreamTokenizer(sr);
            Vector tokens = new Vector();


            // clear syntax tables of the StreamTokenizer
            st.resetSyntax();


            // set all characters as word characters
            st.wordChars(0,Character.MAX_VALUE);


            // set up characters for quoting

View Full Code Here

            if (sql.indexOf("?") == -1)
                return sql;


            paramOrder.clear();
            StreamTokenizer tok = new StreamTokenizer(new StringReader(sql));
            tok.resetSyntax();
            tok.quoteChar('\'');
            tok.wordChars('0', '9');
            tok.wordChars('?', '?');


            StringBuilder buf = new StringBuilder(sql.length());

View Full Code Here

        if (sql.indexOf("?1") == -1)
            return sql;


        List paramOrder = new ArrayList();
        StreamTokenizer tok = new StreamTokenizer(new StringReader(sql));
        tok.resetSyntax();
        tok.quoteChar('\'');
        tok.wordChars('0', '9');
        tok.wordChars('?', '?');


        StringBuffer buf = new StringBuffer(sql.length());

View Full Code Here

    return operations.build();
  }


  public static StreamTokenizer tokenize(String input) {
    StreamTokenizer result =  new StreamTokenizer(new StringReader(input));
    result.resetSyntax();
    result.wordChars('a', 'z');
    result.wordChars('A', 'Z');
    result.wordChars('0', '9');
    result.whitespaceChars(0, 32);
    result.quoteChar('"');

View Full Code Here

     * @param set the hash set
     * @param reader the reader
     */
    protected static void addWords(FullTextSettings setting, HashSet<String> set, Reader reader) {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        tokenizer.resetSyntax();
        tokenizer.wordChars(' ' + 1, 255);
        for (char ch : " \t\n\r\f+\"*%&/()=?'!,.;:-_#@|^~`{}[]".toCharArray()) {
            tokenizer.whitespaceChars(ch, ch);
        }
        try {

View Full Code Here

    }


    private static StreamTokenizer createTokenizer(Reader queryReader)
    {
        StreamTokenizer tokenizer = new StreamTokenizer(queryReader);
        tokenizer.resetSyntax();
        tokenizer.eolIsSignificant(true);
        tokenizer.parseNumbers();
        tokenizer.wordChars('a', 'z');
        tokenizer.wordChars('A', 'Z');
        tokenizer.wordChars('0', '9');

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.