Examples of com.googlecode.psiprobe.tokenizer.Tokenizer

    public static String getJSPEncoding(InputStream is) throws IOException {


        String encoding = null;
        String contentType = null;


        Tokenizer jspTokenizer = new Tokenizer();
        jspTokenizer.addSymbol("\n", true);
        jspTokenizer.addSymbol(" ", true);
        jspTokenizer.addSymbol("\t", true);
        jspTokenizer.addSymbol(new TokenizerSymbol("dir", "<%@", "%>", false, false, true, false));


        StringTokenizer directiveTokenizer = new StringTokenizer();
        directiveTokenizer.addSymbol("\n", true);
        directiveTokenizer.addSymbol(" ", true);
        directiveTokenizer.addSymbol("\t", true);
        directiveTokenizer.addSymbol("=");
        directiveTokenizer.addSymbol("\"", "\"", false);
        directiveTokenizer.addSymbol("'", "'", false);


        StringTokenizer contentTypeTokenizer = new StringTokenizer();
        contentTypeTokenizer.addSymbol(" ", true);
        contentTypeTokenizer.addSymbol(";", true);




        Reader reader = new InputStreamReader(is, "ISO-8859-1");
        try {
            jspTokenizer.setReader(reader);
            while (jspTokenizer.hasMore()) {
                Token token = jspTokenizer.nextToken();
                if ("dir".equals(token.getName())) {
                    directiveTokenizer.setString(token.getInnerText());
                    if (directiveTokenizer.hasMore() && directiveTokenizer.nextToken().getText().equals("page")) {
                        while (directiveTokenizer.hasMore()) {
                            Token dTk = directiveTokenizer.nextToken();

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            jspRenderer.highlight(name, input, bos, encoding, true);


            ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());


            Tokenizer tokenizer = new Tokenizer(new InputStreamReader(bis, encoding));
            tokenizer.addSymbol(new TokenizerSymbol("EOL", "\n", null, false, false, true, false));
            tokenizer.addSymbol(new TokenizerSymbol("EOL", "\r\n", null, false, false, true, false));


            //
            // JHighlight adds HTML comment as the first line, so if
            // we number the lines we could end up with a line number and no line
            // to avoid that we just ignore the first line alltogether.
            //
            StringBuffer buffer = new StringBuffer();
            long counter = 0;
            while (tokenizer.hasMore()) {
                Token tk = tokenizer.nextToken();
                if ("EOL".equals(tk.getName())) {
                    counter++;
                    buffer.append(tk.getText());
                } else if (counter > 0) {
                    buffer.append("<span class=\"codeline\">");

Examples of com.googlecode.psiprobe.tokenizer.Tokenizer

Related Classes of com.googlecode.psiprobe.tokenizer.Tokenizer