Package cambridge.parser.tokens

Examples of cambridge.parser.tokens.Token

103
104
105
106
107
108
109
110
111
112
113
   private int getIndex(int no) {
      return no % BUFFER_SIZE;
   }

   private void fillBuffer() throws IOException {
      Token tok;
      while (tokenizer.hasMoreTokens()) {
         tok = tokenizer.nextToken();
         writeIndex++;
         buf[getIndex(writeIndex)] = tok;
View Full Code Here
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
                        TemplateTokenizer at = new TemplateTokenizer(new StringReader(currentToken.getValue()));

                        ArrayList<AttributeFragment> fragments = new ArrayList<AttributeFragment>();
                        while (at.hasMoreTokens()) {
                           Token attrToken = at.nextToken();
                           switch (attrToken.getType()) {
                              case EXPRESSION:
                                 ExpressionToken expTok = (ExpressionToken) attrToken;
                                 try {
                                    ExpressionNode expNode = new ExpressionNode(attrToken.value, expressionLanguage.parse(attrToken.value, attrToken.getLineNo(), attrToken.getColumn()), expTok.isRawExpression());

                                    if (expTok.getFilters() != null) {
                                       expNode.setFilters(expTok.getFilters());
                                    }
                                    fragments.add(expNode);
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
   @Test
   public void testScannerOutput() throws IOException {
      StringBuilder inputBuilder = new StringBuilder();
      StringBuilder outputBuilder = new StringBuilder();

      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
         if (token.value != null) {
            outputBuilder.append(token.getActualValue());
         }
      }

      InputStream fileIn = TokenizerTest.class.getResourceAsStream("input.html");
      InputStreamReader reader = new InputStreamReader(fileIn);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
      assertEquals(inputBuilder.toString(), outputBuilder.toString());
   }

   @Test
   public void test() throws IOException {
      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
      }
   }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
   public static void main(String[] args) {
      try {
         Tokenizer tokenizer = new TemplateTokenizer(TokenizerTest.class.getResourceAsStream("input.html"));

         Token token;
         while (tokenizer.hasMoreTokens()) {
            token = tokenizer.nextToken();
            System.out.println(token);
         }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
    private int getIndex(int no) {
        return no % BUFFER_SIZE;
    }

    private void fillBuffer() throws IOException {
        Token tok;
        while (tokenizer.hasMoreTokens()) {
            tok = tokenizer.nextToken();
            writeIndex++;
            buf[getIndex(writeIndex)] = tok;
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
                                TemplateTokenizer at = new TemplateTokenizer(new StringReader(currentToken.getValue()));

                                ArrayList<AttributeFragment> fragments = new ArrayList<AttributeFragment>();
                                while (at.hasMoreTokens()) {
                                    Token attrToken = at.nextToken();
                                    switch (attrToken.getType()) {
                                        case EXPRESSION:
                                            ExpressionToken expTok = (ExpressionToken) attrToken;
                                            try {
                                                ExpressionNode expNode = new ExpressionNode(attrToken.value, expressionLanguage.parse(attrToken.value, attrToken.getLineNo(), attrToken.getColumn()), expTok.isRawExpression());

                                                if (expTok.getFilters() != null) {
                                                    expNode.setFilters(expTok.getFilters());
                                                }
                                                fragments.add(expNode);
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
   @Test
   public void testScannerOutput() throws IOException {
      StringBuilder inputBuilder = new StringBuilder();
      StringBuilder outputBuilder = new StringBuilder();

      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
         if (token.value != null) {
            outputBuilder.append(token.getActualValue());
         }
      }

      InputStream fileIn = TokenizerTest.class.getResourceAsStream("input.html");
      InputStreamReader reader = new InputStreamReader(fileIn);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
      assertEquals(inputBuilder.toString(), outputBuilder.toString());
   }

   @Test
   public void test() throws IOException {
      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
      }
   }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
   public static void main(String[] args) {
      try {
         Tokenizer tokenizer = new TemplateTokenizer(TokenizerTest.class.getResourceAsStream("input.html"));

         Token token;
         while (tokenizer.hasMoreTokens()) {
            token = tokenizer.nextToken();
            System.out.println(token);
         }
View Full Code Here
TOP

Related Classes of cambridge.parser.tokens.Token

  • cambridge.parser.TemplateParser
  • cambridge.TokenizerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.