Package cambridge

Source Code of cambridge.TokenizerTest

package cambridge;

import cambridge.parser.TemplateTokenizer;
import cambridge.parser.Tokenizer;
import cambridge.parser.tokens.Token;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

/**
* @author Erdinc Yilmazel
* Date: Oct 13, 2009
* Time: 10:51:25 AM
*/
public class TokenizerTest {
   private Tokenizer tokenizer;

   @Before
   public void setUp() {
      try {
         tokenizer = new TemplateTokenizer(new BufferedReader(new InputStreamReader(TokenizerTest.class.getResourceAsStream("input.html"))));
      } catch (IOException e) {
         e.printStackTrace();
      }
   }

   @Test
   public void testScannerOutput() throws IOException {
      StringBuilder inputBuilder = new StringBuilder();
      StringBuilder outputBuilder = new StringBuilder();

      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
         if (token.value != null) {
            outputBuilder.append(token.getActualValue());
         }
      }

      InputStream fileIn = TokenizerTest.class.getResourceAsStream("input.html");
      InputStreamReader reader = new InputStreamReader(fileIn);
      int i;
      while ((i = reader.read()) != -1) {
         inputBuilder.append((char) i);
      }

      fileIn.close();

      assertEquals(inputBuilder.toString(), outputBuilder.toString());
   }

   @Test
   public void test() throws IOException {
      Token token;
      while (tokenizer.hasMoreTokens()) {
         token = tokenizer.nextToken();
         assertNotNull(token);
      }
   }

   @After
   public void close() {
      try {
         tokenizer.close();
      } catch (IOException e) {
         e.printStackTrace();
      }
   }

   public static void main(String[] args) {
      try {
         Tokenizer tokenizer = new TemplateTokenizer(TokenizerTest.class.getResourceAsStream("input.html"));

         Token token;
         while (tokenizer.hasMoreTokens()) {
            token = tokenizer.nextToken();
            System.out.println(token);
         }

         tokenizer.close();
      } catch (IOException e) {
         e.printStackTrace();
      }
   }
}
TOP

Related Classes of cambridge.TokenizerTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.