Package webuicheck

Source Code of webuicheck.Crawler

package webuicheck;

import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.event.KeyEvent;
import java.awt.image.BufferedImage;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Random;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.imageio.ImageIO;

import org.openqa.selenium.server.RemoteControlConfiguration;
import org.openqa.selenium.server.SeleniumServer;

import com.thoughtworks.selenium.DefaultSelenium;
import com.thoughtworks.selenium.SeleneseTestCase;
import com.thoughtworks.selenium.SeleniumException;

public class Crawler extends SeleneseTestCase {

  private int TAKESCREENSHOTS = 0;
  private String platform = "";
  String mainURL = "";
  // links that must be skipped
  String logoutURL = "modify-this-to-something-else";

  public String[] visitedLinks = new String[500];
  int visitedLinksNumber = 0;
  public String password = "";
  public String validField = "text, password, select-one, radio, checkbox, file, submit";
  public String loginUser = "[giveemailaddresshere]";
  public String loginPass = "yourpassword";
  public BufferedWriter outFile;
  public String cheatFile = "cheats.csv";
  public String outputFile = "links.csv";
  public String outputMap = "";
  private String outputUIDir = "";
  private PrintWriter pwPageLoad;
  private PrintWriter pwPages;
  private String[] errors = { "error", "Error", "Application Error", };
  public FileWriter fw;
  private String baseURL;
  private String[] excludeURLs;
  private String[] excludeElements;
  private String[] XLinks; // when saving links, exclude part of it
                // (parameters, etc)

  /*
   * Added selServer, selClient variables to enable Selenium Server and Client
   * to be used dinamically (start automatically) Author Tudor Enache
   */
  private SeleniumServer selServer;
  private DefaultSelenium selClient;
  private int serverPort = 4444;
  private String browser = "*safari";

  Random r = new Random();

  public Crawler(String loginUser, String loginPass, int serverPort,
      String browser, String savePath, String platformURL,
      String platform, int takeScreenshots, String baseURL,
      String[] excludeURLs) {

    this.platform = platform;
    mainURL = platformURL;

    // logoutURL = mainURL + "/?logout=1"; IF NEEDED uncomment
    this.loginUser = loginUser;
    this.loginPass = loginPass;
    this.serverPort = serverPort;
    this.browser = browser;
    // savePath.replace('\\', '/');
    savePath += System.getProperty("file.separator");
    this.outputUIDir = savePath;
    this.TAKESCREENSHOTS = takeScreenshots;

    this.baseURL = baseURL;

    // separate URLs from Elements
    ArrayList<String> xTempURL = new ArrayList<String>();
    ArrayList<String> xTempElem = new ArrayList<String>();
    ArrayList<String> xTempXLinks = new ArrayList<String>();
    for (int k = 0; k < excludeURLs.length; k++) {
      if (excludeURLs[k].indexOf("elem.") >= 0) {
        xTempElem.add(excludeURLs[k].substring(5));
      } else if (excludeURLs[k].indexOf("url.") >= 0) {
        xTempURL.add(excludeURLs[k].substring(4));
      } else if (excludeURLs[k].indexOf("xlink.") >= 0) {
        xTempXLinks.add(excludeURLs[k].substring(6));
      }
    }
    this.excludeElements = new String[xTempElem.size()];
    this.excludeURLs = new String[xTempURL.size()];
    this.XLinks = new String[xTempXLinks.size()];
    xTempURL.toArray(this.excludeURLs);
    xTempElem.toArray(this.excludeElements);
    xTempXLinks.toArray(this.XLinks);
  }

  /*
   * Modified by Tudor Enache
   */
  @Override
  public void setUp() {
    try {
      // Creates custom configuration for the selenium server
      // Tudor Enache
      RemoteControlConfiguration remConCon = new RemoteControlConfiguration();
      if (browser.contains("iexploreproxy")) {
        remConCon.setSingleWindow(true);
      }
      remConCon.setPort(serverPort);
      remConCon.trustAllSSLCertificates();
      remConCon.setAvoidProxy(true);

      // Instantiates selenium server with custom control configuration
      // Tudor Enache
      selServer = new SeleniumServer(remConCon);
      // Instantiates new client with parameters
      // Tudor Enache
      selClient = new DefaultSelenium("localhost", serverPort, browser,
          mainURL);
      // Starts selServer + selClient
      // Tudor Enache
      selServer.start();
      selClient.start();
      selClient.windowMaximize();

    } catch (Exception e) {
      e.printStackTrace();
    }

  }

  /*
   * Added to shutdown selServer Author Tudor Enache
   */
  @Override
  public void tearDown() throws Exception {

    pwPageLoad.close();
    pwPages.close();
    selClient.stop();
    selServer.stop();
    Thread.currentThread().interrupt();

    Exception e = new Exception("Game over");
    throw e;
  }

  /*
   * returns the number of forms on current page getFormsNumber() Author -
   * Ciprian-Alex Baltatescu
   */
  public int getFormsNumber() {
    String jsFormsNumber = "function getFormsNumber(){"
        + "var node_list=window.document.getElementsByTagName('form'); "
        + "return node_list.length;" + "}" + "getFormsNumber();";
    int formsNumber;
    formsNumber = Integer.parseInt(selClient.getEval(jsFormsNumber));
    return formsNumber;
  }

  /*
   * returns a form length on the current page (the number of elements
   * included in a form) getFormLength(i) i represents the index of the form
   * and can be between 0 and getFormsNumber()-1 Author - Ciprian-Alex
   * Baltatescu
   */
  public int getFormLength(int i) {
    String jsFormLength = "function getFormLength(){"
        + "return window.document.forms[" + i + "].length;" + "}"
        + "getFormLength();";
    int elementsNumber;
    elementsNumber = Integer.parseInt(selClient.getEval(jsFormLength));
    return elementsNumber;
  }

  /*
   * returns a form id on the current page getFormLength(i) i represents the
   * index of the form and can be between 0 and getFormsNumber()-1 Author -
   * Ciprian-Alex Baltatescu
   */
  public String getFormId(int i) {
    String jsFormId = "function getFormId(){"
        + "return window.document.forms[" + i + "].id;" + "}"
        + "getFormId();";
    String formId;
    formId = selClient.getEval(jsFormId);
    return formId;
  }

  /*
   * returns a form id on the current page getFormLength(i) i represents the
   * index of the form and can be between 0 and getFormsNumber()-1 Author -
   * Ciprian-Alex Baltatescu
   */
  public String getFormName(int i) {
    String jsFormName = "function getFormName(){"
        + "return window.document.forms[" + i + "].name;" + "}"
        + "getFormName();";
    String formName;
    formName = selClient.getEval(jsFormName);
    return formName;
  }

  /*
   * returns the type of a field included in a form on the current page
   * getFormElementType(i, j) i represents the index of the form and can be
   * between 0 and getFormsNumber()-1 j represents the index of the field and
   * can be between 0 and getFormLength()-1 Author - Ciprian-Alex Baltatescu
   */
  public String getFormElementType(int i, int j) {
    String jsFormElementType = "function getFormElementType(){"
        + "return window.document.forms[" + i + "].elements[" + j
        + "].type;" + "}" + "getFormElementType();";
    String ElementType = selClient.getEval(jsFormElementType);
    return ElementType;
  }

  /*
   * returns the id of a field included in a form on the current page
   * getFormElementId(i, j) i represents the index of the form and can be
   * between 0 and getFormsNumber()-1 j represents the index of the field and
   * can be between 0 and getFormLength()-1 Author - Ciprian-Alex Baltatescu
   */
  public String getFormElementId(int i, int j) {
    String jsFormElementId = "function getFormElementId(){"
        + "return window.document.forms[" + i + "].elements[" + j
        + "].id;" + "}" + "getFormElementId();";
    String ElementId = selClient.getEval(jsFormElementId);
    return ElementId;
  }

  /*
   * returns the name of a field included in a form on the current page
   * getFormElementName(i, j) i represents the index of the form and can be
   * between 0 and getFormsNumber()-1 j represents the index of the field and
   * can be between 0 and getFormLength()-1 Author - Ciprian-Alex Baltatescu
   */
  public String getFormElementName(int i, int j) {
    String jsFormElementName = "function getFormElementName(){"
        + "return window.document.forms[" + i + "].elements[" + j
        + "].name;" + "}" + "getFormElementName();";
    String ElementName = selClient.getEval(jsFormElementName);
    return ElementName;
  }

  /*
   * verifies if a element is disabled or not return TRUE (not disabled) /
   * FALSE (disabled) getFormElementNotDisable(i, j) i represents the index of
   * the form and can be between 0 and getFormsNumber()-1 j represents the
   * index of the field and can be between 0 and getFormLength()-1 Author -
   * Ciprian-Alex Baltatescu
   */
  public boolean getFormElementNotDisabled(int i, int j) {
    String jsFormElementNotDisabled = "function getFormElementNotDisabled(){"
        + "return window.document.forms["
        + i
        + "].elements["
        + j
        + "].disabled;" + "}" + "getFormElementNotDisabled();";
    if (selClient.getEval(jsFormElementNotDisabled).equals("true")) {
      return false;
    } else {
      return true;
    }
  }

  /*
   * verifies if a element is read-only or not return TRUE (not read-only) /
   * FALSE (read-only) getFormElementNotReadonly(i, j) i represents the index
   * of the form and can be between 0 and getFormsNumber()-1 j represents the
   * index of the field and can be between 0 and getFormLength()-1 Author -
   * Ciprian-Alex Baltatescu
   */
  public boolean getFormElementNotReadonly(int i, int j) {
    String jsFormElementReadonly = "function getFormElementReadonly(){"
        + "return window.document.forms[" + i + "].elements[" + j
        + "].readOnly;" + "}" + "getFormElementReadonly();";
    if (selClient.getEval(jsFormElementReadonly).equals("true")) {
      return false;
    } else {
      return true;
    }
  }

  /*
   * verifies if a element is displayed or not return TRUE / FALSE
   * getFormElementStyle(i, j) i represents the index of the form and can be
   * between 0 and getFormsNumber()-1 j represents the index of the field and
   * can be between 0 and getFormLength()-1 Author - Ciprian-Alex Baltatescu
   */
  public boolean getFormElementStyle(int i, int j) {
    String jsFormElementStyle = "function getFormElementStyle(){"
        + "return window.document.forms[" + i + "].elements[" + j
        + "].style.display;" + "}" + "getFormElementStyle();";
    if (selClient.getEval(jsFormElementStyle).equals("none")) {
      return false;
    } else {
      return true;
    }
  }

  /*
   * verifies if a field included in a form on the current page can be
   * completed, checked, selected or submitted returns TRUE/FALSE
   * getFormElementIsValid(i, j) i represents the index of the form and can be
   * between 0 and getFormsNumber()-1 j represents the index of the field and
   * can be between 0 and getFormLength()-1 Author - Ciprian-Alex Baltatescu
   */
  public boolean getFormElementIsValid(int i, int j) {
    if (validField.indexOf(getFormElementType(i, j)) > -1) {
      return true;
    } else {
      return false;
    }
  }

  /*
   * verifies if a field included in a form on the current page is displayed,
   * not read-only, not disabled returns TRUE/FALSE
   * getFormElementAccessible(i, j) i represents the index of the form and can
   * be between 0 and getFormsNumber()-1 j represents the index of the field
   * and can be between 0 and getFormLength()-1 Author - Ciprian-Alex
   * Baltatescu
   */
  public boolean getFormElementAccessible(int i, int j) {
    if (getFormElementStyle(i, j) && getFormElementNotReadonly(i, j)
        && getFormElementNotDisabled(i, j)) {
      return true;
    } else {
      return false;
    }
  }

  /*
   * returns the number of links on current page getLinksNumber() Author -
   * Ciprian-Alex Baltatescu
   */
  public int getLinksNumber() {
    String jsLinksNumber = "function getLinksNumber(){"
        + "return node_list=window.document.links!=null?window.document.links.length:1; "
        + "}" + "getLinksNumber();";
    int linksNumber;
    linksNumber = Integer.parseInt(selClient.getEval(jsLinksNumber));
    return linksNumber;
  }

  /*
   * returns the URL (href) of a link on current page getLinkHref(i) i
   * represents the index of the link and can be between 0 and
   * getLinksNumber()-1 Author - Ciprian-Alex Baltatescu
   */
  public String getLinkHref(int i) {
    String name = "";
    try {
      String jsLinkHref = "function getLinkHref(){"
          + "return window.document.links[" + i + "].href;" + "}"
          + "getLinkHref();";
      name = selClient.getEval(jsLinkHref);
    } catch (Exception e) {
      System.out
          .println("OOPS.. something bad with my Javascript happend");
      e.printStackTrace();
      this.takeScreenShots("Error-getLinkHref"
          + truncLink(selClient.getLocation()));
    }
    return name;
  }

  /*
   * returns the text of a link on current page getLinkText(i) i represents
   * the index of the link and can be between 0 and getLinksNumber()-1 It's
   * not used yet; maybe it will be deleted Author - Ciprian-Alex Baltatescu
   */
  public String getLinkText(int i) {
    String jsLinkText = "function getLinkText(){"
        + "return window.document.links[" + i + "].innerText;" + "}"
        + "getLinkText();";
    String name = selClient.getEval(jsLinkText);
    return name;
  }

  /*
   * verifies if a link is valid (same domain, no downloads, no sorting, no
   * exports) returns TRUE/FALSE getLinkIsValid(i) i represents the index of
   * the link and can be between 0 and getLinksNumber()-1 Author -
   * Ciprian-Alex Baltatescu Mihai Barcun. Weg mit beliebige links
   */
  public boolean getLinkIsValid(int i) {
    boolean isValid = false;
    try {
      String jsLinkIsValid = "function getIsValid(){"
          + "if (((window.document.links["
          + i
          + "].href).indexOf('"
          + baseURL
          + "') > -1) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "list_sort_property_"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "sortby="
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "PDF"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "#"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "CSV"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "pdf"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "download="
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "_xmlonly=1"
          + "') < 0) &&"
          + "getIsHeaderOrFooterLink(window.document.links["
          + i
          + "]) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "_editmode=admin"
          + "') < 0) &&"
          + "((window.document.links["
          + i
          + "].href).indexOf('"
          + "EXPORT_"
          + "') < 0)) {"
          + "return 'true';"
          + "}"
          + "else {"
          + "return 'false';"
          + "}"
          + "}"
          + "function getIsHeaderOrFooterLink(myLinkObj){" // eliminate
                                    // any
                                    // unwished
                                    // elements
                                    // on
                                    // the
                                    // pages,
                                    // which
                                    // occur
                                    // repeatedly
                                    // and
                                    // are
                                    // not
                                    // required
                                    // to be
                                    // checked
          + "var ok = false; " + "return (";
      if (excludeElements.length != 0) {
        for (int k = 0; k < excludeElements.length - 1; k++) {
          jsLinkIsValid += " isChildOf(myLinkObj, \""
              + excludeElements[k] + "\") == false && ";
        }
        jsLinkIsValid += "isChildOf(myLinkObj, \""
            + excludeElements[excludeElements.length - 1]
            + "\") == false);";
      } else {
        jsLinkIsValid += " 'true');";
      }

      jsLinkIsValid += "} "
          + "function isChildOf(ChildObject,ContainerObject)"
          + "{"
          + "     var curobj;"
          + "     if(typeof(ContainerObject)==\"string\")"
          + "     {        ContainerObject=window.document.getElementById(ContainerObject);    }"
          + "     if(typeof(ChildObject)==\"string\")"
          + "     {        ChildObject=window.document.getElementById(ChildObject);    }"
          + "     for(curobj=ChildObject.parentNode; ( (curobj!=undefined) && (curobj!=window.document.body) &&(curobj.id!=ContainerObject.id) );curobj=curobj.parentNode)"
          + "     {    }"
          + "     return (curobj.id==ContainerObject.id);" + " }"
          + "getIsValid();";
      isValid = Boolean.parseBoolean(selClient.getEval(jsLinkIsValid));
    } catch (Exception e) {
      System.out.println("Error when trying to validate link");
      e.printStackTrace();
      this.takeScreenShots("Error-getLinkIsValid"
          + truncLink(selClient.getLocation()));
    }
    return isValid;
  }

  /*
   * verifies if a link is not LogOut returns TRUE/FALSE getLinkNotLogout(i) i
   * represents the index of the link and can be between 0 and
   * getLinksNumber()-1 Author - Ciprian-Alex Baltatescu
   */
  public boolean getLinkNotLogout(int i) {

    String jsLinkNotLogout = "function getNotLogout(){"
        + "if ((window.document.links[" + i + "].href).indexOf('"
        + logoutURL + "') < 0) {" + "return 'true';" + "}" + "else {"
        + "return 'false';" + "}" + "}" + "getNotLogout();";
    boolean notLogout = Boolean.parseBoolean(selClient
        .getEval(jsLinkNotLogout));
    return notLogout;
  }

  /*
   * verifies that the link does not contain the strings specified in exclude
   * list
   */
  public boolean getLinkNotInExcludeList(int i) {
    boolean NotInExcludeList = true;

    for (int k = 0; k < excludeURLs.length; k++) {
      String jsLinkNotIN = "function getNotIN(){"
          + "if ((window.document.links[" + i + "].href).indexOf('"
          + excludeURLs[k] + "') < 0) {" + "return 'true';" + "}"
          + "else {" + "return 'false';" + "}" + "}" + "getNotIN();";
      NotInExcludeList = Boolean.parseBoolean(selClient
          .getEval(jsLinkNotIN));
      if (NotInExcludeList == false) {
        break;
      }
    }
    return NotInExcludeList;
  }

  /*
   * verifies if a link wasn't visited returns TRUE/FALSE getLinkNotVisited(i)
   * i represents the index of the link and can be between 0 and
   * getLinksNumber()-1 Author - Ciprian-Alex Baltatescu
   */
  public boolean getLinkNotVisited(String link) {
    boolean notVisited = true;
    for (int i = 0; i < visitedLinks.length; i++) {
      if (link.toLowerCase().equals(visitedLinks[i])) {
        notVisited = false;
        break;
      }
    }
    return notVisited;
  }

  /*
   * returns the current opened link, the URL address.
   */
  public String getCurrentURL() {
    /*
     * String jsCurrentURL = "function getURL(){" +
     * "var currentUrl = window.document.location;" +
     * "return currentUrl.toString();" + "}" + "getURL();"; String jsStr =
     * selClient.getEval(jsCurrentURL);
     */
    return selClient.getLocation();
  }

  /*
   * accesses a link openLink(link) link represents the link to be accessed -
   * Ciprian-Alex Baltatescu cut parts of the links that are unnecessary for
   * output
   */
  public void openLink(String link) {
    try {
      System.out.println("........Opening link");
      String a = link;
      if ((XLinks != null) && (XLinks.length >= 1)) {
        a = (link.split(XLinks[0]))[0];
      }
      Date t0 = new Date();
      selClient.open(link);
      selClient.waitForPageToLoad("100000");
      Date t1 = new Date();
      long timp = (t1.getTime() - t0.getTime());
      double secunde = (double) timp / 1000;
      pwPages.write(a + "\t");
      pwPages.write(String.valueOf(secunde) + "\n");
      pwPages.flush();
      pwPageLoad.write(String.valueOf(secunde) + "\n");
      pwPageLoad.flush();

      if (TAKESCREENSHOTS == 1) {
        takeScreenShots(link);
      }
      // checkForErrors(link);

    } catch (SeleniumException e) {
      e.printStackTrace();
      this.takeScreenShots("Error-openLink-"
          + truncLink(selClient.getLocation()));
    }
  }

  public void takeScreenShots(String link) {

    String a[] = link.split(";jsessionid");
    link = a[0];
    String tempLink = link.replace('/', '-');
    tempLink = tempLink.replace(':', '-');
    tempLink = tempLink.replace('?', '-');
    tempLink = tempLink.replace('\"', '-');

    try {
      delay(1000);
      selClient.waitForPageToLoad("5000");
      if (browser.contains("*firefox")
          || browser.contains("*iexploreproxy")
          || browser.contains("*chrome")
          && !(link.contains("Error-"))) {
        selClient.captureEntirePageScreenshot(outputUIDir + tempLink
            + ".png", "");
      } else {
        // selClient.captureScreenshot(outputUIDir + tempLink + ".png");
        Robot r = new Robot();
        // take first part screenshot
        Rectangle captureSize = new Rectangle(Toolkit
            .getDefaultToolkit().getScreenSize());
        BufferedImage bi = r.createScreenCapture(captureSize);
        ImageIO.write(bi, "png", new File(outputUIDir + tempLink
            + " pic1.png"));
        r.keyPress(KeyEvent.VK_PAGE_DOWN);
        delay(2000);
        if (!(link.contains("Error-"))) {
          try {
            // take second part screenshot
            bi = r.createScreenCapture(captureSize);
            ImageIO.write(bi, "png", new File(outputUIDir
                + tempLink + " pic2.png"));
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      }

    } catch (IOException ex) {
      Logger.getLogger(Crawler.class.getName()).log(Level.SEVERE, null,
          ex);
    } catch (AWTException ex) {
      Logger.getLogger(Crawler.class.getName()).log(Level.SEVERE, null,
          ex);
    }
  }

  /*
   * Check for errors on page
   */
  public void checkForErrors(String link) {

    for (String s : errors) {
      if (selClient.isTextPresent(s)) {
        String tempLink = link.replace('/', '-');
        tempLink = tempLink.replace(':', '-');
        tempLink = tempLink.replace('?', '-');
        tempLink = tempLink.replace('\"', '-');
        tempLink += ".png";
        selClient.captureScreenshot(outputUIDir + tempLink);

      }
    }
  }

  /*
   * returns a truncated lower case string, removing "/" and ".php" at the end
   * of the string truncLink(link) link - the string to be truncated Author -
   * Ciprian-Alex Baltatescu
   */
  public String truncLink(String link) {

    if (link.charAt(link.length() - 1) == '/') {
      link = link.substring(0, link.length() - 1);
    }
    if (link.substring(link.length() - 4).equals(".php")) {
      link = link.substring(0, link.length() - 4);
    }
    link = link.toLowerCase();
    return link;
  }

  /*
   * back page function (browser behaviour) backPage() not used yet; can be
   * used openLink(link) function; TBD Author - Ciprian-Alex Baltatescu
   */
  public void backPage() {
    selClient.goBack();
    selClient.waitForPageToLoad("10000");
  }

  /*
   * pageTest() - recursive function a link is accessed and the function
   * verifies the new page (links, forms and fields) then accesses a new link
   * from the new page and calls itself Author - Ciprian-Alex Baltatescu
   *
   * Mihai Barcun - Handlers special test cases, forms or anything wished on
   * the page
   */
  public void pageTest() {

    String link = null;
    for (int k = 1; k < getLinksNumber(); k++) {
      link = getLinkHref(k);
      // debug
      // displayAllLinks();
      if (link == "") {
        continue;
      }
      System.out.println("New check...." + k + " " + link);
      if (link.contains("Formular")) {
        System.out.println("Here");
      }
      if ((getLinkIsValid(k)) && (getLinkNotLogout(k))
          && (getLinkNotVisited(truncLink(link)))
          && (getLinkNotInExcludeList(k))) {
        String alink = selClient.getLocation();
        if (!(alink.equals(link))) {
          openLink(link);
          visitedLinks[visitedLinksNumber] = truncLink(link);
          visitedLinksNumber++;
          try {
            fw.append("LINK " + link);
            fw.write(System.getProperty("line.separator"));
            fw.flush();
            System.out.println("writing.." + link);
          } catch (IOException e) {
            System.out.println("There was a problem:" + e);
          }
          // treating special cases. Webpages with forms I might want
          // to get over!
          SpecialCasesHandler sch = new SpecialCasesHandler(
              this.platform);
          sch.selClient = this.selClient;
          boolean xOK = sch.checkIsThisSpecialCase();
          // if yes, perform the special handler
          if (xOK) {
            String xStr = getCurrentURL();
            visitedLinks[visitedLinksNumber] = truncLink(xStr);
            visitedLinksNumber++;
            sch.handleMe();
            try {
              fw.append("LINK " + xStr);
              fw.write(System.getProperty("line.separator"));
              fw.flush();
            } catch (IOException ioe) {
              ioe.printStackTrace();
            }
          }
          pageTest();
          openLink(alink);
        }
      }
    }
  }

  public void delay(int s) {
    try {
      Thread.sleep(s); // do nothing for 1000 miliseconds (1 second)
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }

  /*
   * testPage() - the main function of the JUNIT test opens the first page
   * first link then calls pageTest() function Author - Ciprian-Alex
   * Baltatescu updated - Mihai Barcun
   */
  public void testPage() {
    try {

      System.out.println("test started!!!!!!!!!!");
      DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss");
      Date d = new Date();
      outputUIDir += platform + "-" + df.format(d)
          + System.getProperty("file.separator");
      outputXUIDir(outputUIDir);
      File f = new File(outputUIDir);
      f.mkdirs();

      try {
        pwPageLoad = new PrintWriter(new File(outputUIDir
            + "cpJustLoadTimes.txt"));
        pwPages = new PrintWriter(new File(outputUIDir
            + "cpPagesLoadTimes.txt"));
      } catch (FileNotFoundException fnfe) {
        System.out.println("error!!!!!!!!!!" + outputUIDir);
        System.err.println(fnfe.getMessage());
      }

      File f2 = new File(outputUIDir + "map.csv");
      f2.createNewFile();

      fw = new FileWriter(outputUIDir + "map.csv");
      selClient.open(mainURL);
      selClient.waitForPageToLoad("10000");

      visitedLinks[0] = mainURL;
      visitedLinks[1] = mainURL + "/";

      visitedLinksNumber = 2;
      fw.append(visitedLinks[1]);
      fw.write(System.getProperty("line.separator"));
      fw.flush();

      pageTest();
      fw.close();

    } catch (Exception e) {
      System.err.println("Error: " + e.getMessage());
      try {
        fw.close();
      } catch (IOException ex) {
        ex.printStackTrace();
      }
    }
  }

  /*
   * just ouputs the UIDir name for response time measurements python scripts
   */
  private void outputXUIDir(String Path) {
    try {
      FileWriter fstream = new FileWriter("out.temp");
      BufferedWriter out = new BufferedWriter(fstream);
      out.write(Path);
      out.close();
    } catch (Exception e) {
      System.err.println("Error: " + e.getMessage());
    }
  }

  /*
   * only for debug
   */

  public void displayAllLinks() {
    String link = "";
    for (int k = 1; k < getLinksNumber(); k++) {
      link = getLinkHref(k);
      System.out.println(k + " " + link);
    }

  }
}
TOP

Related Classes of webuicheck.Crawler

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.