Package org.vietspider.common.io

Examples of org.vietspider.common.io.DataWriter


    response = webClient.execute(httpHost, httpGet);
    entity = response.getEntity();

    HttpResponseReader httpResponseReader = new HttpResponseReader();
    byte [] bytes = httpResponseReader.readBody(response);
    new DataWriter().save(new File("google_mail.html"), bytes);

    System.out.println("Login form get: " + response.getStatusLine());
    if (entity != null) entity.consumeContent();

    System.out.println("Post logon cookies:");
View Full Code Here


    entity = response.getEntity();


    HttpResponseReader httpResponseReader = new HttpResponseReader();
    byte [] bytes = httpResponseReader.readBody(response);
    new DataWriter().save(new File("my_yahoo.html"), bytes);

    System.out.println("Login form get: " + response.getStatusLine());
    if (entity != null) entity.consumeContent();

    System.out.println("Post logon cookies:");
View Full Code Here

    System.out.println(home);
   
    webClient.setUserAgent("Mozilla/5.0 (compatible; Yahoo! VN Slurp; http://help.yahoo.com/help/us/ysearch/slurp)");
//    webClient.registryProxy("216.194.70.3", 8118, null, null);
   
    final DataWriter writer = new DataWriter();
    final DataReader reader = new DataReader();
    File file  = new File("F:\\Temp2\\webclient\\proxies.txt") ;
    String textValue  = new String(reader.load(file), "utf-8");
    String [] proxies = textValue.split("\n");
    File newFile  = new File("F:\\Temp2\\webclient\\good.proxies.txt") ;
  
    final File folder = new File("F:\\Temp2\\webclient\\proxy\\");
    deleteFolder(folder);

    final List<String> aliveList = new ArrayList<String>();

    for(int i = 0; i < proxies.length; i++) {
      try {
        proxies[i] = proxies[i].trim();
        webClient.setURL(null, new URL(home));
        System.out.println("=== >"+ proxies[i]+" ==> "+i+"/"+proxies.length);
//      System.out.println("=== >"+Thread.currentThread().getId()+ " : "+ array[i]);
        String [] elements = proxies[i].split(":");
        String proxyHost = elements[0].trim();
        int proxyPort = Integer.parseInt(elements[1].trim());

        long start = System.currentTimeMillis();
        webClient.registryProxy(proxyHost, proxyPort, null, null);
        byte[] bytes = loadContent(address);
        long end = System.currentTimeMillis();
        if((start - end) > WAIT) {
          System.out.println(" timeout ");
          continue;
        }
       
        if(bytes == null) continue;
        String fileName = proxies[i].replace('.', '_');
        fileName = fileName.replace(':', '_');
        writer.save(new File(folder, fileName+".html"), bytes);
       
        System.out.println(bytes.length+ " : "+ (bytes.length > 15000));
        if(bytes.length > 30000) {
          String value = proxyHost + ":" + proxyPort;
          if(aliveList.contains(value)) continue;
          aliveList.add(value);
          writer.append(newFile, (value+"\n").getBytes());
        }
      } catch (Exception e) {
        e.printStackTrace();
      }
    }//het for
View Full Code Here

  File getHTMLFile(int code) {
    File file = UtilFile.getFolder("track/temp/");
    file  = new File(file, String.valueOf(code)+".html");
    try {
      new DataWriter().save(file, getSelectedHTML().getBytes("utf-8"));
    } catch (Exception e) {
      ClientLog.getInstance().setMessage(browser.getShell(), e);
    }
    return file;
  }
View Full Code Here

  File getDocumentFile(int code) {
    File file = UtilFile.getFolder("track/temp/");
    file  = new File(file, String.valueOf(code)+".html");
    try {
      new DataWriter().save(file, browser.getText().getBytes("utf-8"));
    } catch (Exception e) {
      ClientLog.getInstance().setMessage(browser.getShell(), e);
    }
    return file;
  }
View Full Code Here

    response = webClient.execute(httpHost, httpGet);
    entity = response.getEntity();

    HttpResponseReader httpResponseReader = new HttpResponseReader();
    byte [] bytes = httpResponseReader.readBody(response);
    new DataWriter().save(new File("java_net.html"), bytes);

    System.out.println("Login form get: " + response.getStatusLine());
    if (entity != null) entity.consumeContent();

    System.out.println("Post logon cookies:");
View Full Code Here

    String text = builder.toString();
    if(text.isEmpty()) return;
   
    File file  = UtilFile.getFile("system/proxy/", "bad.proxies.txt");
    try {
      new DataWriter().save(file, text.getBytes("utf-8"));
    } catch (Exception e) {
    }
  }
View Full Code Here

   
    System.out.println(doc.getRoot().getTextValue());
   
    File file = new File("a.html");
    byte[] data = doc.getTextValue().getBytes();
    new DataWriter().save(file, data);
  }
View Full Code Here

TOP

Related Classes of org.vietspider.common.io.DataWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.