package cputils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.RecursiveTask;
import jsoup.ConnectionManager;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import common.Utils;
public class CPHMWYService extends AbstractLinkAndData{
private static final Logger logger = Logger.getLogger(CPHMWYService.class);
// private static DateFormat format = new SimpleDateFormat("MM-dd HH:mm:ss");
private boolean onsale = true;
private int dataLinkTotalPage = 1;
public boolean isOnsale() {
return onsale;
}
public void setOnsale(boolean onsale) {
this.onsale = onsale;
}
public CPHMWYService(){
}
public static void main(String[] args) throws IOException {
Map<String, Object> last = CPHMTBService.lastIssuoInfo("SSQ");
String ssqonsaleNo = (String) last.get("onsaleIssueNo");
last = CPHMTBService.lastIssuoInfo("DLT");
String dltonsaleNo = (String) last.get("onsaleIssueNo");
CPHMWYService service = new CPHMWYService();
/*service.setOnsale(false);
try {
// System.out.println("*********SSQ*******");
// long b = System.currentTimeMillis();
// for (int i = 2014070; i < Integer.parseInt(ssqonsaleNo); i++) {
// service.writeWYHMData(""+i, false);
// }
// System.out.println("****SSQ time:"+(System.currentTimeMillis()-b));
// System.out.println("*********DLT*******");
// b = System.currentTimeMillis();
// for (int i = 14038; i < Integer.parseInt(dltonsaleNo); i++) {
// service.writeWYHMData(""+i, false);
// }
// System.out.println("****DLT time:"+(System.currentTimeMillis()-b));
service.writeWYHMData(ssqonsaleNo, true);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}*/
service.stat(Integer.parseInt(ssqonsaleNo)-1+"");
}
public Map<String, Map<Integer, Integer>> stat(String issueNo) throws IOException{
Map<String, Map<Integer, Integer>> fullMap = stat(issueNo,3);
System.out.println(fullMap);
Map<String, Map<Integer, Integer>> notfullMap = stat(issueNo,2);
System.out.println(notfullMap);
Map<String, Map<Integer, Integer>> map = stat(issueNo,100);
System.out.println(map);
return map;
}
private Map<String, Map<Integer, Integer>> stat(String issueNo,int is_not_full) throws IOException{
Map<String, Map<Integer, Integer>> res = stat(issueNo, is_not_full, "wy");
return res;
}
public void writeWYHMData(String issueNo,boolean onsale) throws InterruptedException, ExecutionException{
setOnsale(onsale);
writeWYHMData(issueNo, 3);//full
if(onsale){
writeWYHMData(issueNo, 2);//not full
}
}
public void writeWYHMData(String issueNo,int is_not_full) throws InterruptedException, ExecutionException{
String fullOrNotStr = getFullOrNotName(is_not_full);
String lotteryType = getLotteryType(issueNo, "wy").toUpperCase();
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks begin...");
long b1 = System.currentTimeMillis();
List<String> lottOrderIds = wyDataLinks(issueNo, is_not_full);
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks ok..."+lottOrderIds.size()+"...time:"+(System.currentTimeMillis()-b1)+"ms");
// int size = lottOrderIds.size();
List<String> dataList = new ArrayList<String>();
// int i = 1;
int totalRec = 0;
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMData begin...");
long b2 = System.currentTimeMillis();
dataList = wyDatas(lottOrderIds);
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMData ok..."+dataList.size()+".time:"+(System.currentTimeMillis()-b2));
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMData write begin...");
long b3 = System.currentTimeMillis();
StringBuilder cBuilder = new StringBuilder();
for (String data : dataList) {
totalRec++;
cBuilder.append(data).append("\r\n");
}
Utils.print("****WY "+lotteryType+" "+issueNo+"("+fullOrNotStr+") HMData write ok..."+dataList.size()+".time:"+(System.currentTimeMillis()-b3));
XmlUtils.writeConfigFile(cBuilder.toString(), "wy_"+issueNo+"_"+fullOrNotStr+".txt", false);
Utils.print("WY issueNo: "+issueNo+"("+fullOrNotStr+") finished...users:"+lottOrderIds.size()+" records:"+totalRec);
}
public Map<String,Object> lastIssuoInfo(String game){
String url = "http://caipiao.taobao.com//lottery/ajax/get_united_list.htm" +
"?page=1&issue=&lotteryType="+game+"&playType=&commission_rate=-1"+
"&confidential=1&united_fee=0-0&is_not_full=1&creator=&sort_obj=bonusRatio&sort=desc&change_sort=false"+
"&lowAmount=0&highAmount=0&from=&onsale=&t="+System.currentTimeMillis();
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
String resultStr = doc.text();
Map<String,Object> infoMap = new HashMap<String, Object>();
JSONObject object = JSONObject.fromObject("{\""+resultStr.substring(resultStr.indexOf("totalItem")));
infoMap.put("totalItem", object.get("totalItem"));
infoMap.put("lotteryType", object.get("lotteryType"));
infoMap.put("totalPage", object.get("totalPage"));
List<String> issueList = new ArrayList<String>();
JSONArray arr = object.getJSONArray("issueList");
for (Object object2 : arr) {
JSONObject object3 = (JSONObject) object2;
issueList.add(object3.getString("issue"));
if (object3.getBoolean("onsale")) {
infoMap.put("onsaleIssueNo", object3.get("issue"));
}
}
infoMap.put("issueList", issueList);
return infoMap;
}
public int dataLinkPages(String issueNo,int is_not_full){
String lotteryType = getLotteryType(issueNo, "wy");
String url = "http://caipiao.163.com/groupbuy/" +lotteryType+
"/?pageNum=1&sortType=desc&orderBy=remain_pieces" +
"¶meter.status="+is_not_full+"&showTop=not&difdisplay=true"+
"¶meter.proportion=-1¶meter.createrNickName="+
"¶meter.period="+issueNo;
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return 0;
}
int totalPage = 0;
if(doc!=null&&doc.getElementsByClass("jumptoTxt")!=null
&&doc.getElementsByClass("jumptoTxt").size()>0){
totalPage = Integer.parseInt(doc.getElementsByClass("jumptoTxt").get(0).attr("maxPage"));
dataLinkTotalPage = totalPage;
}
return totalPage;
}
public List<String> wyDataLinks(String issueNo,int is_not_full) throws InterruptedException, ExecutionException{
//ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
ForkJoinPool forkJoinPool = new ForkJoinPool(5);
int linkPages = dataLinkPages(issueNo, is_not_full);
DataLinkThread thread = new DataLinkThread(1,linkPages,issueNo, is_not_full,this);
Future<List<String>> result = forkJoinPool.submit(thread);
forkJoinPool.shutdown();
return result.get();
}
public List<String> wyDatas(List<String> lottOrderIds) throws InterruptedException, ExecutionException{
ForkJoinPool forkJoinPool = new ForkJoinPool(5);
int size = lottOrderIds.size();
DataThread thread = new DataThread(0,size-1,lottOrderIds,this);
Future<List<String>> result = forkJoinPool.submit(thread);
forkJoinPool.shutdown();
return result.get();
}
/**
* @param game
* @param issueNo
* @param is_not_full
* @return detailLink 详情界面链接
*/
public List<String> readDetailLink163(String issueNo,int is_not_full,int linkPage){
String fullOrNotStr = getFullOrNotName(is_not_full);
/*<input type="hidden" id="currentGroupbuyPeriod" value="14032" />
parameter.status "2" 未满员 "3"已满员 "4"已撤单
parameter.proportion
"-1" 不限佣金 "0" = 0 "0.01" <= 1
("0.02" <= 2)("0.03" <= 3)("0.04" <= 4)("0.05" <= 5)
("0.06" <= 6)("0.07" <= 7)("0.08" <= 8)("0.09" <= 9)
guarantee ("0" 不限保底)("0.05" >= 5)("0.1" >= 10)("0.15" >= 15)
("0.2" >= 20)("0.25" >= 25)("0.3" >= 30)
orderBy=bonus(奖金排序)
*/
String lotteryType = getLotteryType(issueNo, "wy");
String url = "http://caipiao.163.com/groupbuy/" +lotteryType+
"/?pageNum="+linkPage+"&sortType=desc&orderBy=remain_pieces" +
"¶meter.status="+is_not_full+"&showTop=not&difdisplay=true"+
"¶meter.proportion=-1¶meter.createrNickName="+
"¶meter.period="+issueNo;
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
List<String> resList = new ArrayList<String>();
Element tableE = doc.getElementsByClass("group_buy").get(0);
Elements trs = tableE.getElementsByTag("tr");
for (int m = 1,len=trs.size();m<len;m++) {
Element element = trs.get(m);
if ("public".equals(element.child(0).child(1).attr("class"))) {
//resList.add("http://caipiao.163.com"+element.child(8).child(1).attr("href"));
if(onsale){
resList.add(element.child(8).child(0).attr("lotoorder"));
}else{
String href = element.child(7).child(0).attr("href");
resList.add(href.substring(href.indexOf("_")+1, href.indexOf(".")));
}
}
}
if (linkPage%10==0) {
Utils.print("********WY "+lotteryType.toUpperCase()+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks pageNo:"+linkPage+" totalPage:"+dataLinkTotalPage);
}
return resList;
}
private int hmDataPages(String lottOrderId){
int dataPages = 0;
String url = "http://caipiao.163.com/hit/gd_shuzi_detail.html?pageSize=10&pageNum=1"
+"&lottOrderId="+lottOrderId+"&stamp="+System.currentTimeMillis();
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return 0;
}
if (doc.getElementById("fcase_page_num")!=null) {
int totalItems = Integer.parseInt(doc.getElementById("fcase_page_num").attr("value"));
dataPages = (totalItems/10)+(totalItems%10==0?0:1);
}
return dataPages;
}
public List<String> readDataHMWY(String lottOrderId){
List<String> resList = new ArrayList<String>();
int dataPages = hmDataPages(lottOrderId);
for (int i = 1; i <= dataPages; i++) {
resList.addAll(readDataHMWY(lottOrderId, i));
}
return resList;
}
public List<String> readDataHMWY(String lottOrderId,int dataPage){
//System.out.println(lottOrderId+" read data-->"+Thread.currentThread().getName());
//http://caipiao.163.com/hit/gd_shuzi_detail.html?pageNum=1&pageSize=10&
//lottOrderId=2014031922CP05952095&stamp=1395403769673
String url = "http://caipiao.163.com/hit/gd_shuzi_detail.html?pageSize=10&pageNum="+dataPage
+"&lottOrderId="+lottOrderId+"&stamp="+System.currentTimeMillis();
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
List<String> resList = new ArrayList<String>();
if (doc!=null&&doc.getElementById("fcase_page_num")!=null) {
Element tableE = doc.getElementsByClass("table_szc").get(0);
Elements trs = tableE.getElementsByTag("tr");
for (int m = 1,len=trs.size();m<len;m++) {
Element element = trs.get(m);
String b = "";
Elements spansb = element.child(1).getElementsByClass("c_ba2636");
Iterator<Element> bIterator = spansb.iterator();
while(bIterator.hasNext()){
Element element2 = bIterator.next();
if(!"span".equals(element2.tagName())){
//spansb.remove(element2);
bIterator.remove();
}
}
if (spansb.size()==1) {
if (element.child(1).child(1).text().indexOf("...")==-1) {
b = element.child(1).child(1).text();
}else {
b = element.child(1).child(1).attr("inf");
}
}else {
for (int j = 0,size=spansb.size();j<size; j++) {
Element element2 = spansb.get(j);
//System.out.println(element2.nextElementSibling().text());
Element next = element2.nextElementSibling();
if (next.text().indexOf("...")==-1) {
if (j==0) {
b+="("+next.text()+")";
}else {
b+=next.text();
}
}else {
if (j==0) {
b+="("+next.attr("inf")+")";
}else {
b+=next.attr("inf");
}
}
}
}
String a = "";
Elements spansa = element.child(1).getElementsByClass("c_1e50a2");
Iterator<Element> aIterator = spansa.iterator();
while(aIterator.hasNext()){
Element element2 = aIterator.next();
if(!"span".equals(element2.tagName())){
//spansb.remove(element2);
aIterator.remove();
}
}
if (spansa.size()==1) {
if (element.child(1).child(4).text().indexOf("...")==-1) {
a = element.child(1).child(4).text();
}else {
a = element.child(1).child(4).attr("inf");
}
}else {
for (int j = 0,size=spansa.size();j<size; j++) {
Element element2 = spansa.get(j);
Element next = element2.nextElementSibling();
if (next.text().indexOf("...")==-1) {
if (j==0) {
a+="("+next.text()+")";
}else {
a+=next.text();
}
}else {
if (j==0) {
a+="("+next.attr("inf")+")";
}else {
a+=next.attr("inf");
}
}
}
//System.out.println("-----------------------------------------------------------"+lottOrderId);
//System.out.println("-----------------------------------------------------------"+b+":"+a);
}
resList.add(b+":"+a);
//System.out.println(b+":"+a);
}
}
return resList;
}
@Override
public List<String> dataLinks(int pageNo) {
// TODO Auto-generated method stub
return null;
}
@Override
public List<String> data(String dataLink) {
// TODO Auto-generated method stub
return null;
}
}
class DataLinkThread extends RecursiveTask<List<String>>{
private static final long serialVersionUID = 604742182971692811L;
private static final int THRESHOLD = 30;
private int start;
private int end;
private String issueNo;
private int full_flag;
private CPHMWYService wyService = null;
public DataLinkThread(int start, int end,String issueNo,int full_flag,CPHMWYService wyService) {
this.start = start;
this.end = end;
this.issueNo = issueNo;
this.full_flag = full_flag;
this.wyService = wyService;
}
@Override
protected List<String> compute() {
List<String> alllist = new ArrayList<String>();
List<String> list = null;;
if((end - start) < THRESHOLD){
for(int i = start; i<= end;i++){
list = wyService.readDetailLink163(this.issueNo, this.full_flag, i);
alllist.addAll(list);
}
// System.out.println("["+Thread.currentThread().getName()+"]start:"+start+" end:"+end);
return alllist;
}
int middle = (start + end) /2;
DataLinkThread left = new DataLinkThread(start, middle,this.issueNo, this.full_flag,wyService);
DataLinkThread right = new DataLinkThread(middle + 1, end,this.issueNo, this.full_flag,wyService);
left.fork();
right.fork();
alllist.addAll(left.join());
alllist.addAll(right.join());
return alllist;
}
}
class DataThread extends RecursiveTask<List<String>>{
private static final long serialVersionUID = 604742182971692811L;
private static final int THRESHOLD = 500;
private int start;
private int end;
private List<String> lottOrderIds;
private CPHMWYService wyService = null;
public DataThread(int start, int end,List<String> lottOrderIds,CPHMWYService wyService) {
this.start = start;
this.end = end;
this.lottOrderIds = lottOrderIds;
this.wyService = wyService;
}
@Override
protected List<String> compute() {
List<String> alllist = new ArrayList<String>();
List<String> list = null;;
if((end - start) < THRESHOLD){
for(int i = start; i<= end;i++){
list = getData(lottOrderIds.get(i));
alllist.addAll(list);
}
//if (alllist.size()%200==0) {
Utils.print("********WY HMData...("+start+"--"+end+")->"+alllist.size());
//}
return alllist;
}
int middle = (start + end) /2;
DataThread left = new DataThread(start, middle,lottOrderIds,wyService);
DataThread right = new DataThread(middle + 1, end,lottOrderIds,wyService);
left.fork();
right.fork();
alllist.addAll(left.join());
alllist.addAll(right.join());
return alllist;
}
private List<String> getData(String lottOrderId){
return wyService.readDataHMWY(lottOrderId);
}
}