package cputils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import jsoup.ConnectionManager;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import common.Combination;
import common.Utils;
public class CPHMLCService extends AbstractLinkAndData{
private static final Logger logger = Logger.getLogger(CPHMLCService.class);
// private static DateFormat format = new SimpleDateFormat("MM-dd HH:mm:ss");
private static Map<String,String> gameMap = new HashMap<String, String>();
static{
gameMap.put("ssq", "SSQ");
gameMap.put("dlt", "DLT");
}
public static void main(String[] args) throws IOException {
CPHMLCService service = new CPHMLCService();
//writeAicaiDLTHMData("14026");
// writeAicaiSSQHMData("2014025");
// service.writeTaobaoHMData("SSQ","2014028",false);
// writeTaobaoHMData("SSQ","2014027",false);
// writeTaobaoHMData("SSQ","2014026",false);
/*for (int i = 2014031; i <= 2014031; i++) {
Thread t = new Thread(new ServiceThread("SSQ",i+"",false));
t.start();
}*/
for (int i = 2014031; i <= 2014031; i++) {
//Thread t = new Thread(new ServiceThread("SSQ",i+"","",true));
//t.start();
service.writeLecaiHMData(i+"", "", false);
}
//Thread t = new Thread(new ServiceThreadLC("SSQ","2014036","",true));
//t.start();
//service.readDetailLinkTaobao("SSQ", "2014031", 1, "", true);
service.writeLecaiHMData("2014043", "", true);
// service.tjHM4Lecai("14044");
/*Map<String,Object> last = service.lastIssuoInfo("SSQ");
System.out.println(last.get("totalItem"));
System.out.println(last.get("totalPage"));
System.out.println(last.get("issueList"));
System.out.println(last.get("onsaleIssueNo"));*/
}
public Map<String, Map<Integer, Integer>> tjHM4Lecai(String issueNo) throws IOException{
Map<String, Map<Integer, Integer>> fullMap = tjHM4Lecai(issueNo,5);
System.out.println(fullMap);
Map<String, Map<Integer, Integer>> notfullMap = tjHM4Lecai(issueNo,4);
System.out.println(notfullMap);
Map<String, Map<Integer, Integer>> map = tjHM4Lecai(issueNo,100);
System.out.println(map);
return map;
}
private Map<String, Map<Integer, Integer>> tjHM4Lecai(String issueNo,int is_not_full) throws IOException{
Map<String, Map<Integer, Integer>> res = new HashMap<String, Map<Integer,Integer>>();
List<String> list = new ArrayList<String>();
String fileName = "lc_" + issueNo;
switch (is_not_full) {
case 5:
fileName = fileName+"_full";
list = XmlUtils.getHMData(fileName);
break;
case 4:
fileName = fileName+"_notfull";
list = XmlUtils.getHMData(fileName);
break;
case 100:
list = XmlUtils.getHMData(fileName+"_notfull");
list.addAll(XmlUtils.getHMData(fileName+"_full"));
fileName = fileName+"_all";
break;
default:
break;
}
Map<Integer, Integer> bMap = new HashMap<Integer, Integer>();
Map<Integer, Integer> aMap = new HashMap<Integer, Integer>();
for (int i = 1; i <= 35; i++) {
bMap.put(i, 0);
}
for (int i = 1; i <= 16; i++) {
aMap.put(i, 0);
}
Combination comb = new Combination();
List<String> allList = new ArrayList<String>();
for (String s : list) {
allList.addAll(comb.getAll(s, issueNo));
}
for (String item : allList) {
String[] sArr = item.split(":");
if (sArr!=null&&sArr.length==2) {
String[] bArr = sArr[0].split(" ");
String[] aArr = sArr[1].split(" ");
for (int i = 0; i < bArr.length; i++) {
if (StringUtils.isNotBlank(bArr[i])) {
int num = Integer.parseInt(bArr[i].trim());
if (bMap.get(num)!=null) {
bMap.put(num, bMap.get(num)+1);
}
}
}
for (int i = 0; i < aArr.length; i++) {
if (StringUtils.isNotBlank(aArr[i])) {
int num = Integer.parseInt(aArr[i].trim());
if (aMap.get(num)!=null) {
aMap.put(num, aMap.get(num)+1);
}
}
}
}
}
res.put("b", bMap);
res.put("a", aMap);
Utils.print(fileName+" line:"+list.size());
res = XmlUtils.makeSortedMap(res, true);
return res;
}
public void writeLecaiHMData(String issueNo,String userName,boolean onsale){
writeLecaiHMData(issueNo, 5,userName,onsale);//已满员
writeLecaiHMData(issueNo, 4,userName,onsale);
}
public void writeLecaiHMData(String issueNo,int is_not_full,String userName,boolean onsale){
String fullOrNotStr = getFullOrNotName(is_not_full);
String game = getGame(getLotteryType(issueNo, "lc"));
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks begin...");
long b1 = System.currentTimeMillis();
List<String> detailLinks = readDetailLinkLecai(issueNo, is_not_full,userName,onsale);
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks ok..."+detailLinks.size()+"...time:"+(System.currentTimeMillis()-b1)+"ms");
int size = detailLinks.size();
List<String> dataList = new ArrayList<String>();
int i = 1;
int totalRec = 0;
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMData begin...");
long b2 = System.currentTimeMillis();
for (String li : detailLinks) {
List<String> datas = readDataHMLecai(li);
if(i%300==0){
Utils.print("********LC "+game+" "+issueNo+"("+fullOrNotStr+") HMData..."+(i)+"("+datas.size()+")->"+size);
}
dataList.addAll(datas);
i++;
}
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMData ok..."+dataList.size()+".time:"+(System.currentTimeMillis()-b2));
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMData write begin...");
long b3 = System.currentTimeMillis();
StringBuilder cBuilder = new StringBuilder();
for (String data : dataList) {
totalRec++;
//data = data.replaceAll(" ", "").replaceAll(" ", "");
data = data.replaceAll("\\[单式\\]", "").replaceAll("\\[复式\\]", "")
.replaceAll("红球", "").replaceAll("蓝球", ":")
.replaceAll("前区", "").replaceAll("后区", ":")
.replaceAll("胆码", "(").replaceAll("拖码", ")")
.replaceAll("\\[胆拖\\]", "").replaceAll(",", " ").trim();
cBuilder.append(data).append("\r\n");
}
Utils.print("****LC "+game+" "+issueNo+"("+fullOrNotStr+") HMData write ok..."+dataList.size()+".time:"+(System.currentTimeMillis()-b3));
XmlUtils.writeConfigFile(cBuilder.toString(), "lc_"+issueNo+"_"+fullOrNotStr+".txt", false);
Utils.print("LC issueNo: "+issueNo+"("+fullOrNotStr+") finished...users:"+detailLinks.size()+" records:"+totalRec);
}
public static Map<String,Object> lastIssuoInfo(String game){
String url = "http://caipiao.taobao.com//lottery/ajax/get_united_list.htm" +
"?page=1&issue=&lotteryType="+game+"&playType=&commission_rate=-1"+
"&confidential=1&united_fee=0-0&is_not_full=1&creator=&sort_obj=bonusRatio&sort=desc&change_sort=false"+
"&lowAmount=0&highAmount=0&from=&onsale=&t="+System.currentTimeMillis();
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
String resultStr = doc.text();
Map<String,Object> infoMap = new HashMap<String, Object>();
JSONObject object = JSONObject.fromObject("{\""+resultStr.substring(resultStr.indexOf("totalItem")));
infoMap.put("totalItem", object.get("totalItem"));
infoMap.put("lotteryType", object.get("lotteryType"));
infoMap.put("totalPage", object.get("totalPage"));
List<String> issueList = new ArrayList<String>();
JSONArray arr = object.getJSONArray("issueList");
for (Object object2 : arr) {
JSONObject object3 = (JSONObject) object2;
issueList.add(object3.getString("issue"));
if (object3.getBoolean("onsale")) {
infoMap.put("onsaleIssueNo", object3.get("issue"));
}
}
infoMap.put("issueList", issueList);
return infoMap;
}
private int dataLinkPageNo = 1;
/**
* @param game
* @param issueNo
* @param is_not_full
* @return detailLink 详情界面链接
*/
private List<String> readDetailLinkLecai(String issueNo,int is_not_full,String userName,boolean onsale){
String fullOrNotStr = getFullOrNotName(is_not_full);
String lotteryType = getLotteryType(issueNo, "lc");
//condition=2,4(可跟单) condition=2,5(已满员)
//lottery_type=50 双色球 lottery_type=1 大乐透
//http://www.lecai.com/hemai/?page=2&pagesize=20&sort=a_0&lottery_type=50&condition=2,4
String url = "http://www.lecai.com/hemai/?page="+dataLinkPageNo+"&pagesize=20&sort=a_0&lottery_type="+lotteryType+"&condition=2,"+is_not_full;
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
List<String> resList = new ArrayList<String>();
int totalPage = 1;
if(!doc.getElementsByClass("last").isEmpty()){
String lastpageHref = doc.getElementsByClass("last").attr("href");
//System.out.println(lastpageHref.substring(lastpageHref.indexOf("page=")+5, lastpageHref.indexOf("&pagesize")));
if(lastpageHref.length()>0)
totalPage = Integer.parseInt(lastpageHref.substring(lastpageHref.indexOf("page=")+5, lastpageHref.indexOf("&pagesize")));
}
if(!doc.getElementsByClass("table-plan-area").isEmpty()){
Elements divs = doc.getElementsByClass("syndicate-plan-area");
for (Element element : divs) {
resList.add("http://www.lecai.com"+element.getElementsByTag("a").get(0).attr("href"));
}
/*Element table = doc.getElementsByClass("table-plan-area").get(0);
Elements trs = table.getElementsByTag("tr");
for(int n=1,size=trs.size();n<size;n++){
Element element = trs.get(n);
System.out.println(element.child(6));
}*/
}
if (dataLinkPageNo%10==0) {
Utils.print("********LC "+getGame(lotteryType)+" "+issueNo+"("+fullOrNotStr+") HMDetailLinks pageNo:"+dataLinkPageNo+" totalPage:"+totalPage);
}
dataLinkPageNo++;
if (totalPage>1&&totalPage>=dataLinkPageNo) {
resList.addAll(readDetailLinkLecai(issueNo, is_not_full,userName,onsale));
}else {
dataLinkPageNo = 1;
}
return resList;
}
private int hmdatapageNo = 1;
private List<String> readDataHMLecai(String dataLink){
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(dataLink);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
List<String> resList = new ArrayList<String>();
Element planDiv = doc.getElementById("plan_content");
if(planDiv!=null){
Elements ps = planDiv.getElementsByTag("p");
for (Element element : ps) {
resList.add(element.text());
}
}else{
if(doc.getElementById("det_upload_content")!=null){
String uploadHref = doc.getElementById("det_upload_content").getElementsByTag("a").get(0).attr("href");
resList.addAll(readUploadDataHMLecai("http://www.lecai.com"+uploadHref));
}
}
Elements pageEs = doc.getElementsByAttributeValue("class", "page-info");
if (pageEs.size()>0) {
String pageInfo = pageEs.get(0).ownText();
int pages = Integer.parseInt(pageInfo.substring(pageInfo.indexOf("/")+1));
hmdatapageNo++;
if (pages>1&&pages>=hmdatapageNo) {
resList.addAll(readDataHMLecai(dataLink));
}else {
hmdatapageNo = 1;
}
}
return resList;
}
private int uploadPageNo = 1;
private List<String> readUploadDataHMLecai(String dataLink){
String url = dataLink+"?page="+uploadPageNo;
Document doc = null;
try {
doc = ConnectionManager.getInstance().getDocument(url);
} catch (Exception e) {
logger.error("connect fail..", e);
return null;
}
List<String> resList = new ArrayList<String>();
int totalPage = 1;
if(!doc.getElementsByClass("last").isEmpty()){
String lastpageHref = doc.getElementsByClass("last").attr("href");
//System.out.println(lastpageHref.substring(lastpageHref.indexOf("page=")+5, lastpageHref.indexOf("&pagesize")));
if(lastpageHref.length()>0)
totalPage = Integer.parseInt(lastpageHref.substring(lastpageHref.indexOf("page=")+5));
}
Element table = doc.getElementById("upload_plan_detail");
if(table!=null){
Elements trs = table.getElementsByTag("tr");
for (int j = 1,size = trs.size(); j < size; j++) {
resList.add(trs.get(j).child(1).text());
}
}
uploadPageNo++;
if (totalPage>1&&totalPage>=uploadPageNo) {
resList.addAll(readUploadDataHMLecai(dataLink));
}else {
hmdatapageNo = 1;
}
return resList;
}
private String getGame(String game){
String gameName = "";
switch (Integer.parseInt(game)) {
case 1:
gameName = "DLT";
break;
case 50:
gameName = "SSQ";
break;
default:
break;
}
return gameName;
}
@Override
public List<String> dataLinks(int pageNo) {
// TODO Auto-generated method stub
return null;
}
@Override
public List<String> data(String dataLink) {
// TODO Auto-generated method stub
return null;
}
}
class ServiceThreadLC implements Runnable{
private String issueNo;
private boolean onsale;
private String userName;
public ServiceThreadLC(String issueNo,String userName,boolean onsale){
this.issueNo = issueNo;
this.onsale = onsale;
this.userName = userName;
}
@Override
public void run() {
//synchronized(CPHMService.class){
CPHMLCService service = new CPHMLCService();
service.writeLecaiHMData(issueNo,userName, onsale);
//}
}
}