if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_1")) {
projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
JSONArray jsonStr=new JSONArray(queryStr.trim());
JSONObject obj=new JSONObject();
obj.put("key", "app_type");
obj.put("operate", "1");
obj.put("value", "1");
jsonStr.put(obj);
queryStr = WebServiceParams.query(jsonStr.toString());
}
if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_2")) {
projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
JSONArray jsonStr=new JSONArray(queryStr.trim());
JSONObject obj=new JSONObject();
obj.put("key", "app_type");
obj.put("operate", "1");
obj.put("value", "2");
jsonStr.put(obj);
queryStr = WebServiceParams.query(jsonStr.toString());
}
if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_3")) {
projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
JSONArray jsonStr=new JSONArray(queryStr.trim());
JSONObject obj=new JSONObject();
obj.put("key", "app_type");
obj.put("operate", "1");
obj.put("value", "3");
jsonStr.put(obj);
queryStr = WebServiceParams.query(jsonStr.toString());
}
if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_4")) {
projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
JSONArray jsonStr=new JSONArray(queryStr.trim());
JSONObject obj=new JSONObject();
obj.put("key", "app_type");
obj.put("operate", "1");
obj.put("value", "4");
jsonStr.put(obj);
queryStr = WebServiceParams.query(jsonStr.toString());
}
TablePartion part = GetPartions.partion(projectName);
ShardsList[] cores = GetShards.getCoresNonCheck(part);
ShardsList[] ms = GetShards.getMergers(part.name);
String[] partionsAll = AdhocHivePartions.get(queryStr, part.parttype);
GetPartions.Shards shard = GetPartions.getshard(part, partionsAll,cores, ms);
LinkedHashMap<String, String> filetypeMap = MdrillFieldInfo.readFieldsFromSchemaXml(stormconf,part.name);
ArrayList<String> fqList = WebServiceParams.fqListHive(false,hpart,queryStr,
isPartionByPt, filetypeMap,null,null,null);
StringBuffer sqlWhere =AdhocWebServiceParams.makeWhere(fqList,new ArrayList<String>(),"userid");
ArrayList<String> groupFields = WebServiceParams.groupFields(groupby);
ArrayList<String> showFields = WebServiceParams.showHiveFields(fl);
HigoAdhocJoinParams[] joins=AdhocWebServiceParams.parseJoinsHive(leftjoin, shard);
String daycols=AdhocWebServiceParams.parseDayCols(hpart,groupFields,showFields);
HashMap<String,String> colMap=new HashMap<String, String>();
HashMap<String,String> colMapforStatFilter=new HashMap<String, String>();
StringBuffer cols_inner = new StringBuffer();
StringBuffer cols = new StringBuffer();
String join = "";
AtomicInteger nameindex=new AtomicInteger(0);
String hql="";
if(joins.length<=0)
{
AdhocWebServiceParams.parseColsNoJoins(cols, groupFields, showFields,colMapforStatFilter, nameindex);
hql = "select " + cols.toString() + " from " + projectName + " "
+ sqlWhere.toString() + " " ;
StringBuffer sqlGroup = new StringBuffer();
join = " group by ";
for (String field : groupFields) {
sqlGroup.append(join);
sqlGroup.append(field);
join = ",";
}
hql+=" "+sqlGroup.toString();
}else
{
AdhocWebServiceParams.parseColsWithJoins(cols,cols_inner, joins, colMap,colMapforStatFilter, groupFields, showFields, nameindex);
StringBuffer buffer=new StringBuffer();
buffer.append("select ");
buffer.append(cols.toString());
buffer.append(" from ");
buffer.append(" (select "+cols_inner.toString()+" from " + projectName + " " + sqlWhere.toString() + " ) jl1 ");
for(int i=0;i<joins.length;i++)
{
HigoAdhocJoinParams jp=joins[i];
buffer.append(" join ("+jp.frQuer+") jr"+i+" on (jl1."+colMap.get(jp.leftkey)+" is not null and trim(jl1."+colMap.get(jp.leftkey)+")==trim(jr"+i+"."+jp.rightkey+")) ");
}
hql=buffer.toString();
StringBuffer sqlGroup = new StringBuffer();
join = " group by ";
for (String field : groupFields) {
sqlGroup.append(join);
sqlGroup.append("jl1."+colMap.get(field));
join = ",";
}
if(AdhocWebServiceParams.hasStatFiled(showFields))
{
for(int i=0;i<joins.length;i++)
{
HigoAdhocJoinParams jp=joins[i];
for(String s:jp.fl)
{
sqlGroup.append(join);
sqlGroup.append("jr"+i+"."+s);
join=",";
}
}
}
hql+=" "+sqlGroup.toString();
}
ArrayList<String> fq2list=WebServiceParams.fqListHive(true,hpart,fq2,
isPartionByPt, filetypeMap,colMap,colMapforStatFilter,"fq2");
if(fq2list.size()>0||orderby2!=null)
{
StringBuffer buffer=new StringBuffer();
buffer.append("select * from ");
buffer.append("("+hql+") fq2");
if(fq2list.size()>0)
{
String join2 = " where ";
for (String fq : fq2list) {
buffer.append(join2);
buffer.append(fq);
join2 = " and ";
}
}
if(orderby2!=null)
{
buffer.append(" order by "+WebServiceParams.parseFqAlias(orderby2, colMap, colMapforStatFilter, "fq2")+" "+desc2);
}
hql=buffer.toString();
}
if(limit >1000000)
{
limit=1000000;
}
if(limit>0)
{
hql=hql+" limit "+limit;
}
hql=hql.replaceAll("dist\\((.*)\\)", "count(distinct($1))");
String md5 = MD5.getMD5(hql);
SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
String day = fmt.format(new Date());
String hdpConf = (String) stormconf.get("hadoop.conf.dir");
String connstr = (String) stormconf.get("higo.download.offline.conn");
String uname = (String) stormconf.get("higo.download.offline.username");
String passwd = (String) stormconf.get("higo.download.offline.passwd");
String store = (String) stormconf.get("higo.download.offline.store")
+ "/" + day + "/" + java.util.UUID.randomUUID().toString();
MySqlConn conn = new MySqlConn(connstr, uname, passwd);
MysqlInfo info = new MysqlInfo(conn);
if (username == null || username.length() <= 0) {
username = "default";
}
StringBuffer sqlbuff = new StringBuffer();
// int jobsize=info.getUserJobname(username, jobname, sqlbuff).size();
sqlbuff.append(";");
int size = info.getUser(username, true, sqlbuff).size();
JSONObject jsonObj = new JSONObject();
jsonObj.put("sqlbuff", sqlbuff.toString());
jsonObj.put("size", size);
// if(jobsize>0)
// {
// jsonObj.put("code", "0");
// jsonObj.put("message", "之前已经有叫"+jobname+"的任务,请换个名字");
// }else
//
if (size < 5) {
MysqlCallback callback = new MysqlCallback(conn);
String[] pcols = params == null ? new String[0] : new String(daycols
+ params.replaceAll("维度指标：", "").replaceAll("。.*$", ""))
.split(",");
StringBuffer liststat = new StringBuffer();
StringBuffer listgroup = new StringBuffer();
for (String s : pcols) {
if (AdhocOfflineService.isStatFields(s)) {
liststat.append(s);
liststat.append(",");
} else {
listgroup.append(s);
listgroup.append(",");
}
}
callback.setCols((params == null || params.isEmpty()) ? cols
.toString() : listgroup.toString() + liststat.toString());
OfflineDownload download = new OfflineDownload();
download.setOffline(callback);
if (mailto == null || mailto.length() <= 0) {
mailto = "yannian.mu@alipay.com";
}
StringBuffer setSql=new StringBuffer();
StringBuffer cleanSql=new StringBuffer();
for(int i=0;i<joins.length;i++)
{
HigoAdhocJoinParams jp=joins[i];
setSql.append(jp.createSql);
setSql.append(";");
setSql.append(jp.addData);
setSql.append(";");
cleanSql.append(";");
cleanSql.append(jp.DropSql);
}
download.setMailto(mailto);
download.setHql(setSql.toString()+" INSERT OVERWRITE DIRECTORY '" + store + "' " + hql+" "+cleanSql.toString());
download.setUseName(username);
if (jobname == null || jobname.length() <= 0) {
jobname = day + "_" + md5;
}
download.setJobName(jobname);
download.setMemo(String.valueOf(memo));
download.setDisplayParams((params == null || params.isEmpty()) ? hql
: params);
download.setStoreDir(store);
download.setConfdir(hdpConf);
download.setSqlMd5(md5);
download.run();
long t2 = System.currentTimeMillis();
jsonObj.put("code", "1");
jsonObj.put("message",
"数据下载中...完成后将会通过<b style=\"color:red\">旺旺</b>和<b style=\"color:red\">邮件</b>通知");
jsonObj.put("uuid", callback.getUuid());
jsonObj.put("debug", callback.toString());
jsonObj.put("timedebug", String.valueOf(t2 - t1));
} else {
jsonObj.put("code", "0");
jsonObj.put("message", "还有" + size + "个任务没有完成数据下载，请稍后提交");
}
if (jsoncallback != null && jsoncallback.length() > 0) {
return jsoncallback + "(" + jsonObj.toString() + ")";
} else {
return jsonObj.toString();
}
}