Package com.alimama.mdrill.json

Examples of com.alimama.mdrill.json.JSONObject


      String uuuuid=uuid.trim().replaceAll("\t", "");
      if(uuuuid.isEmpty())
      {
        continue;
      }
      JSONObject info=readAbtestInfo(uuuuid);
      LOG.info(uuid+">>"+info.toString());
      if(String.valueOf(info.get("code")).equals("1"))
      {
        exlist.add(String.valueOf(info.getJSONObject("data").get("A")));
        exlist.add(String.valueOf(info.getJSONObject("data").get("B")));
       
      }
    }
   
   
    String[] number_important=Json2Array(number_important_json);
    String[] colls_important=Json2Array(colls_important_json);
    String[] colls=Json2Array(colls_json);
    String[] numbers=Json2Array(numbers_json);
   
    long t1 = System.currentTimeMillis();
 
    Map stormconf = Utils.readStormConfig();
   
    hivePartion hp=getPartion(stormconf, projectName);
    boolean isPartionByPt=hp.isPartionByPt;
    String hpart=hp.hpart;
    projectName=hp.projectName;

    String mode=String.valueOf(stormconf.get("higo.mode."+projectName));

    queryStr = WebServiceParams.query(queryStr);


    TablePartion part = GetPartions.partion(projectName);
    KmeansQueryParse kmeansData = AdhocHivePartions.getKmeansDays(queryStr);
   
    LinkedHashMap<String, String> filetypeMap = MdrillFieldInfo.readFieldsFromSchemaXml(stormconf,part.name);
    ArrayList<String> fqList = WebServiceParams.fqListHive(false,hpart,kmeansData.queryStr,isPartionByPt, filetypeMap,null,null,null);
   
    StringBuffer sqlWhere =AdhocWebServiceParams.makeWhere(fqList,exlist,idcols);
    LOG.info("queryStr:"+queryStr+">>>>kmeansData.queryStr:"+kmeansData.queryStr+","+sqlWhere.toString()+","+fqList.toString());

 
//    String hql = "select custid from " + projectName + " " + sqlWhere.toString() + " " ;
    final String[] sqlparams=PrintSql.makeSql(projectName, sqlWhere.toString(), idcols, kmeansData.getSortDays(), colls_important, number_important, colls, numbers);;
    String hql=sqlparams[0];
   
    String md5 = MD5.getMD5(hql);
 
    SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
    String day = fmt.format(new Date());
 
    final String hdpConf = (String) stormconf.get("hadoop.conf.dir");
    String connstr = (String) stormconf.get("higo.download.offline.conn");
    String uname = (String) stormconf.get("higo.download.offline.username");
    String passwd = (String) stormconf.get("higo.download.offline.passwd");
    final String store = (String) stormconf.get("higo.download.offline.store")+ "/" + day + "/" + java.util.UUID.randomUUID().toString();
    MySqlConn conn = new MySqlConn(connstr, uname, passwd);
    MysqlInfo info = new MysqlInfo(conn);
    if (username == null || username.length() <= 0) {
      username = "default";
    }
 
    StringBuffer sqlbuff = new StringBuffer();
    sqlbuff.append(";");
    int size = info.getUser(username, true, sqlbuff).size();
    JSONObject jsonObj = new JSONObject();
    jsonObj.put("sqlbuff", sqlbuff.toString());
    jsonObj.put("size", size);

    final String storepath=store+"/abtest/cluster_abtest";
    if (size < 5) {
      final MysqlCallbackKmeans callback = new MysqlCallbackKmeans(conn);
     
      Runnable process=new Runnable() {
       
        @Override
        public void run() {
          Configuration conf=new Configuration();
           HadoopBaseUtils.grabConfiguration(hdpConf, conf);
          try {
            ToolRunner.run(conf, new KMeansDriver(callback), new String[]{
              ""
              ,store+"/hive"
              ,store+"/abtest"
              ,"20"
              ,"1000" //k
              ,"0.00001" //delta
              ,String.valueOf(count)
              ,String.valueOf(rep)
              ,String.valueOf(100)
              ,sqlparams[1]
              ,String.valueOf(548576)
            });
           
            makeAB(storepath, conf);

          } catch (Exception e) {
            throw new RuntimeException(e);
          }
         
        }
      };
      callback.setCols("");
      OfflineDownload download = new OfflineDownload();
      download.setOffline(callback);
      if (mailto == null || mailto.length() <= 0) {
        mailto = "yannian.mu@alipay.com";
      }
     
      download.setMailto(mailto);
      download.setHql(" INSERT OVERWRITE DIRECTORY '" + store+"/hive" + "' " + hql+"  ");
      download.setUseName(username);
      if (jobname == null || jobname.length() <= 0) {
        jobname = day + "_" + md5;
      }
      download.setJobName(jobname);
      download.setMemo(String.valueOf(memo));
      download.setDisplayParams(params);
      download.setStoreDir(storepath);
      download.setConfdir(hdpConf);
      download.setSqlMd5(md5);
      download.setProcesser(process);
      download.run();
 
      long t2 = System.currentTimeMillis();
      jsonObj.put("code", "1");
 
      jsonObj.put("message","数据下载中...完成后将会通过<b style=\"color:red\">旺旺</b>和<b style=\"color:red\">邮件</b>通知");
      jsonObj.put("uuid", callback.getUuid());
      jsonObj.put("debug", callback.toString());
      jsonObj.put("timedebug", String.valueOf(t2 - t1));
 
    } else {
      jsonObj.put("code", "0");
      jsonObj.put("message", "还有" + size + "个任务没有完成数据下载,请稍后提交");
    }
 
    if (jsoncallback != null && jsoncallback.length() > 0) {
      return jsoncallback + "(" + jsonObj.toString() + ")";
    } else {
      return jsonObj.toString();
    }
 
 
  }
View Full Code Here


   
    if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_1")) {
      projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
      JSONArray jsonStr=new JSONArray(queryStr.trim());
      JSONObject obj=new JSONObject();
      obj.put("key", "app_type");
      obj.put("operate", "1");
      obj.put("value", "1");
      jsonStr.put(obj);
      queryStr = WebServiceParams.query(jsonStr.toString());
    }
    if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_2")) {
      projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
      JSONArray jsonStr=new JSONArray(queryStr.trim());
      JSONObject obj=new JSONObject();
      obj.put("key", "app_type");
      obj.put("operate", "1");
      obj.put("value", "2");
      jsonStr.put(obj);
      queryStr = WebServiceParams.query(jsonStr.toString());
    }
    if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_3")) {
      projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
      JSONArray jsonStr=new JSONArray(queryStr.trim());
      JSONObject obj=new JSONObject();
      obj.put("key", "app_type");
      obj.put("operate", "1");
      obj.put("value", "3");
      jsonStr.put(obj);
      queryStr = WebServiceParams.query(jsonStr.toString());
    }
   
    if (projectName.equals("fact_wirelesspv_clickcosteffect1_app_adhoc_d_4")) {
      projectName="fact_wirelesspv_clickcosteffect1_app_adhoc_d";
      JSONArray jsonStr=new JSONArray(queryStr.trim());
      JSONObject obj=new JSONObject();
      obj.put("key", "app_type");
      obj.put("operate", "1");
      obj.put("value", "4");
      jsonStr.put(obj);
      queryStr = WebServiceParams.query(jsonStr.toString());
    }
   
   
 
 
    TablePartion part = GetPartions.partion(projectName);
    ShardsList[] cores = GetShards.getCoresNonCheck(part);
    ShardsList[] ms = GetShards.getMergers(part.name);
    String[] partionsAll = AdhocHivePartions.get(queryStr, part.parttype);
    GetPartions.Shards shard = GetPartions.getshard(part, partionsAll,cores, ms);
   
   
   
    LinkedHashMap<String, String> filetypeMap = MdrillFieldInfo.readFieldsFromSchemaXml(stormconf,part.name);
    ArrayList<String> fqList = WebServiceParams.fqListHive(false,hpart,queryStr,
        isPartionByPt, filetypeMap,null,null,null);
   
   
   
   
    StringBuffer sqlWhere =AdhocWebServiceParams.makeWhere(fqList,new ArrayList<String>(),"userid");
 
 
   
   
    ArrayList<String> groupFields = WebServiceParams.groupFields(groupby);
    ArrayList<String> showFields = WebServiceParams.showHiveFields(fl);
    HigoAdhocJoinParams[] joins=AdhocWebServiceParams.parseJoinsHive(leftjoin, shard);
    String daycols=AdhocWebServiceParams.parseDayCols(hpart,groupFields,showFields);
    HashMap<String,String> colMap=new HashMap<String, String>();
    HashMap<String,String> colMapforStatFilter=new HashMap<String, String>();

    StringBuffer cols_inner = new StringBuffer();
    StringBuffer cols = new StringBuffer();
    String join = "";
    AtomicInteger nameindex=new AtomicInteger(0);
    String hql="";
    if(joins.length<=0)
    {
      AdhocWebServiceParams.parseColsNoJoins(cols, groupFields, showFields,colMapforStatFilter, nameindex);
      hql = "select " + cols.toString() + " from " + projectName + " "
      + sqlWhere.toString() + " " ;
     
      StringBuffer sqlGroup = new StringBuffer();
      join = " group by ";
      for (String field : groupFields) {
        sqlGroup.append(join);
        sqlGroup.append(field);
        join = ",";
      }
      hql+=" "+sqlGroup.toString();

    }else
    {
      AdhocWebServiceParams.parseColsWithJoins(cols,cols_inner, joins, colMap,colMapforStatFilter, groupFields, showFields, nameindex);
     
      StringBuffer buffer=new StringBuffer();
      buffer.append("select ");
      buffer.append(cols.toString());
      buffer.append(" from ");
     
      buffer.append(" (select "+cols_inner.toString()+"  from " + projectName + " "  + sqlWhere.toString() + " ) jl1 ");
     
      for(int i=0;i<joins.length;i++)
      {
        HigoAdhocJoinParams jp=joins[i];
        buffer.append(" join ("+jp.frQuer+") jr"+i+" on (jl1."+colMap.get(jp.leftkey)+" is not null and trim(jl1."+colMap.get(jp.leftkey)+")==trim(jr"+i+"."+jp.rightkey+")) ");
      }
     
      hql=buffer.toString();
     
      StringBuffer sqlGroup = new StringBuffer();
      join = " group by ";
      for (String field : groupFields) {
        sqlGroup.append(join);
        sqlGroup.append("jl1."+colMap.get(field));
        join = ",";
      }
     
      if(AdhocWebServiceParams.hasStatFiled(showFields))
      {
        for(int i=0;i<joins.length;i++)
        {
          HigoAdhocJoinParams jp=joins[i];
          for(String s:jp.fl)
          {
            sqlGroup.append(join);
            sqlGroup.append("jr"+i+"."+s);
            join=",";
          }
        }
      }
     
      hql+=" "+sqlGroup.toString();

     
    }
   
   
   
   
     ArrayList<String> fq2list=WebServiceParams.fqListHive(true,hpart,fq2,
          isPartionByPt, filetypeMap,colMap,colMapforStatFilter,"fq2");
     if(fq2list.size()>0||orderby2!=null)
     {
        StringBuffer buffer=new StringBuffer();
        buffer.append("select * from ");
        buffer.append("("+hql+") fq2");
       
        if(fq2list.size()>0)
        {
          String join2 = " where ";
          for (String fq : fq2list) {
            buffer.append(join2);
            buffer.append(fq);
            join2 = " and ";
          }
        }
       
         if(orderby2!=null)
         {
           buffer.append(" order by "+WebServiceParams.parseFqAlias(orderby2, colMap, colMapforStatFilter, "fq2")+" "+desc2);
         }
        hql=buffer.toString();
     }
    
   

    
   
    
     if(limit >1000000)
     {
       limit=1000000;
     }
     if(limit>0)
     {
       hql=hql+" limit "+limit;
     }
    
     hql=hql.replaceAll("dist\\((.*)\\)", "count(distinct($1))");
   
 
    String md5 = MD5.getMD5(hql);
 
    SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
    String day = fmt.format(new Date());
 
    String hdpConf = (String) stormconf.get("hadoop.conf.dir");
    String connstr = (String) stormconf.get("higo.download.offline.conn");
    String uname = (String) stormconf.get("higo.download.offline.username");
    String passwd = (String) stormconf.get("higo.download.offline.passwd");
    String store = (String) stormconf.get("higo.download.offline.store")
        + "/" + day + "/" + java.util.UUID.randomUUID().toString();
    MySqlConn conn = new MySqlConn(connstr, uname, passwd);
    MysqlInfo info = new MysqlInfo(conn);
    if (username == null || username.length() <= 0) {
      username = "default";
    }
 
    StringBuffer sqlbuff = new StringBuffer();
    // int jobsize=info.getUserJobname(username, jobname, sqlbuff).size();
    sqlbuff.append(";");
    int size = info.getUser(username, true, sqlbuff).size();
    JSONObject jsonObj = new JSONObject();
    jsonObj.put("sqlbuff", sqlbuff.toString());
    jsonObj.put("size", size);
    // if(jobsize>0)
    // {
    // jsonObj.put("code", "0");
    // jsonObj.put("message", "之前已经有叫"+jobname+"的任务,请换个名字");
    // }else
    //
    if (size < 5) {
      MysqlCallback callback = new MysqlCallback(conn);
 
      String[] pcols = params == null ? new String[0] : new String(daycols
          + params.replaceAll("维度指标:", "").replaceAll("。.*$", ""))
          .split(",");
      StringBuffer liststat = new StringBuffer();
      StringBuffer listgroup = new StringBuffer();
      for (String s : pcols) {
        if (AdhocOfflineService.isStatFields(s)) {
          liststat.append(s);
          liststat.append(",");
        } else {
          listgroup.append(s);
          listgroup.append(",");
        }
      }
 
      callback.setCols((params == null || params.isEmpty()) ? cols
          .toString() : listgroup.toString() + liststat.toString());
      OfflineDownload download = new OfflineDownload();
      download.setOffline(callback);
      if (mailto == null || mailto.length() <= 0) {
        mailto = "yannian.mu@alipay.com";
      }
     
      StringBuffer setSql=new StringBuffer();
      StringBuffer cleanSql=new StringBuffer();
      for(int i=0;i<joins.length;i++)
      {
        HigoAdhocJoinParams jp=joins[i];
       
        setSql.append(jp.createSql);
        setSql.append(";");
        setSql.append(jp.addData);
        setSql.append(";");

        cleanSql.append(";");
        cleanSql.append(jp.DropSql);
      }
 
      download.setMailto(mailto);
      download.setHql(setSql.toString()+" INSERT OVERWRITE DIRECTORY '" + store + "' " + hql+"  "+cleanSql.toString());
 
      download.setUseName(username);
      if (jobname == null || jobname.length() <= 0) {
        jobname = day + "_" + md5;
      }
      download.setJobName(jobname);
      download.setMemo(String.valueOf(memo));
      download.setDisplayParams((params == null || params.isEmpty()) ? hql
          : params);
      download.setStoreDir(store);
      download.setConfdir(hdpConf);
      download.setSqlMd5(md5);
      download.run();
 
      long t2 = System.currentTimeMillis();
      jsonObj.put("code", "1");
 
      jsonObj.put("message",
          "数据下载中...完成后将会通过<b style=\"color:red\">旺旺</b>和<b style=\"color:red\">邮件</b>通知");
      jsonObj.put("uuid", callback.getUuid());
      jsonObj.put("debug", callback.toString());
      jsonObj.put("timedebug", String.valueOf(t2 - t1));
 
    } else {
      jsonObj.put("code", "0");
      jsonObj.put("message", "还有" + size + "个任务没有完成数据下载,请稍后提交");
    }
 
    if (jsoncallback != null && jsoncallback.length() > 0) {
      return jsoncallback + "(" + jsonObj.toString() + ")";
    } else {
      return jsonObj.toString();
    }
 
  }
View Full Code Here

      if(!s.startsWith("_sb="))
      {
        continue;
      }
      String json=decodeString(s.substring(4));
      return new JSONObject(json);
    }
    return new JSONObject();
  }
View Full Code Here

       if(s==null)
       {
         return null;
       }
       try{
         JSONObject obj= new JSONObject(s);
         if(obj.has(key))
         {
           return String.valueOf(obj.get(key));
         }
       }catch(Throwable e)
       {
         return null;
       }
View Full Code Here

   
   
    public static void readAbtestInfo(String varstr,String uuid, OutputStreamWriter outStream)
    throws SQLException, IOException, JSONException {
 
  JSONObject jsonObjresult = readAbtestInfo(uuid);

 
  if(varstr!=null&&varstr.length()>0)
  {
    outStream.write(""+varstr+"("+jsonObjresult.toString()+");");
  }else{
    outStream.write(jsonObjresult.toString());
  }

}
View Full Code Here

        exehql = result.get("_exehql");
      }

      Path A;
      Path B;
      JSONObject jsonObj = new JSONObject();
      if (result != null && result.containsKey("storedir")
          && result.containsKey("cols")) {
 
       
        Path base=makeAB(result.get("storedir"), conf);
        A=new Path(base,"A");
        B=new Path(base,"B");

        jsonObj.put("A", A.toString());
        jsonObj.put("B", B.toString());
      }else{
        jsonObj.put("A","");
        jsonObj.put("B", "");
      }
     
     
     
      jsonObj.put("params", result);
     
      JSONObject jsonObjresult = new JSONObject();
      jsonObjresult.put("code", 1);
      jsonObjresult.put("data", jsonObj);

     
      return jsonObjresult;

    }
View Full Code Here

      JSONArray jsonStr=new JSONArray(queryStr.trim());
      JSONArray jsonStr_new=new JSONArray();

      for(int j=0;j<jsonStr.length();j++)
      {
        JSONObject obj=jsonStr.getJSONObject(j);
        if(!obj.has("thedate"))
        {
          jsonStr_new.put(obj);
          continue;
        }
       
       

   
      }
     
      for(int j=0;j<jsonStr.length();j++)
      {
        JSONObject obj=jsonStr.getJSONObject(j);
        if(!obj.has("thedate"))
        {
          continue;
        }
       
       

      JSONObject thedate=obj.getJSONObject("thedate");
      Integer operate=Integer.parseInt(thedate.get("operate").toString());
      String[] val=WebServiceParams.parseFqValue(thedate.getString("value"), operate).split(",");
      if(operate==1 ||operate==5||operate==6||operate==7||operate==8||operate==9)//=,range
      {
        Long min=Long.MAX_VALUE;
        Long max=Long.MIN_VALUE;
            for(String day:val)
View Full Code Here

        Long bigger=-1l;
        JSONArray jsonStr=new JSONArray(queryStr.trim());

        for(int j=0;j<jsonStr.length();j++)
        {
          JSONObject obj=jsonStr.getJSONObject(j);
          if(obj.has("thedate"))
          {
            JSONObject thedate=obj.getJSONObject("thedate");
            Integer operate=Integer.parseInt(thedate.get("operate").toString());
            String[] val=WebServiceParams.parseFqValue(thedate.getString("value"), operate).split(",");
            if(operate==1 ||operate==5||operate==6||operate==7||operate==8||operate==9)//=,range
            {
              Long min=Long.MAX_VALUE;
              Long max=Long.MIN_VALUE;
                  for(String day:val)
View Full Code Here

        try {
          String strday=ColsDefine.formatDay.format(new Date(System.currentTimeMillis()-1000l*3600*24*(j+1)));
          String fqstr="[{\"thedate\":{\"operate\":6,\"value\":[\""+strday+"\"]}}]";
          String jsonstr=MdrillService.result("ods_quanjing_416_pc_pid", null, "0", "10000", fqstr, null, "putin_date,pid,url,count(*)", "putin_date,pid,url", "count(*)", "desc", null, null);
          LOG.info("fetch "+j+"@"+jsonstr);
          JSONObject jsonObj = new JSONObject(jsonstr);
          if(!jsonObj.getString("code").equals("1"))
          {
            sleep();
            continue;
          }
   
          JSONArray list=jsonObj.getJSONObject("data").getJSONArray("docs");

          if(list.length()==0)
          {
            sleep();
            continue;
          }
         
          for(int i=0;i<list.length();i++)
          {
            try {
              JSONObject item = list.getJSONObject(i);
              String delivey_date= String.valueOf(item.get("putin_date"));
              String url_lwfrom=ColsDefine.getName(String.valueOf(item.get("url")), "lwfrom");
              String adid = String.valueOf(url_lwfrom);
              String pid = String.valueOf(item.get("pid"));
              rtn.put(delivey_date+"@"+adid, pid);
            } catch (Exception e) {
              LOG.info("fetch parse error", e);
            }
          }
View Full Code Here

        try {
          String strday=ColsDefine.formatDay.format(new Date(System.currentTimeMillis()-1000l*3600*24));
          String fqstr="[{\"thedate\":{\"operate\":1,\"value\":[\""+strday+"\"]}}]";
          String jsonstr=MdrillService.result("ods_quanjing_416_wireless_pid", null, "0", "10000", fqstr, null, "putin_date,pid,lwfrom,count(*)", "putin_date,pid,lwfrom", "count(*)", "desc", null, null);
          LOG.info("fetch "+j+"@"+jsonstr);
          JSONObject jsonObj = new JSONObject(jsonstr);
          if(!jsonObj.getString("code").equals("1"))
          {
            sleep();
            continue;
          }
   
          JSONArray list=jsonObj.getJSONObject("data").getJSONArray("docs");

          if(list.length()==0)
          {
            sleep();
            continue;
          }
         
          for(int i=0;i<list.length();i++)
          {
            try {
              JSONObject item = list.getJSONObject(i);
              String delivey_date= String.valueOf(item.get("putin_date"));
              String adid = String.valueOf(item.get("lwfrom"));
              String pid = String.valueOf(item.get("pid"));
              rtn.put(delivey_date+"@"+adid, pid);
            } catch (Exception e) {
              LOG.info("fetch parse error", e);
            }
          }
View Full Code Here

TOP

Related Classes of com.alimama.mdrill.json.JSONObject

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.