Package com.sap.hadoop.windowing.query2.translate

Examples of com.sap.hadoop.windowing.query2.translate.WindowFunctionTranslation


     */
    @Override
    public void setupOutputOI() throws WindowingException
    {
      NPath evaluator = (NPath) getEvaluator();
      TableFuncDef tDef = evaluator.getTableDef();
     
      ArrayList<ArgDef> args = tDef.getArgs();
      int argsNum = args == null ? 0 : args.size();
     
      if ( argsNum < 4 )
      {
        throwErrorWithSignature("at least 4 arguments required");
      }
     
      /*
       * validate and setup patternStr
       */
      ArgDef symboPatternArg = args.get(0);
      ObjectInspector symbolPatternArgOI = symboPatternArg.getOI();
     
      if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) ||
          (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
          ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
      {
        throwErrorWithSignature("Currently the symbol Pattern must be a Constant String.");
      }
     
      evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).getWritableConstantValue().toString();
     
      /*
       * validate and setup SymbolInfo
       */
      int symbolArgsSz = argsNum - 2;
      if ( symbolArgsSz % 2 != 0)
      {
        throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: there are odd number of symbol args");
      }
     
      evaluator.symInfo = new SymbolsInfo(symbolArgsSz/2);
      for(int i=1; i <= symbolArgsSz; i += 2)
      {
        ArgDef symbolNameArg = args.get(i);
        ObjectInspector symbolNameArgOI = symbolNameArg.getOI();
       
        if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) ||
            (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
            ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
        {
          throwErrorWithSignature(sprintf("Currently a Symbol Name(%s) must be a Constant String", symbolNameArg.getExpression().toStringTree()));
        }
        String symbolName = ((ConstantObjectInspector)symbolNameArgOI).getWritableConstantValue().toString();
       
        ArgDef symolExprArg = args.get(i+1);
        ObjectInspector symolExprArgOI = symolExprArg.getOI();
        if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
              ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN )
        {
          throwErrorWithSignature(sprintf("Currently a Symbol Expression(%s) must be a boolean expression", symolExprArg.getExpression().toStringTree()));
        }
        evaluator.symInfo.add(symbolName, symolExprArg);
      }
     
      /*
       * validate and setup resultExprStr
       */
      ArgDef resultExprArg = args.get(argsNum - 1);
      ObjectInspector resultExprArgOI = resultExprArg.getOI();
     
      if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) ||
            (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
            ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
      {
        throwErrorWithSignature("Currently the result Expr parameter must be a Constant String.");
      }
       
      evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).getWritableConstantValue().toString();
     
      /*
       * setup SymbolFunction chain.
       */
      SymbolParser syP = new SymbolParser(evaluator.patternStr,
          evaluator.symInfo.symbolExprsNames,
          evaluator.symInfo.symbolExprsEvaluators, evaluator.symInfo.symbolExprsOIs);
      syP.parse();
      evaluator.syFn = syP.getSymbolFunction();
     
      /*
       * setup OI for input to resultExpr select list
       */
      StructObjectInspector selectListInputOI = (StructObjectInspector) NPathUtils.createSelectListInputOI(tDef.getInput().getOI());
     
      /*
       * parse ResultExpr Str and setup OI.
       */
      ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputOI);
View Full Code Here


    public QueryInputDef next()
    {
      QueryInputDef curr = nextInput;
      if ( curr instanceof TableFuncDef)
      {
        TableFuncDef tFunc = (TableFuncDef) curr;
        nextInput = tFunc.getInput();
      }
      else
      {
        nextInput = null;
      }
View Full Code Here

    }
    // OI for FileSinkOperator is taken from select-list (reduce-side)
    // OI for ReduceSinkOperator is taken from TODO
    if (isMapOperator)
    {
      TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
      outputObjInspector = tDef.getMapOI();
    }
    else
    {
      outputObjInspector = qDef.getSelectList().getOI();
    }
View Full Code Here

 
  protected void processMapFunction() throws HiveException
  {
    try
    {
      TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
      Partition outPart = tDef.getFunction().transformRawInput(inputPart);
      PartitionIterator<Object> pItr = outPart.iterator();
      while (pItr.hasNext())
      {
        Object oRow = pItr.next();
        forward(oRow, outputObjInspector);
View Full Code Here

   * @throws WindowingException
   */
  public static boolean addPTFMapOperator(QueryDef qdef) throws WindowingException
  {
    boolean hasMap = false;
    TableFuncDef tabDef = RuntimeUtils.getFirstTableFunction(qdef);
    TableFunctionEvaluator tEval = tabDef.getFunction();
    if (tEval.isTransformsRawInput())
    {
      hasMap = true;
    }
    return hasMap;
View Full Code Here

   * @throws WindowingException
   */
  public void initialize() throws WindowingException
  {

    TableFuncDef tabDef = RuntimeUtils.getFirstTableFunction(qdef);
    hiveTableDef = tabDef.getHiveTableDef();
    InputInfo inputInfo;
    ArrayList<ColumnDef> partColList = tabDef.getWindow().getPartDef()
        .getColumns();

    TableFunctionEvaluator tEval = tabDef.getFunction();

    /*
     * If the query has a map phase, the inputInfo is retrieved from the map
     * output info of the table function definition. This is constructed
     * using the map output oi of the table function definition. If the
     * query does not have a map phase, the inputInfo is retrieved from the
     * QueryInputDef (either HiveTableDef or HiveQueryDef) of the query.
     */
    if (tEval.isTransformsRawInput())
    {
      inputInfo = qdef.getTranslationInfo().getMapInputInfo(tabDef);
    }
    else
    {
      inputInfo = qdef.getTranslationInfo().getInputInfo(hiveTableDef);
    }

    for (ColumnDef colDef : partColList)
    {
      partCols.add(colDef.getExprNode());
    }

    ArrayList<OrderColumnDef> orderColList = tabDef.getWindow()
        .getOrderDef().getColumns();

    for (OrderColumnDef colDef : orderColList)
    {
      Order order = colDef.getOrder();
View Full Code Here

    if ( tFn == null)
    {
      throw new WindowingException(sprintf("Unknown Table Function %s", tSpec.getName()));
    }
   
    TableFuncDef tDef = new TableFuncDef();
    tDef.setSpec(tSpec);
    tDef.setInput(inputDef);
    InputInfo iInfo = tInfo.getInputInfo(inputDef);
   
    /*
     * translate args
     */
    ArrayList<ASTNode> args = tSpec.getArgs();
    if ( args != null)
    {
      for(ASTNode expr : args)
      {
        ArgDef argDef = translateTableFunctionArg(qDef, tDef, iInfo,  expr);
        tDef.addArg(argDef);
      }
    }
   
    tFn.initialize(qDef, tDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();   
    tDef.setFunction(tEval);
    tFn.setupRawInputOI();
    tDef.setWindow(WindowSpecTranslation.translateWindow(qDef, tDef));
    tFn.setupOutputOI();
    TranslateUtils.setupSerdeAndOI(tDef, inputDef, tInfo, tEval);

    return tDef;
  }
View Full Code Here

  public PartitionsIterator(WindowingInput wIn, QueryDef qDef) throws WindowingException
  {
    super();
    this.wIn = wIn;
    this.qDef = qDef;
    TableFuncDef tabDef = (TableFuncDef) qDef.getInput();
    TableFunctionEvaluator tEval = tabDef.getFunction();
    partClassName = tEval.getPartitionClass();
    partMemSize = tEval.getPartitionMemSize();
   
    serDe = (SerDe) wIn.getDeserializer();
    try
    {
      OI = (StructObjectInspector) serDe.getObjectInspector();
      w  = wIn.createRow();
    }
    catch(Exception se)
    {
      throw new WindowingException(se);
    }
   
    stdOI = (StructObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(OI);
   
    partColumns = new ArrayList<String>();
    objFields = new ArrayList<StructField>();
    stdObjFields = new ArrayList<StructField>();
    ArrayList<ColumnDef> cols = tabDef.getWindow().getPartDef().getColumns();
    for(ColumnDef colDef : cols)
    {
      String colName = colDef.getAlias();
      StructField f = OI.getStructFieldRef(colName);
      StructField stdF = stdOI.getStructFieldRef(colName);
View Full Code Here

   
    ASTNode wExpr = spec.getWhereExpr();
   
    if ( wExpr == null ) return;
   
    WhereDef whDef = new WhereDef();
    whDef.setExpression(wExpr);
   
    QueryInputDef iDef = qDef.getInput();
    InputInfo iInfo = tInfo.getInputInfo(iDef);
   
    ExprNodeDesc exprNode = TranslateUtils.buildExprNode(wExpr, iInfo.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(tInfo, exprNode);
    ObjectInspector oi = TranslateUtils.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo);
   
    try
    {
      ObjectInspectorConverters.getConverter(oi, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    }
    catch(Throwable t)
    {
      throw new WindowingException("Where Expr must be convertible to a boolean value", t);
    }
   
    whDef.setExprNode(exprNode);
    whDef.setExprEvaluator(exprEval);
    whDef.setOI(oi);
   
    qDef.setWhere(whDef);
  }
View Full Code Here

    if ( RANKING_FUNCS.contains(wFnSpec.getName()))
    {
      setupRankingArgs(qDef, windowTableFnDef, wFnDef, wFnSpec);
    }
   
    WindowDef wDef = translateWindowSpec(qDef, iInfo, wFnSpec);
    wFnDef.setWindow(wDef);
    validateWindowDefForWFn(windowTableFnDef, wFnDef);
   
    setupEvaluator(wFnDef);
   
View Full Code Here

TOP

Related Classes of com.sap.hadoop.windowing.query2.translate.WindowFunctionTranslation

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.