Package com.sap.hadoop.windowing.query2.definition

Examples of com.sap.hadoop.windowing.query2.definition.ColumnDef


   * @return
   * @throws WindowingException
   */
  static TableDesc createOutputTableDesc(QueryDef qDef) throws WindowingException
  {
    QueryOutputDef oDef = qDef.getOutput();
    Class<? extends SerDe> serDeClass = oDef.getSerDe().getClass();
    Properties p = oDef.getSpec().getSerDeProps();
    String columnNamesList = p.getProperty(Constants.LIST_COLUMNS);
    String columnTypesList = p.getProperty(Constants.LIST_COLUMN_TYPES);
    String fieldSeparator = p.getProperty(Constants.FIELD_DELIM, Integer.toString(Utilities.ctrlaCode));
    return PlanUtils.getTableDesc(serDeClass, fieldSeparator,
            columnNamesList, columnTypesList, false);
View Full Code Here


  public static void translateSelectExprs(QueryDef qDef) throws WindowingException
  {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    QueryInputDef iDef = qDef.getInput();
    InputInfo iInfo = tInfo.getInputInfo(iDef);
    SelectDef selectDef = qDef.getSelectList();
    SelectSpec selectSpec = qDef.getSpec().getSelectList();
    Iterator<Object> selectExprsAndAliases = selectSpec.getColumnListAndAlias();
    int i = 0;
    ColumnDef cDef = null;
   
   
    while(selectExprsAndAliases.hasNext())
    {
      Object[] o = (Object[]) selectExprsAndAliases.next();
      boolean isWnFn = ((Boolean) o[0]).booleanValue();
     
      if ( isWnFn )
      {
        cDef = translateWindowFnAlias(qDef, iInfo, i++, (String) o[1]);
      }
      else
      {
        cDef = translateSelectExpr(qDef, iInfo, i++, (String) o[1], (ASTNode) o[2]);
      }
      selectDef.addColumn(cDef);
    }
    TranslateUtils.setupSelectOI(selectDef);
   
  }
View Full Code Here

     */
    @Override
    public void setupOutputOI() throws WindowingException
    {
      NPath evaluator = (NPath) getEvaluator();
      TableFuncDef tDef = evaluator.getTableDef();
     
      ArrayList<ArgDef> args = tDef.getArgs();
      int argsNum = args == null ? 0 : args.size();
     
      if ( argsNum < 4 )
      {
        throwErrorWithSignature("at least 4 arguments required");
      }
     
      /*
       * validate and setup patternStr
       */
      ArgDef symboPatternArg = args.get(0);
      ObjectInspector symbolPatternArgOI = symboPatternArg.getOI();
     
      if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) ||
          (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
          ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
      {
        throwErrorWithSignature("Currently the symbol Pattern must be a Constant String.");
      }
     
      evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).getWritableConstantValue().toString();
     
      /*
       * validate and setup SymbolInfo
       */
      int symbolArgsSz = argsNum - 2;
      if ( symbolArgsSz % 2 != 0)
      {
        throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: there are odd number of symbol args");
      }
     
      evaluator.symInfo = new SymbolsInfo(symbolArgsSz/2);
      for(int i=1; i <= symbolArgsSz; i += 2)
      {
        ArgDef symbolNameArg = args.get(i);
        ObjectInspector symbolNameArgOI = symbolNameArg.getOI();
       
        if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) ||
            (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
            ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
        {
          throwErrorWithSignature(sprintf("Currently a Symbol Name(%s) must be a Constant String", symbolNameArg.getExpression().toStringTree()));
        }
        String symbolName = ((ConstantObjectInspector)symbolNameArgOI).getWritableConstantValue().toString();
       
        ArgDef symolExprArg = args.get(i+1);
        ObjectInspector symolExprArgOI = symolExprArg.getOI();
        if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
              ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN )
        {
          throwErrorWithSignature(sprintf("Currently a Symbol Expression(%s) must be a boolean expression", symolExprArg.getExpression().toStringTree()));
        }
        evaluator.symInfo.add(symbolName, symolExprArg);
      }
     
      /*
       * validate and setup resultExprStr
       */
      ArgDef resultExprArg = args.get(argsNum - 1);
      ObjectInspector resultExprArgOI = resultExprArg.getOI();
     
      if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) ||
            (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
            ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
      {
        throwErrorWithSignature("Currently the result Expr parameter must be a Constant String.");
      }
       
      evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).getWritableConstantValue().toString();
     
      /*
       * setup SymbolFunction chain.
       */
      SymbolParser syP = new SymbolParser(evaluator.patternStr,
          evaluator.symInfo.symbolExprsNames,
          evaluator.symInfo.symbolExprsEvaluators, evaluator.symInfo.symbolExprsOIs);
      syP.parse();
      evaluator.syFn = syP.getSymbolFunction();
     
      /*
       * setup OI for input to resultExpr select list
       */
      StructObjectInspector selectListInputOI = (StructObjectInspector) NPathUtils.createSelectListInputOI(tDef.getInput().getOI());
     
      /*
       * parse ResultExpr Str and setup OI.
       */
      ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputOI);
View Full Code Here

    public QueryInputDef next()
    {
      QueryInputDef curr = nextInput;
      if ( curr instanceof TableFuncDef)
      {
        TableFuncDef tFunc = (TableFuncDef) curr;
        nextInput = tFunc.getInput();
      }
      else
      {
        nextInput = null;
      }
View Full Code Here

    }
    // OI for FileSinkOperator is taken from select-list (reduce-side)
    // OI for ReduceSinkOperator is taken from TODO
    if (isMapOperator)
    {
      TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
      outputObjInspector = tDef.getMapOI();
    }
    else
    {
      outputObjInspector = qDef.getSelectList().getOI();
    }
View Full Code Here

 
  protected void processMapFunction() throws HiveException
  {
    try
    {
      TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
      Partition outPart = tDef.getFunction().transformRawInput(inputPart);
      PartitionIterator<Object> pItr = outPart.iterator();
      while (pItr.hasNext())
      {
        Object oRow = pItr.next();
        forward(oRow, outputObjInspector);
View Full Code Here

   * @throws WindowingException
   */
  public static boolean addPTFMapOperator(QueryDef qdef) throws WindowingException
  {
    boolean hasMap = false;
    TableFuncDef tabDef = RuntimeUtils.getFirstTableFunction(qdef);
    TableFunctionEvaluator tEval = tabDef.getFunction();
    if (tEval.isTransformsRawInput())
    {
      hasMap = true;
    }
    return hasMap;
View Full Code Here

   * @throws WindowingException
   */
  public void initialize() throws WindowingException
  {

    TableFuncDef tabDef = RuntimeUtils.getFirstTableFunction(qdef);
    hiveTableDef = tabDef.getHiveTableDef();
    InputInfo inputInfo;
    ArrayList<ColumnDef> partColList = tabDef.getWindow().getPartDef()
        .getColumns();

    TableFunctionEvaluator tEval = tabDef.getFunction();

    /*
     * If the query has a map phase, the inputInfo is retrieved from the map
     * output info of the table function definition. This is constructed
     * using the map output oi of the table function definition. If the
     * query does not have a map phase, the inputInfo is retrieved from the
     * QueryInputDef (either HiveTableDef or HiveQueryDef) of the query.
     */
    if (tEval.isTransformsRawInput())
    {
      inputInfo = qdef.getTranslationInfo().getMapInputInfo(tabDef);
    }
    else
    {
      inputInfo = qdef.getTranslationInfo().getInputInfo(hiveTableDef);
    }

    for (ColumnDef colDef : partColList)
    {
      partCols.add(colDef.getExprNode());
    }

    ArrayList<OrderColumnDef> orderColList = tabDef.getWindow()
        .getOrderDef().getColumns();

    for (OrderColumnDef colDef : orderColList)
    {
      Order order = colDef.getOrder();
View Full Code Here

    if ( tFn == null)
    {
      throw new WindowingException(sprintf("Unknown Table Function %s", tSpec.getName()));
    }
   
    TableFuncDef tDef = new TableFuncDef();
    tDef.setSpec(tSpec);
    tDef.setInput(inputDef);
    InputInfo iInfo = tInfo.getInputInfo(inputDef);
   
    /*
     * translate args
     */
    ArrayList<ASTNode> args = tSpec.getArgs();
    if ( args != null)
    {
      for(ASTNode expr : args)
      {
        ArgDef argDef = translateTableFunctionArg(qDef, tDef, iInfo,  expr);
        tDef.addArg(argDef);
      }
    }
   
    tFn.initialize(qDef, tDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();   
    tDef.setFunction(tEval);
    tFn.setupRawInputOI();
    tDef.setWindow(WindowSpecTranslation.translateWindow(qDef, tDef));
    tFn.setupOutputOI();
    TranslateUtils.setupSerdeAndOI(tDef, inputDef, tInfo, tEval);

    return tDef;
  }
View Full Code Here

  public PartitionsIterator(WindowingInput wIn, QueryDef qDef) throws WindowingException
  {
    super();
    this.wIn = wIn;
    this.qDef = qDef;
    TableFuncDef tabDef = (TableFuncDef) qDef.getInput();
    TableFunctionEvaluator tEval = tabDef.getFunction();
    partClassName = tEval.getPartitionClass();
    partMemSize = tEval.getPartitionMemSize();
   
    serDe = (SerDe) wIn.getDeserializer();
    try
    {
      OI = (StructObjectInspector) serDe.getObjectInspector();
      w  = wIn.createRow();
    }
    catch(Exception se)
    {
      throw new WindowingException(se);
    }
   
    stdOI = (StructObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(OI);
   
    partColumns = new ArrayList<String>();
    objFields = new ArrayList<StructField>();
    stdObjFields = new ArrayList<StructField>();
    ArrayList<ColumnDef> cols = tabDef.getWindow().getPartDef().getColumns();
    for(ColumnDef colDef : cols)
    {
      String colName = colDef.getAlias();
      StructField f = OI.getStructFieldRef(colName);
      StructField stdF = stdOI.getStructFieldRef(colName);
View Full Code Here

TOP

Related Classes of com.sap.hadoop.windowing.query2.definition.ColumnDef

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.