Package org.apache.hadoop.hive.ql.plan.PTFDesc

Examples of org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef


      if (!conf.forWindowing()) {
        return getDefaultProc().process(nd, stack, ctx, nodeOutputs);
      }

      ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
      WindowTableFunctionDef def = (WindowTableFunctionDef) conf.getFuncDef();
      ArrayList<ColumnInfo> sig = new ArrayList<ColumnInfo>();

      List<String> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0));
      //we create a copy of prunedCols to create a list of pruned columns for PTFOperator
      prunedCols = new ArrayList<String>(prunedCols);
View Full Code Here


    init(semAly, hCfg, inputRR, unparseT);
    windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);

    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();

    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver)
        FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
      throw new SemanticException(String.format("Internal Error: Unknown Table Function %s",
          FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);

    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if (partiSpec == null) {
      throw new SemanticException(
          "Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);

    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);

    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if (wdwSpec.getWindowExpressions() != null) {
      for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
        if (expr instanceof WindowFunctionSpec) {
          WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec) expr);
          windowFunctions.add(wFnDef);
        }
      }
      wdwTFnDef.setWindowFunctions(windowFunctions);
    }

    /*
     * set outputFromWdwFnProcessing
     */
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : windowFunctions) {
      aliases.add(wFnDef.getAlias());
      if (wFnDef.isPivotResult()) {
        fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
      } else {
        fieldOIs.add(wFnDef.getOI());
      }
    }
    PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
        aliases, fieldOIs);
    tFn.setWdwProcessingOutputOI(wdwOutOI);
    RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
    ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
    wdwTFnDef.setOutputShape(wdwOutShape);

    tFn.setupOutputOI();

    return ptfDesc;
  }
View Full Code Here

  @Override
  public PTFPartition execute(PTFPartition iPart)
      throws HiveException
  {
    WindowTableFunctionDef wFnDef = (WindowTableFunctionDef) getTableDef();
    PTFPartitionIterator<Object> pItr = iPart.iterator();
    PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);

    if ( outputPartition == null ) {
      outputPartition = new PTFPartition(getPartitionClass(),
          getPartitionMemSize(), wFnDef.getOutputFromWdwFnProcessing().getSerde(), OI);
    }
    else {
      outputPartition.reset();
    }
View Full Code Here

      inputOI = (StructObjectInspector) iPart.getSerDe().getObjectInspector();
    } catch (SerDeException se) {
      throw new HiveException(se);
    }

    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
    Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder();

    for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions())
    {
      boolean processWindow = processWindow(wFn);
      pItr.reset();
      if ( !processWindow )
      {
View Full Code Here

   * @throws HiveException
   */
  private void executeWindowExprs(PTFPartition oPart)
      throws HiveException
  {
    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) conf.getFuncDef();
    /*
     * inputOI represents the row with WindowFn results present.
     * So in the e.g. above it will have a column for 'rank()'
     */
    StructObjectInspector inputOI = wTFnDef.getOutputFromWdwFnProcessing().getOI();
    /*
     * outputOI represents the final row with the Windowing Expressions added.
     * So in the e.g. above it will have a column for 'lead(rank())' in addition to
     * all columns in inputOI.
     */
    StructObjectInspector outputOI = wTFnDef.getOutputShape().getOI();
    int numCols = outputOI.getAllStructFieldRefs().size();
    ArrayList<WindowExpressionDef> wdwExprs = wTFnDef.getWindowExpressions();
    int numWdwExprs = wdwExprs == null ? 0 : wdwExprs.size();
    Object[] output = new Object[numCols];

    /*
     * If this Windowing invocation has no Window Expressions and doesn't need to be filtered,
View Full Code Here

    ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
    PTFPartition iPart = pItr.getPartition();
    StructObjectInspector inputOI;
    inputOI = (StructObjectInspector) iPart.getOutputOI();

    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
    Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder();

    for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) {
      boolean processWindow = processWindow(wFn);
      pItr.reset();
      if ( !processWindow ) {
        GenericUDAFEvaluator fEval = wFn.getWFnEval();
        Object[] args = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()];
View Full Code Here

    init(semAly, hCfg, inputRR, unparseT);
    windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);

    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();

    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver)
        FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
      throw new SemanticException(String.format("Internal Error: Unknown Table Function %s",
          FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);

    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if (partiSpec == null) {
      throw new SemanticException(
          "Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);

    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);

    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if (wdwSpec.getWindowExpressions() != null) {
      for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
        if (expr instanceof WindowFunctionSpec) {
          WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec) expr);
          windowFunctions.add(wFnDef);
        }
      }
      wdwTFnDef.setWindowFunctions(windowFunctions);
    }

    /*
     * set outputFromWdwFnProcessing
     */
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : windowFunctions) {
      aliases.add(wFnDef.getAlias());
      if (wFnDef.isPivotResult()) {
        fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
      } else {
        fieldOIs.add(wFnDef.getOI());
      }
    }
    PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
        aliases, fieldOIs);
    tFn.setWdwProcessingOutputOI(wdwOutOI);
    RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
    ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
    wdwTFnDef.setOutputShape(wdwOutShape);

    tFn.setupOutputOI();

    return ptfDesc;
  }
View Full Code Here

      if (!conf.forWindowing()) {
        return getDefaultProc().process(nd, stack, ctx, nodeOutputs);
      }

      ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
      WindowTableFunctionDef def = (WindowTableFunctionDef) conf.getFuncDef();
      ArrayList<ColumnInfo> sig = new ArrayList<ColumnInfo>();

      List<String> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0));
      //we create a copy of prunedCols to create a list of pruned columns for PTFOperator
      prunedCols = new ArrayList<String>(prunedCols);
View Full Code Here

      throws SemanticException {
    init(semAly, hCfg, inputRR, unparseT);
    this.windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);

    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();

    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver)
        FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
      throw new SemanticException(String.format("INternal Error: Unknown Table Function %s",
          FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1 );
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);

    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if ( partiSpec == null ) {
      throw new SemanticException(
          "Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);

    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);

    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if ( wdwSpec.getWindowExpressions() != null ) {
      for(WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
        if ( expr instanceof WindowFunctionSpec) {
          WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec)expr);
          windowFunctions.add(wFnDef);
        }
      }
      wdwTFnDef.setWindowFunctions(windowFunctions);
    }

    /*
     * set outputFromWdwFnProcessing
     */
    if ( windowFunctions.size() > 0 ) {
      ArrayList<String> aliases = new ArrayList<String>();
      ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
      for(WindowFunctionDef wFnDef : windowFunctions) {
        aliases.add(wFnDef.getAlias());
        if ( wFnDef.isPivotResult() ) {
          fieldOIs.add(((ListObjectInspector)wFnDef.getOI()).getListElementObjectInspector());
        } else {
          fieldOIs.add(wFnDef.getOI());
        }
      }
      PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
      StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
          aliases, fieldOIs);
      tFn.setWdwProcessingOutputOI(wdwOutOI);
      RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef, false);
      ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
      wdwTFnDef.setOutputFromWdwFnProcessing(wdwOutShape);
    }
    else {
      wdwTFnDef.setOutputFromWdwFnProcessing(inpShape);
    }

    /*
     * process Wdw expressions
     */
    ShapeDetails wdwOutShape = wdwTFnDef.getOutputFromWdwFnProcessing();
    ArrayList<WindowExpressionDef> windowExpressions = new ArrayList<WindowExpressionDef>();
    if ( wdwSpec.getWindowExpressions() != null ) {
      for(WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
        if ( !(expr instanceof WindowFunctionSpec) ) {
          try {
            PTFExpressionDef eDef = buildExpressionDef(wdwOutShape, expr.getExpression());
            WindowExpressionDef wdwEDef = new WindowExpressionDef(eDef);
            wdwEDef.setAlias(expr.getAlias());
            windowExpressions.add(wdwEDef);
          }
          catch(HiveException he) {
            throw new SemanticException(he);
          }
        }
      }
      wdwTFnDef.setWindowExpressions(windowExpressions);
    }

    /*
     * set outputOI
     */
    if ( windowExpressions.size() > 0 ) {
      ArrayList<String> aliases = new ArrayList<String>();
      ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
      for(WindowExpressionDef wEDef : windowExpressions) {
        aliases.add(wEDef.getAlias());
        fieldOIs.add(wEDef.getOI());
      }
      PTFTranslator.addInputColumnsToList(wdwOutShape, aliases, fieldOIs);
      StructObjectInspector outOI = ObjectInspectorFactory.getStandardStructObjectInspector(
          aliases, fieldOIs);
      RowResolver outRR = buildRowResolverForWindowing(wdwTFnDef, true);
      ShapeDetails outShape = setupShape(outOI, null, outRR);
      wdwTFnDef.setOutputShape(outShape);
    }
    else {
      wdwTFnDef.setOutputShape(copyShape(wdwOutShape));
    }

    tFn.setupOutputOI();

    /*
     * If we have windowExpressions then we convert to Std. Object to process;
     * we just stream these rows; no need to put in an output Partition.
     */
    if ( windowExpressions.size() > 0 ) {
      StructObjectInspector oi = (StructObjectInspector)
          ObjectInspectorUtils.getStandardObjectInspector(wdwTFnDef.getOutputShape().getOI());
      wdwTFnDef.getOutputShape().setOI(oi);
    }

    return ptfDesc;
  }
View Full Code Here

      if (!conf.forWindowing()) {
        return getDefaultProc().process(nd, stack, ctx, nodeOutputs);
      }

      ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
      WindowTableFunctionDef def = (WindowTableFunctionDef) conf.getFuncDef();
      ArrayList<ColumnInfo> sig = new ArrayList<ColumnInfo>();

      List<String> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0));
      //we create a copy of prunedCols to create a list of pruned columns for PTFOperator
      prunedCols = new ArrayList<String>(prunedCols);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.