Examples of PreOrderWalker

org.apache.hadoop.hive.ql.lib.PreOrderWalker
base class for operator graph walker this class takes list of starting ops and walks them one by one.

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      ReduceWorkVectorizationNodeProcessor vnp =
              new ReduceWorkVectorizationNodeProcessor(reduceColumnNames);
      addReduceWorkRules(opRules, vnp);
      Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
      GraphWalker ogw = new PreOrderWalker(disp);
      // iterator the reduce operator tree
      ArrayList<Node> topNodes = new ArrayList<Node>();
      topNodes.add(reduceWork.getReducer());
      LOG.info("vectorizeReduceWork reducer Operator: " +
              reduceWork.getReducer().getName() + "...");
      HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
      ogw.startWalking(topNodes, nodeOutput);


      // Necessary since we are vectorizing the root operator in reduce.
      reduceWork.setReducer(vnp.getRootVectorOp());


      Operator<? extends OperatorDesc> reducer = reduceWork.getReducer();

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker


    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(OpTraitsRulesProcFactory.getDefaultRule(), opRules,
        annotateCtx);
    GraphWalker ogw = new PreOrderWalker(disp);


    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);


    return pctx;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

      LOG.info("Looking for table scans where optimization is applicable");


      // The dispatcher fires the processor corresponding to the closest
      // matching rule and passes the context along
      Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
      GraphWalker ogw = new PreOrderWalker(disp);


      // Create a list of topOp nodes
      ArrayList<Node> topNodes = new ArrayList<Node>();
      // Get the top Nodes for this task
      for (Operator<? extends OperatorDesc>
             workOperator : topOperators) {
        if (parseContext.getTopOps().values().contains(workOperator)) {
          topNodes.add(workOperator);
        }
      }


      Operator<? extends OperatorDesc> reducer = task.getReducer(mapWork);
      if (reducer != null) {
        topNodes.add(reducer);
      }


      ogw.startWalking(topNodes, null);


      LOG.info(String.format("Found %d null table scans",
          walkerCtx.getMetadataOnlyTableScans().size()));
      Iterator<TableScanOperator> iterator
        = walkerCtx.getMetadataOnlyTableScans().iterator();

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

    if(!isNoArgumentMacro) {
      /*
       * Walk down expression to see which arguments are actually used.
       */
      Node expression = (Node) ast.getChild(2);
      PreOrderWalker walker = new PreOrderWalker(new Dispatcher() {
        @Override
        public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
            throws SemanticException {
          if(nd instanceof ASTNode) {
            ASTNode node = (ASTNode)nd;
            if(node.getType() == HiveParser.TOK_TABLE_OR_COL) {
              actualColumnNames.add(node.getChild(0).getText());
            }
          }
          return null;
        }
      });
      walker.startWalking(Collections.singletonList(expression), null);
    }
    for (FieldSchema argument : arguments) {
      TypeInfo colType =
          TypeInfoUtils.getTypeInfoFromTypeString(argument.getType());
      rowResolver.put("", argument.getName(),

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

        RewriteCanApplyProcFactory.canApplyOnSelectOperator());


    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this);
    GraphWalker ogw = new PreOrderWalker(disp);


    // Create a list of topop nodes
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.add(topOp);


    try {
      ogw.startWalking(topNodes, null);
    } catch (SemanticException e) {
      LOG.error("Exception in walking operator tree. Rewrite variables not populated");
      LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
      throw new SemanticException(e.getMessage(), e);
    }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

    String FS = FileSinkOperator.getOperatorName() + "%";


    opRules.put(new RuleRegExp("Sorted Dynamic Partition", FS), getSortDynPartProc(pCtx));


    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new PreOrderWalker(disp);


    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);


    return pCtx;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

    String FS = FileSinkOperator.getOperatorName() + "%";


    opRules.put(new RuleRegExp("Sorted Dynamic Partition", FS), getSortDynPartProc(pCtx));


    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new PreOrderWalker(disp);


    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);


    return pCtx;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

      opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" +
          ReduceSinkOperator.getOperatorName()), vnp);
      opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*"
          + FileSinkOperator.getOperatorName()), vnp);
      Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
      GraphWalker ogw = new PreOrderWalker(disp);
      // iterator the mapper operator tree
      ArrayList<Node> topNodes = new ArrayList<Node>();
      topNodes.addAll(mapWork.getAliasToWork().values());
      HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
      ogw.startWalking(topNodes, nodeOutput);


      Map<String, Map<Integer, String>> columnVectorTypes = vnp.getScratchColumnVectorTypes();
      mapWork.setScratchColumnVectorTypes(columnVectorTypes);
      Map<String, Map<String, Integer>> columnMap = vnp.getScratchColumnMap();
      mapWork.setScratchColumnMap(columnMap);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

        new FileSinkProcessor());


      // The dispatcher fires the processor corresponding to the closest
      // matching rule and passes the context along
      Dispatcher disp = new DefaultRuleDispatcher(null, opRules, walkerCtx);
      GraphWalker ogw = new PreOrderWalker(disp);


      // Create a list of topOp nodes
      ArrayList<Node> topNodes = new ArrayList<Node>();
      // Get the top Nodes for this map-reduce task
      for (Operator<? extends OperatorDesc>
           workOperator : topOperators) {
        if (parseContext.getTopOps().values().contains(workOperator)) {
          topNodes.add(workOperator);
        }
      }


      if (task.getReducer() != null) {
        topNodes.add(task.getReducer());
      }


      ogw.startWalking(topNodes, null);


      LOG.info(String.format("Found %d metadata only table scans",
          walkerCtx.getMetadataOnlyTableScans().size()));
      Iterator<TableScanOperator> iterator
        = walkerCtx.getMetadataOnlyTableScans().iterator();

View Full Code Here

Examples of org.apache.hadoop.hive.ql.lib.PreOrderWalker

    if(!isNoArgumentMacro) {
      /*
       * Walk down expression to see which arguments are actually used.
       */
      Node expression = (Node) ast.getChild(2);
      PreOrderWalker walker = new PreOrderWalker(new Dispatcher() {
        @Override
        public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
            throws SemanticException {
          if(nd instanceof ASTNode) {
            ASTNode node = (ASTNode)nd;
            if(node.getType() == HiveParser.TOK_TABLE_OR_COL) {
              actualColumnNames.add(node.getChild(0).getText());
            }
          }
          return null;
        }
      });
      walker.startWalking(Collections.singletonList(expression), null);
    }
    for (FieldSchema argument : arguments) {
      TypeInfo colType =
          TypeInfoUtils.getTypeInfoFromTypeString(argument.getType());
      rowResolver.put("", argument.getName(),

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.