Package org.apache.hadoop.hive.ql.lib

Examples of org.apache.hadoop.hive.ql.lib.GraphWalker

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    // context along
    UnionProcContext uCtx = new UnionProcContext();
    uCtx.setParseContext(pCtx);
    Dispatcher disp = new DefaultRuleDispatcher(UnionProcFactory.getNoUnion(),
        opRules, uCtx);
    GraphWalker ogw = new PreOrderWalker(disp);

    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    pCtx.setUCtx(uCtx);

    // Walk the tree again to see if the union can be removed completely
    HiveConf conf = pCtx.getConf();
    opRules.clear();
    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE)) {

      if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES)) {
        throw new
        SemanticException(ErrorMsg.HIVE_UNION_REMOVE_OPTIMIZATION_NEEDS_SUBDIRECTORIES.getMsg());
      }

      opRules.put(new RuleRegExp("R5", UnionOperator.getOperatorName() + "%" +
                                 ".*" + FileSinkOperator.getOperatorName() + "%"),
        UnionProcFactory.getUnionNoProcessFile());

      disp = new DefaultRuleDispatcher(UnionProcFactory.getNoUnion(), opRules, uCtx);
      ogw = new PreOrderWalker(disp);

      // Create a list of topop nodes
      topNodes.clear();
      topNodes.addAll(pCtx.getTopOps().values());
      ogw.startWalking(topNodes, null);
    }

    return pCtx;
  }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
        LocalMapJoinProcFactory.getGroupByProc());
      // The dispatcher fires the processor corresponding to the closest
      // matching rule and passes the context along
      Dispatcher disp = new DefaultRuleDispatcher(LocalMapJoinProcFactory.getDefaultProc(),
          opRules, localMapJoinProcCtx);
      GraphWalker ogw = new DefaultGraphWalker(disp);
      // iterator the reducer operator tree
      ArrayList<Node> topNodes = new ArrayList<Node>();
      topNodes.addAll(childOps);
      ogw.startWalking(topNodes, null);
    }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + "%"),
        getSortedMergeBucketMapjoinProc(pctx));
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, null);
    GraphWalker ogw = new DefaultGraphWalker(disp);

    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);

    return pctx;
  }
View Full Code Here
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
    opRules.put(new RuleRegExp(new String("R11"), "MAPJOIN%SEL%"), MapJoinFactory.getMapJoin());

    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules, procCtx);

    GraphWalker ogw = new GenMapRedWalker(disp);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(this.topOps.values());
    ogw.startWalking(topNodes, null);

    // reduce sink does not have any kids - since the plan by now has been broken up into multiple
    // tasks, iterate over all tasks.
    // For each task, go over all operators recursively
    for(Task<? extends Serializable> rootTask: rootTasks)
View Full Code Here
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
                               TypeCheckProcFactory.getBoolExprProcessor());
    opRules.put(new RuleRegExp("R5", HiveParser.TOK_TABLE_OR_COL + "%"), TypeCheckProcFactory.getColumnExprProcessor());

    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(TypeCheckProcFactory.getDefaultExprProcessor(), opRules, tcCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
  
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.add(expr);
    HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
    ogw.startWalking(topNodes, nodeOutputs);
    exprNodeDesc desc = (exprNodeDesc)nodeOutputs.get(expr);
    if (desc == null) {
      throw new SemanticException(tcCtx.getError());
    }
View Full Code Here
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
    exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
 
    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, pprCtx);
    GraphWalker egw = new DefaultGraphWalker(disp);
 
    List<Node> startNodes = new ArrayList<Node>();
    startNodes.add(pred);
   
    HashMap<Node, Object> outputMap = new HashMap<Node, Object>();
    egw.startWalking(startNodes, outputMap);
    hasNonPartCols = pprCtx.getHasNonPartCols();

    // Get the exprNodeDesc corresponding to the first start node;
    return (exprNodeDesc)outputMap.get(pred);
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    opRules.put(new RuleRegExp("R1", "(TS%FIL%)|(TS%FIL%FIL%)"),
                OpProcFactory.getFilterProc());

    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, opWalkerCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
   
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    pctx.setHasNonPartCols(opWalkerCtx.getHasNonPartCols());

    return pctx;
  }
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
397
398
    opRules.put(new RuleRegExp(new String("R4"), "MAPJOIN%.*UNION%"), getMapJoinDefault());

    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefault(), opRules, new MapJoinWalkerCtx(listMapJoinOpsNoRed));

    GraphWalker ogw = new GenMapRedWalker(disp);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(listMapJoinOps);
    ogw.startWalking(topNodes, null);
   
    pGraphContext.setListMapJoinOpsNoReducer(listMapJoinOpsNoRed);
    return pGraphContext;
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    opRules.put(new RuleRegExp("R6", "SCR%"), OpProcFactory.getSCRProc());
    opRules.put(new RuleRegExp("R6", "LIM%"), OpProcFactory.getLIMProc());

    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, opWalkerInfo);
    GraphWalker ogw = new DefaultGraphWalker(disp);
   
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pGraphContext.getTopOps().values());
    ogw.startWalking(topNodes, null);

    return pGraphContext;
  }
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
    exprRules.put(new RuleRegExp("R4", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
 
    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, exprContext);
    GraphWalker egw = new DefaultGraphWalker(disp);
 
    List<Node> startNodes = new ArrayList<Node>();
    List<exprNodeDesc> clonedPreds = new ArrayList<exprNodeDesc>();
    for (exprNodeDesc node : preds) {
      clonedPreds.add((exprNodeDesc) node.clone());
    }
    startNodes.addAll(clonedPreds);
   
    egw.startWalking(startNodes, null);
   
    // check the root expression for final candidates
    for (exprNodeDesc pred : clonedPreds) {
      extractFinalCandidates(pred, exprContext);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.lib.GraphWalker

  • com.sap.hadoop.windowing.query2.translate.WindowingTypeCheckProcFactory
  • org.apache.hadoop.hive.accumulo.predicate.AccumuloPredicateHandler
  • org.apache.hadoop.hive.accumulo.predicate.TestAccumuloRangeGenerator
  • org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer
  • org.apache.hadoop.hive.ql.optimizer.BucketingSortingReduceSinkOptimizer
  • org.apache.hadoop.hive.ql.optimizer.BucketMapJoinOptimizer
  • org.apache.hadoop.hive.ql.optimizer.ColumnPruner
  • org.apache.hadoop.hive.ql.optimizer.ConstantPropagate
  • org.apache.hadoop.hive.ql.optimizer.correlation.CorrelationOptimizer
  • org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.