Package org.apache.hadoop.hive.ql.lib

Examples of org.apache.hadoop.hive.ql.lib.RuleRegExp

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1",
      FilterOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getFilterProc());
    opRules.put(new RuleRegExp("R2",
      GroupByOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getGroupByProc());
    opRules.put(new RuleRegExp("R3",
      ReduceSinkOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getReduceSinkProc());
    opRules.put(new RuleRegExp("R4",
      SelectOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getSelectProc());
    opRules.put(new RuleRegExp("R5",
      CommonJoinOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getJoinProc());
    opRules.put(new RuleRegExp("R6",
      MapJoinOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getMapJoinProc());
    opRules.put(new RuleRegExp("R7",
      TableScanOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getTableScanProc());
    opRules.put(new RuleRegExp("R8",
      LateralViewJoinOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getLateralViewJoinProc());
    opRules.put(new RuleRegExp("R9",
      LateralViewForwardOperator.getOperatorName() + "%"),
      ColumnPrunerProcFactory.getLateralViewForwardProc());
    opRules.put(new RuleRegExp("R10",
        PTFOperator.getOperatorName() + "%"),
        ColumnPrunerProcFactory.getPTFProc());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
    exprRules.put(
        new RuleRegExp("R1", ExprNodeColumnDesc.class.getName() + "%"),
        getColumnProcessor());
    exprRules.put(
        new RuleRegExp("R2", ExprNodeFieldDesc.class.getName() + "%"),
        getFieldProcessor());
    exprRules.put(new RuleRegExp("R3", ExprNodeGenericFuncDesc.class.getName()
        + "%"), getGenericFuncProcessor());

    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(),
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    HiveConf conf = pctx.getConf();

    if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEGROUPBYSKEW)) {
      // process group-by pattern
      opRules.put(new RuleRegExp("R1",
          GroupByOperator.getOperatorName() + "%" +
              ReduceSinkOperator.getOperatorName() + "%" +
              GroupByOperator.getOperatorName() + "%"),
          getMapSortedGroupbyProc(pctx));
    } else {
      // If hive.groupby.skewindata is set to true, the operator tree is as below
      opRules.put(new RuleRegExp("R2",
          GroupByOperator.getOperatorName() + "%" +
              ReduceSinkOperator.getOperatorName() + "%" +
              GroupByOperator.getOperatorName() + "%" +
              ReduceSinkOperator.getOperatorName() + "%" +
              GroupByOperator.getOperatorName() + "%"),
View Full Code Here
295
296
297
298
299
300
301
302
303
304
305
306
307
          return false;
        }
      }
      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      ValidationNodeProcessor vnp = new ValidationNodeProcessor();
      opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*"
          + FileSinkOperator.getOperatorName()), vnp);
      opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*"
          + ReduceSinkOperator.getOperatorName()), vnp);
      Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
      GraphWalker ogw = new DefaultGraphWalker(disp);
      // iterator the mapper operator tree
      ArrayList<Node> topNodes = new ArrayList<Node>();
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
332
333
    private void vectorizeMapWork(MapWork mapWork) throws SemanticException {
      LOG.info("Vectorizing task...");
      mapWork.setVectorMode(true);
      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      VectorizationNodeProcessor vnp = new VectorizationNodeProcessor(mapWork);
      opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" +
          ReduceSinkOperator.getOperatorName()), vnp);
      opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*"
          + FileSinkOperator.getOperatorName()), vnp);
      Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
      GraphWalker ogw = new PreOrderWalker(disp);
      // iterator the mapper operator tree
      ArrayList<Node> topNodes = new ArrayList<Node>();
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
      // RuleExactMatch rules are used to specify exactly what the tree should look like
      // In particular, this guarantees that the first operator is the reducer
      // (and its parent(s) are ReduceSinkOperators) since it begins walking the tree from
      // the reducer.
      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      opRules.put(new RuleRegExp("R1", SelectOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getSelProc());
      // Matches only GroupByOpeartors which are reducers, rather than map group by operators,
      // or multi group by optimization specific operators
      opRules.put(new RuleExactMatch("R2", GroupByOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getGroupByProc());
      // Matches only JoinOperators which are reducers, rather than map joins, SMB map joins, etc.
      opRules.put(new RuleExactMatch("R3", JoinOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getJoinProc());
      opRules.put(new RuleRegExp("R5", FileSinkOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getFileSinkProc());
      // Matches only ExtractOperators which are reducers
      opRules.put(new RuleExactMatch("R6", ExtractOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getExtractProc());
      opRules.put(new RuleRegExp("R7", FilterOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getFilterProc());
      opRules.put(new RuleRegExp("R8", LimitOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getLimitProc());
      opRules.put(new RuleRegExp("R9", LateralViewForwardOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getLateralViewForwardProc());
      opRules.put(new RuleRegExp("R10", LateralViewJoinOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getLateralViewJoinProc());
      // Matches only ForwardOperators which are preceded by some other operator in the tree,
      // in particular it can't be a reducer (and hence cannot be one of the ForwardOperators
      // added by the multi group by optimization)
      opRules.put(new RuleRegExp("R11", ".+" + ForwardOperator.getOperatorName() + "%"),
          BucketingSortingOpProcFactory.getForwardProc());
      // Matches only ForwardOperators which are reducers and are followed by GroupByOperators
      // (specific to the multi group by optimization)
      opRules.put(new RuleExactMatch("R12", ForwardOperator.getOperatorName() + "%" +
          GroupByOperator.getOperatorName() + "%"),
View Full Code Here
130
131
132
133
134
135
136
137
138
139
      return null;
    }

    // We set the pushed predicate from the WHERE clause as the filter expr on
    // all table scan operators, so we look for table scan operators(TS%)
    operatorRules.put(new RuleRegExp("RULEWhere", TableScanOperator.getOperatorName() + "%"),
      new IndexWhereProcessor(indexes));

    return operatorRules;
  }
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
    public void hasGroupBy(Operator<? extends OperatorDesc> mapJoinOp,
        LocalMapJoinProcCtx localMapJoinProcCtx) throws Exception {
      List<Operator<? extends OperatorDesc>> childOps = mapJoinOp.getChildOperators();
      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      opRules.put(new RuleRegExp("R1", GroupByOperator.getOperatorName() + "%"),
        LocalMapJoinProcFactory.getGroupByProc());
      // The dispatcher fires the processor corresponding to the closest
      // matching rule and passes the context along
      Dispatcher disp = new DefaultRuleDispatcher(LocalMapJoinProcFactory.getDefaultProc(),
          opRules, localMapJoinProcCtx);
View Full Code Here
249
250
251
252
253
254
255
256
257
258
259
    private LocalMapJoinProcCtx adjustLocalTask(MapredLocalTask task)
        throws SemanticException {
      LocalMapJoinProcCtx localMapJoinProcCtx = new LocalMapJoinProcCtx(task, physicalContext
          .getParseContext());
      Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
      opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + "%"),
        LocalMapJoinProcFactory.getJoinProc());
      // The dispatcher fires the processor corresponding to the closest
      // matching rule and passes the context along
      Dispatcher disp = new DefaultRuleDispatcher(LocalMapJoinProcFactory.getDefaultProc(),
          opRules, localMapJoinProcCtx);
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();

    opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"),
        getNullExprProcessor());
    opRules.put(new RuleRegExp("R2", HiveParser.Number + "%|" +
        HiveParser.TinyintLiteral + "%|" +
        HiveParser.SmallintLiteral + "%|" +
        HiveParser.BigintLiteral + "%|" +
        HiveParser.DecimalLiteral + "%"),
        getNumExprProcessor());
    opRules
        .put(new RuleRegExp("R3", HiveParser.Identifier + "%|"
        + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL + "%|"
        + HiveParser.TOK_STRINGLITERALSEQUENCE + "%|"
        + "%|" + HiveParser.KW_IF + "%|" + HiveParser.KW_CASE + "%|"
        + HiveParser.KW_WHEN + "%|" + HiveParser.KW_IN + "%|"
        + HiveParser.KW_ARRAY + "%|" + HiveParser.KW_MAP + "%|"
        + HiveParser.KW_STRUCT + "%|" + HiveParser.KW_EXISTS + "%|"
        + HiveParser.TOK_SUBQUERY_OP_NOTIN + "%"),
        getStrExprProcessor());
    opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
        + HiveParser.KW_FALSE + "%"), getBoolExprProcessor());
    opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), getDateExprProcessor());
    opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"),
        getColumnExprProcessor());
    opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"),
        getSubQueryExprProcessor());

    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(),
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.lib.RuleRegExp

  • com.sap.hadoop.windowing.query2.translate.WindowingTypeCheckProcFactory
  • org.apache.hadoop.hive.ql.optimizer.BucketingSortingReduceSinkOptimizer
  • org.apache.hadoop.hive.ql.optimizer.BucketMapJoinOptimizer
  • org.apache.hadoop.hive.ql.optimizer.ColumnPruner
  • org.apache.hadoop.hive.ql.optimizer.ConstantPropagate
  • org.apache.hadoop.hive.ql.optimizer.correlation.CorrelationOptimizer
  • org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication
  • org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization
  • org.apache.hadoop.hive.ql.optimizer.GroupByOptimizer
  • org.apache.hadoop.hive.ql.optimizer.index.RewriteCanApplyCtx
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.