Package org.apache.hadoop.hive.ql.parse

Examples of org.apache.hadoop.hive.ql.parse.PrunedPartitionList

718
719
720
721
722
723
724
725
726
727
728
729
730
          //if it's null then the partition probably doesn't exist so let's use table permission
          if (tbl.isPartitioned() &&
              Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName()))) {
            String alias_id = topOpMap.getKey();

            PrunedPartitionList partsList = PartitionPruner.prune(tableScanOp,
                parseCtx, alias_id);
            Set<Partition> parts = partsList.getPartitions();
            for (Partition part : parts) {
              List<String> existingCols = part2Cols.get(part);
              if (existingCols == null) {
                existingCols = new ArrayList<String>();
              }
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
194
195
    if (HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVEOPTPPD)) {
      ExprNodeDesc pruner = pctx.getOpToPartPruner().get(ts);
      bypassFilter = PartitionPruner.onlyContainsPartnCols(table, pruner);
    }
    if (aggressive || bypassFilter) {
      PrunedPartitionList pruned = pctx.getPrunedPartitions(alias, ts);
      if (aggressive || !pruned.hasUnknownPartitions()) {
        bypassFilter &= !pruned.hasUnknownPartitions();
        return checkOperators(new FetchData(parent, table, pruned, splitSample, bypassFilter), ts,
            aggressive, bypassFilter);
      }
    }
    return null;
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
    @Override
    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
        Object... nodeOutputs) throws SemanticException {
      TableScanOperator tsop = (TableScanOperator) nd;
      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;
      PrunedPartitionList partList =
          aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop);
      Table table = aspCtx.getParseContext().getTopToTable().get(tsop);

      try {
        // gather statistics for the first time and the attach it to table scan operator
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
   */
  public static Set<Partition> checkPartitionsCoveredByIndex(TableScanOperator tableScan,
      ParseContext pctx, List<Index> indexes) throws HiveException {
    Hive hive = Hive.get(pctx.getConf());
    // make sure each partition exists on the index table
    PrunedPartitionList queryPartitionList = pctx.getOpToPartList().get(tableScan);
    Set<Partition> queryPartitions = queryPartitionList.getPartitions();
    if (queryPartitions == null || queryPartitions.isEmpty()) {
      return null;
    }

    for (Partition part : queryPartitions) {
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
      return false;
    }

    Table tbl = topToTable.get(tso);
    if (tbl.isPartitioned()) {
      PrunedPartitionList prunedParts = pGraphContext.getPrunedPartitions(alias, tso);
      List<Partition> partitions = prunedParts.getNotDeniedPartns();
      // Populate the names and order of columns for the first partition of the
      // first table
      if ((pos == 0) && (partitions != null) && (!partitions.isEmpty())) {
        Partition firstPartition = partitions.get(0);
        sortColumnsFirstTable.addAll(firstPartition.getSortCols());
      }

      for (Partition partition : prunedParts.getNotDeniedPartns()) {
        if (!checkSortColsAndJoinCols(partition.getSortCols(),
          joinCols,
          sortColumnsFirstTable)) {
          return false;
        }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
      if (!isFalseExpr(compactExpr)) {
        // Non-strict mode, and all the predicates are on non-partition columns - get everything.
        LOG.debug("Filter " + oldFilter + " was null after compacting");
        return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
      } else {
        return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(new ArrayList<Partition>()),
            new ArrayList<String>(), false);
      }
    }
    LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
        + "; filter w/o compacting: " + oldFilter);

    key = key + compactExpr.getExprString();
    PrunedPartitionList ppList = prunedPartitionsMap.get(key);
    if (ppList != null) {
      return ppList;
    }

    ppList = getPartitionsFromServer(tab, (ExprNodeGenericFuncDesc)compactExpr, conf, alias, partColsUsedInFilter, oldFilter.equals(compactExpr.getExprString()));
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
    return ppList;
  }

  private static PrunedPartitionList getAllPartsFromCacheOrServer(Table tab, String key, boolean unknownPartitions,
    Map<String, PrunedPartitionList> partsCache)  throws SemanticException {
    PrunedPartitionList ppList = partsCache.get(key);
    if (ppList != null) {
      return ppList;
    }
    Set<Partition> parts;
    try {
      parts = getAllPartitions(tab);
    } catch (HiveException e) {
      throw new SemanticException(e);
    }
    ppList = new PrunedPartitionList(tab, parts, null, unknownPartitions);
    partsCache.put(key, ppList);
    return ppList;
  }
View Full Code Here
383
384
385
386
387
388
389
390
391
392
393
        hasUnknownPartitions = pruneBySequentialScan(tab, partitions, compactExpr, conf);
      }
      // The partitions are "unknown" if the call says so due to the expression
      // evaluator returning null for a partition, or if we sent a partial expression to
      // metastore and so some partitions may have no data based on other filters.
      return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(partitions),
          new ArrayList<String>(partColsUsedInFilter),
          hasUnknownPartitions || !isPruningByExactFilter);
    } catch (SemanticException e) {
      throw e;
    } catch (Exception e) {
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
            Set<Partition> confirmedPartns = GenMapRedUtils
                .getConfirmedPartitionsForScan(parseInfo);
            if (confirmedPartns.size() > 0) {
              Table source = parseCtx.getQB().getMetaData().getTableForAlias(alias);
              List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
              PrunedPartitionList partList = new PrunedPartitionList(source, confirmedPartns, partCols, false);
              GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx, partList);
            } else { // non-partitioned table
              GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx);
            }
          }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
        return null;
      }


      ParseContext pctx = owc.getParseContext();
      PrunedPartitionList prunedPartList;
      try {
        String alias = (String) owc.getParseContext().getTopOps().keySet().toArray()[0];
        prunedPartList = pctx.getPrunedPartitions(alias, top);
      } catch (HiveException e) {
        // Has to use full name to make sure it does not conflict with
        // org.apache.commons.lang.StringUtils
        throw new SemanticException(e.getMessage(), e);
      }

      // Otherwise this is not a sampling predicate. We need to process it.
      ExprNodeDesc predicate = fop.getConf().getPredicate();
      String alias = top.getConf().getAlias();

      ArrayList<Partition> partitions = new ArrayList<Partition>();
      if (prunedPartList == null) {
        return null;
      }

      for (Partition p : prunedPartList.getPartitions()) {
        if (!p.getTable().isPartitioned()) {
          return null;
        }
      }

      partitions.addAll(prunedPartList.getPartitions());

      PcrExprProcFactory.NodeInfoWrapper wrapper = PcrExprProcFactory.walkExprTree(
          alias, partitions, top.getConf().getVirtualCols(), predicate);

      if (wrapper.state == PcrExprProcFactory.WalkState.TRUE) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.parse.PrunedPartitionList

  • org.apache.hadoop.hive.ql.Driver
  • org.apache.hadoop.hive.ql.optimizer.AbstractBucketJoinProc
  • org.apache.hadoop.hive.ql.optimizer.AbstractSMBJoinProc
  • org.apache.hadoop.hive.ql.optimizer.AvgPartitionSizeBasedBigTableSelectorForAutoSMJ
  • org.apache.hadoop.hive.ql.optimizer.BucketingSortingReduceSinkOptimizer$BucketSortReduceSinkProcessor
  • org.apache.hadoop.hive.ql.optimizer.BucketMapJoinOptimizer$BucketMapjoinOptProc
  • org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization
  • org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils
  • org.apache.hadoop.hive.ql.optimizer.GenMRTableScan1
  • org.apache.hadoop.hive.ql.optimizer.GlobalLimitOptimizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.