Examples of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer$FieldValidator

org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer
IndexPredicateAnalyzer decomposes predicates, separating the parts which can be satisfied by an index from the parts which cannot. Currently, it only supports pure conjunctions over binary expressions comparing a column reference with a constant value. It is assumed that all column aliases encountered refer to the same table.

  private Map<Index, ExprNodeDesc> decomposePredicate(ExprNodeDesc predicate, List<Index> indexes,
      HiveIndexQueryContext queryContext) {


    Map<Index, ExprNodeDesc> indexPredicates = new HashMap<Index, ExprNodeDesc>();
    // compute overall residual
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(indexes, queryContext.getQueryPartitions());
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);
    // pass residual predicate back out for further processing
    queryContext.setResidualPredicate(residualPredicate);


    if (searchConditions.size() == 0) {
      return null;
    }


    for (Index index : indexes) {
      ArrayList<Index> in = new ArrayList<Index>(1);
      in.add(index);
      analyzer = getIndexPredicateAnalyzer(in, queryContext.getQueryPartitions());
      searchConditions = new ArrayList<IndexSearchCondition>();
      // split predicate into pushed (what we can handle), and residual (what we can't handle)
      // pushed predicate from translateSearchConditions is stored for the current index
      // This ensures that we apply all possible predicates to each index
      analyzer.analyzePredicate(predicate, searchConditions);
      if (searchConditions.size() == 0) {
        indexPredicates.put(index, null);
      } else {
        indexPredicates.put(index, analyzer.translateSearchConditions(searchConditions));
      }
    }


    return indexPredicates;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(List<Index> indexes, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in the list of indexes
    for (Index index : indexes) {
      List<FieldSchema> columnSchemas = index.getSd().getCols();
      for (FieldSchema column : columnSchemas) {
        analyzer.allowColumnName(column.getName());
      }
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
      }
    }


    return analyzer;
  }

View Full Code Here

   * @param index
   * @return
   */
  private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index,
      Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeGenericFuncDesc residualPredicate = (ExprNodeGenericFuncDesc)analyzer.
      analyzePredicate(predicate, searchConditions);


    if (searchConditions.size() == 0) {
      return null;
    }


    int numIndexCols = 0;
    for (IndexSearchCondition searchCondition : searchConditions) {
      if (!partitionCols.contains(searchCondition.getColumnDesc().getColumn())) {
        numIndexCols++;
      }
    }


    // For now, only works if the predicate has a single condition on an index column
    if (numIndexCols == 1) {
      useSorted = true;
    } else {
      useSorted = false;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;


    return decomposedPredicate;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(Index index, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in this index
    List<FieldSchema> columnSchemas = index.getSd().getCols();
    for (FieldSchema column : columnSchemas) {
      analyzer.allowColumnName(column.getName());
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    partitionCols = new HashSet<String>();
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
        partitionCols.add(column);
      }
    }


    return analyzer;

View Full Code Here

    String columnNameProperty = jobConf.get(
      org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS);
    List<String> columnNames =
      Arrays.asList(columnNameProperty.split(","));
    HBaseSerDe hbaseSerde = (HBaseSerDe) deserializer;
    IndexPredicateAnalyzer analyzer =
      HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(
        columnNames.get(hbaseSerde.getKeyColumnOffset()));
    List<IndexSearchCondition> searchConditions =
      new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate =
      analyzer.analyzePredicate(predicate, searchConditions);
    if (searchConditions.size() != 1) {
      // Either there was nothing which could be pushed down (size = 0),
      // or more than one predicate (size > 1); in the latter case,
      // we bail out for now since multiple lookups on the key are
      // either contradictory or redundant.  We'll need to handle
      // this better later when we support more interesting predicates.
      return null;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(
      searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
  }

View Full Code Here


    String columnNameProperty = jobConf.get(Constants.LIST_COLUMNS);
    List<String> columnNames =
      Arrays.asList(columnNameProperty.split(","));


    IndexPredicateAnalyzer analyzer =
      newIndexPredicateAnalyzer(columnNames.get(iKey));


    List<IndexSearchCondition> searchConditions =
      new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate =
      analyzer.analyzePredicate(filterExpr, searchConditions);


    // There should be no residual since we already negotiated
    // that earlier in HBaseStorageHandler.decomposePredicate.
    if (residualPredicate != null) {
      throw new RuntimeException(

View Full Code Here

   * @return preconfigured predicate analyzer
   */
  static IndexPredicateAnalyzer newIndexPredicateAnalyzer(
    String keyColumnName) {


    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    // for now, we only support equality comparisons
    analyzer.addComparisonOp(
      "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual");


    // and only on the key column
    analyzer.clearAllowedColumnNames();
    analyzer.allowColumnName(keyColumnName);


    return analyzer;
  }

View Full Code Here

  private Map<Index, ExprNodeDesc> decomposePredicate(ExprNodeDesc predicate, List<Index> indexes,
      HiveIndexQueryContext queryContext) {


    Map<Index, ExprNodeDesc> indexPredicates = new HashMap<Index, ExprNodeDesc>();
    // compute overall residual
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(indexes, queryContext.getQueryPartitions());
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);
    // pass residual predicate back out for further processing
    queryContext.setResidualPredicate(residualPredicate);


    if (searchConditions.size() == 0) {
      return null;
    }


    for (Index index : indexes) {
      ArrayList<Index> in = new ArrayList<Index>(1);
      in.add(index);
      analyzer = getIndexPredicateAnalyzer(in, queryContext.getQueryPartitions());
      searchConditions = new ArrayList<IndexSearchCondition>();
      // split predicate into pushed (what we can handle), and residual (what we can't handle)
      // pushed predicate from translateSearchConditions is stored for the current index
      // This ensures that we apply all possible predicates to each index
      analyzer.analyzePredicate(predicate, searchConditions);
      if (searchConditions.size() == 0) {
        indexPredicates.put(index, null);
      } else {
        indexPredicates.put(index, analyzer.translateSearchConditions(searchConditions));
      }
    }


    return indexPredicates;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(List<Index> indexes, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in the list of indexes
    for (Index index : indexes) {
      List<FieldSchema> columnSchemas = index.getSd().getCols();
      for (FieldSchema column : columnSchemas) {
        analyzer.allowColumnName(column.getName());
      }
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
      }
    }


    return analyzer;
  }

View Full Code Here

   * @param index
   * @return
   */
  private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index,
      Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);


    if (searchConditions.size() == 0) {
      return null;
    }


    int numIndexCols = 0;
    for (IndexSearchCondition searchCondition : searchConditions) {
      if (!partitionCols.contains(searchCondition.getColumnDesc().getColumn())) {
        numIndexCols++;
      }
    }


    // For now, only works if the predicate has a single condition on an index column
    if (numIndexCols == 1) {
      useSorted = true;
    } else {
      useSorted = false;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;


    return decomposedPredicate;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer$FieldValidator

org.apache.hadoop.hive.accumulo.predicate.AccumuloPredicateHandler

org.apache.hadoop.hive.hbase.AbstractHBaseKeyPredicateDecomposer

org.apache.hadoop.hive.hbase.HBaseStorageHandler

org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat

org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2

org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator

org.apache.hadoop.hive.ql.exec.UDF

org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler

org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler

org.apache.hadoop.hive.ql.lib.DefaultGraphWalker

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.