Examples of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer

org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer
IndexPredicateAnalyzer decomposes predicates, separating the parts which can be satisfied by an index from the parts which cannot. Currently, it only supports pure conjunctions over binary expressions comparing a column reference with a constant value. It is assumed that all column aliases encountered refer to the same table.

    String columnNameProperty = jobConf.get(
      org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS);
    List<String> columnNames =
      Arrays.asList(columnNameProperty.split(","));
    HBaseSerDe hbaseSerde = (HBaseSerDe) deserializer;
    IndexPredicateAnalyzer analyzer =
      HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(
        columnNames.get(hbaseSerde.getKeyColumnOffset()));
    List<IndexSearchCondition> searchConditions =
      new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate =
      analyzer.analyzePredicate(predicate, searchConditions);
    if (searchConditions.size() != 1) {
      // Either there was nothing which could be pushed down (size = 0),
      // or more than one predicate (size > 1); in the latter case,
      // we bail out for now since multiple lookups on the key are
      // either contradictory or redundant.  We'll need to handle
      // this better later when we support more interesting predicates.
      return null;
    }
    
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(
      searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
  }

View Full Code Here

    ExprNodeDesc filterExpr =
      Utilities.deserializeExpression(filterExprSerialized, jobConf);


    String colName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey];
    String colType = jobConf.get(serdeConstants.LIST_COLUMN_TYPES).split(",")[iKey];
    IndexPredicateAnalyzer analyzer = newIndexPredicateAnalyzer(colName,colType, isKeyBinary);


    List<IndexSearchCondition> searchConditions =
      new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate =
      analyzer.analyzePredicate(filterExpr, searchConditions);


    // There should be no residual since we already negotiated
    // that earlier in HBaseStorageHandler.decomposePredicate.
    if (residualPredicate != null) {
      throw new RuntimeException(

View Full Code Here

   * @return preconfigured predicate analyzer
   */
  static IndexPredicateAnalyzer newIndexPredicateAnalyzer(
    String keyColumnName, String keyColType, boolean isKeyBinary) {


    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    // We can always do equality predicate. Just need to make sure we get appropriate
    // BA representation of constant of filter condition.
    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual");
    // We can do other comparisons only if storage format in hbase is either binary
    // or we are dealing with string types since there lexographic ordering will suffice.
    if(isKeyBinary || (keyColType.equalsIgnoreCase("string"))){
      analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic." +
        "GenericUDFOPEqualOrGreaterThan");
      analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan");
      analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan");
      analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan");
    }


    // and only on the key column
    analyzer.clearAllowedColumnNames();
    analyzer.allowColumnName(keyColumnName);


    return analyzer;
  }

View Full Code Here


    HBaseSerDe hbaseSerde = (HBaseSerDe) deserializer;
    int keyColPos = hbaseSerde.getKeyColumnOffset();
    String keyColType = jobConf.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES).
        split(",")[keyColPos];
    IndexPredicateAnalyzer analyzer =
      HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(columnNames.get(keyColPos), keyColType,
        hbaseSerde.getStorageFormatOfCol(keyColPos).get(0));
    List<IndexSearchCondition> searchConditions =
      new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate =
      analyzer.analyzePredicate(predicate, searchConditions);
    int scSize = searchConditions.size();
    if (scSize < 1 || 2 < scSize) {
      // Either there was nothing which could be pushed down (size = 0),
      // there were complex predicates which we don't support yet.
      // Currently supported are one of the form:
      // 1. key < 20                        (size = 1)
      // 2. key = 20                        (size = 1)
      // 3. key < 20 and key > 10           (size = 2)
      return null;
    }
    if (scSize == 2 &&
        (searchConditions.get(0).getComparisonOp()
        .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual") ||
        searchConditions.get(1).getComparisonOp()
        .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual"))) {
      // If one of the predicates is =, then any other predicate with it is illegal.
      return null;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(
      searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
  }

View Full Code Here

   * @param index
   * @return
   */
  private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index,
      Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);


    if (searchConditions.size() == 0) {
      return null;
    }


    int numIndexCols = 0;
    for (IndexSearchCondition searchCondition : searchConditions) {
      if (!partitionCols.contains(searchCondition.getColumnDesc().getColumn())) {
        numIndexCols++;
      }
    }


    // For now, only works if the predicate has a single condition on an index column
    if (numIndexCols == 1) {
      useSorted = true;
    } else {
      useSorted = false;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;


    return decomposedPredicate;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(Index index, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in this index
    List<FieldSchema> columnSchemas = index.getSd().getCols();
    for (FieldSchema column : columnSchemas) {
      analyzer.allowColumnName(column.getName());
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    partitionCols = new HashSet<String>();
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
        partitionCols.add(column);
      }
    }


    return analyzer;

View Full Code Here

   * @param index
   * @return
   */
  private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index,
      Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);


    if (searchConditions.size() == 0) {
      return null;
    }


    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;


    return decomposedPredicate;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(Index index, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in this index
    List<FieldSchema> columnSchemas = index.getSd().getCols();
    for (FieldSchema column : columnSchemas) {
      analyzer.allowColumnName(column.getName());
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
      }
    }


    return analyzer;
  }

View Full Code Here

  private Map<Index, ExprNodeDesc> decomposePredicate(ExprNodeDesc predicate, List<Index> indexes,
      HiveIndexQueryContext queryContext) {


    Map<Index, ExprNodeDesc> indexPredicates = new HashMap<Index, ExprNodeDesc>();
    // compute overall residual
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(indexes, queryContext.getQueryPartitions());
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);
    // pass residual predicate back out for further processing
    queryContext.setResidualPredicate(residualPredicate);


    if (searchConditions.size() == 0) {
      return null;
    }


    for (Index index : indexes) {
      ArrayList<Index> in = new ArrayList<Index>(1);
      in.add(index);
      analyzer = getIndexPredicateAnalyzer(in, queryContext.getQueryPartitions());
      searchConditions = new ArrayList<IndexSearchCondition>();
      // split predicate into pushed (what we can handle), and residual (what we can't handle)
      // pushed predicate from translateSearchConditions is stored for the current index
      // This ensures that we apply all possible predicates to each index
      analyzer.analyzePredicate(predicate, searchConditions);
      if (searchConditions.size() == 0) {
        indexPredicates.put(index, null);
      } else {
        indexPredicates.put(index, analyzer.translateSearchConditions(searchConditions));
      }
    }


    return indexPredicates;
  }

View Full Code Here

   * WHERE clauses that we support
   *
   * @return preconfigured predicate analyzer for WHERE queries
   */
  private IndexPredicateAnalyzer getIndexPredicateAnalyzer(List<Index> indexes, Set<Partition> queryPartitions)  {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();


    analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
    analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
    analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());


    // only return results for columns in the list of indexes
    for (Index index : indexes) {
      List<FieldSchema> columnSchemas = index.getSd().getCols();
      for (FieldSchema column : columnSchemas) {
        analyzer.allowColumnName(column.getName());
      }
    }


    // partitioned columns are treated as if they have indexes so that the partitions
    // are used during the index query generation
    for (Partition part : queryPartitions) {
      if (part.getSpec().isEmpty()) {
        continue; // empty partitions are from whole tables, so we don't want to add them in
      }
      for (String column : part.getSpec().keySet()) {
        analyzer.allowColumnName(column);
      }
    }


    return analyzer;
  }

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer

org.apache.hadoop.hive.accumulo.predicate.AccumuloPredicateHandler

org.apache.hadoop.hive.hbase.AbstractHBaseKeyPredicateDecomposer

org.apache.hadoop.hive.hbase.HBaseStorageHandler

org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat

org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2

org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator

org.apache.hadoop.hive.ql.exec.UDF

org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler

org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler

org.apache.hadoop.hive.ql.lib.DefaultGraphWalker

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.