Package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators

Examples of org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel


   * Inferring Uniqueness for all columns is very expensive right now. The flip
   * side of doing this is, it only works post Field Trimming.
   */
  public Set<BitSet> getUniqueKeys(ProjectRelBase rel, boolean ignoreNulls) {

    HiveTableScanRel tScan = getTableScan(rel.getChild(), false);

    if ( tScan == null ) {
      Function<RelNode, Metadata> fn = RelMdUniqueKeys.SOURCE.apply(
          rel.getClass(), BuiltInMetadata.UniqueKeys.class);
      return ((BuiltInMetadata.UniqueKeys) fn.apply(rel))
          .getUniqueKeys(ignoreNulls);
    }

    Map<Integer, Integer> posMap = new HashMap<Integer, Integer>();
    int projectPos = 0;
    int colStatsPos = 0;

    BitSet projectedCols = new BitSet();
    for (RexNode r : rel.getProjects()) {
      if (r instanceof RexInputRef) {
        projectedCols.set(((RexInputRef) r).getIndex());
        posMap.put(colStatsPos, projectPos);
        colStatsPos++;
      }
      projectPos++;
    }

    double numRows = tScan.getRows();
    List<ColStatistics> colStats = tScan.getColStat(BitSets
        .toList(projectedCols));
    Set<BitSet> keys = new HashSet<BitSet>();

    colStatsPos = 0;
    for (ColStatistics cStat : colStats) {
View Full Code Here


      double childRowCount) {
    if ((leftChild && joinRel.getJoinType().generatesNullsOnRight()) ||
        (!leftChild && joinRel.getJoinType().generatesNullsOnLeft())) {
      return 1.0;
    } else {
      HiveTableScanRel tScan = HiveRelMdUniqueKeys.getTableScan(child, true);
      if (tScan != null) {
        double tRowCount = RelMetadataQuery.getRowCount(tScan);
        return childRowCount / tRowCount;
      } else {
        return 1.0;
View Full Code Here

  }

  @Override
  public void onMatch(RelOptRuleCall call) {
    HiveFilterRel filter = call.rel(0);
    HiveTableScanRel tScan = call.rel(1);
    perform(call, filter, tScan);
  }
View Full Code Here

      return genJoinRelNode(leftRel, rightRel, hiveJoinType, joinCond);
    }

    private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticException {
      RowResolver rr = new RowResolver();
      HiveTableScanRel tableRel = null;

      try {

        // 1. If the table has a Sample specified, bail from Optiq path.
        if ( qb.getParseInfo().getTabSample(tableAlias) != null ||
            SemanticAnalyzer.this.nameToSplitSample.containsKey(tableAlias)) {
          String msg = String.format("Table Sample specified for %s." +
              " Currently we don't support Table Sample clauses in CBO," +
              " turn off cbo for queries on tableSamples.", tableAlias);
          LOG.debug(msg);
          throw new OptiqSemanticException(msg);
        }

        // 2. Get Table Metadata
        Table tab = qb.getMetaData().getSrcForAlias(tableAlias);

        // 3. Get Table Logical Schema (Row Type)
        // NOTE: Table logical schema = Non Partition Cols + Partition Cols +
        // Virtual Cols

        // 3.1 Add Column info for non partion cols (Object Inspector fields)
        StructObjectInspector rowObjectInspector = (StructObjectInspector) tab.getDeserializer()
            .getObjectInspector();
        List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
        ColumnInfo colInfo;
        String colName;
        ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>();
        for (int i = 0; i < fields.size(); i++) {
          colName = fields.get(i).getFieldName();
          colInfo = new ColumnInfo(
              fields.get(i).getFieldName(),
              TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()),
              tableAlias, false);
          colInfo.setSkewedCol((isSkewedCol(tableAlias, qb, colName)) ? true : false);
          rr.put(tableAlias, colName, colInfo);
          cInfoLst.add(colInfo);
        }
        // TODO: Fix this
        ArrayList<ColumnInfo> nonPartitionColumns = new ArrayList<ColumnInfo>(cInfoLst);
        ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();

        // 3.2 Add column info corresponding to partition columns
        for (FieldSchema part_col : tab.getPartCols()) {
          colName = part_col.getName();
          colInfo = new ColumnInfo(colName,
              TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), tableAlias, true);
          rr.put(tableAlias, colName, colInfo);
          cInfoLst.add(colInfo);
          partitionColumns.add(colInfo);
        }

        // 3.3 Add column info corresponding to virtual columns
        Iterator<VirtualColumn> vcs = VirtualColumn.getRegistry(conf).iterator();
        while (vcs.hasNext()) {
          VirtualColumn vc = vcs.next();
          colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true,
              vc.getIsHidden());
          rr.put(tableAlias, vc.getName(), colInfo);
          cInfoLst.add(colInfo);
        }

        // 3.4 Build row type from field <type, name>
        RelDataType rowType = TypeConverter.getType(cluster, rr, null);

        // 4. Build RelOptAbstractTable
        String fullyQualifiedTabName = tab.getDbName();
        if (fullyQualifiedTabName != null && !fullyQualifiedTabName.isEmpty())
          fullyQualifiedTabName = fullyQualifiedTabName + "." + tab.getTableName();
        else
          fullyQualifiedTabName = tab.getTableName();
        RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName,
            tableAlias, rowType, tab, nonPartitionColumns, partitionColumns, conf, partitionCache,
            noColsMissingStats);

        // 5. Build Hive Table Scan Rel
        tableRel = new HiveTableScanRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), optTable,
            rowType);

        // 6. Add Schema(RR) to RelNode-Schema map
        ImmutableMap<String, Integer> hiveToOptiqColMap = buildHiveToOptiqColumnMap(rr, tableRel);
        relToHiveRR.put(tableRel, rr);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.