Examples of org.apache.hadoop.hive.ql.io.sarg.SearchArgument

Package org.apache.hadoop.hive.ql.io.sarg

Examples of org.apache.hadoop.hive.ql.io.sarg.SearchArgument

org.apache.hadoop.hive.ql.io.sarg.SearchArgument
kipedia.org/wiki/Sargable"> SearchArgument, which are the subset of predicates that can be pushed down to the RecordReader. Each SearchArgument consists of a series of SearchClauses that must each be true for the row to be accepted by the filter. This requires that the filter be normalized into conjunctive normal form (CNF).


    @Test
    public void testSimple() throws Exception {
        String q = query + "b = filter a by srcid == 10;" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "expr = leaf-0", sarg.toString());
    }

View Full Code Here


    @Test
    public void testAndOr() throws Exception {
        String q = query + "b = filter a by (srcid > 10 or dstid <= 5) and name == 'foo' and mrkt is null;" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid", "dstid", "name", "mrkt"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (LESS_THAN_EQUALS srcid 10)\n" +
                "leaf-1 = (LESS_THAN_EQUALS dstid 5)\n" +
                "leaf-2 = (EQUALS name foo)\n" +
                "leaf-3 = (IS_NULL mrkt)\n" +
                "expr = (and (or (not leaf-0) leaf-1) leaf-2 leaf-3)", sarg.toString());
    }

View Full Code Here


    @Test
    public void testNot() throws Exception {
        String q = query + "b = filter a by srcid != 10 and mrkt is not null;" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid", "dstid", "name", "mrkt"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "leaf-1 = (IS_NULL mrkt)\n" +
                "expr = (and (not leaf-0) (not leaf-1))", sarg.toString());
    }

View Full Code Here

    @Test
    public void testBetweenExpression() throws Exception {
        // TODO: Add support for OP_BETWEEN expression type
        String q = query + "b = filter a by srcid > 10 or srcid < 20;" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (LESS_THAN_EQUALS srcid 10)\n" +
                "leaf-1 = (LESS_THAN srcid 20)\n" +
                "expr = (or (not leaf-0) leaf-1)", sarg.toString());
    }

View Full Code Here

    @Test
    public void testInExpression() throws Exception {
        // TODO: Add support for OP_IN expression type
        String q = query + "b = filter a by srcid == 10 or srcid == 11;" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "leaf-1 = (EQUALS srcid 11)\n" +
                "expr = (or leaf-0 leaf-1)", sarg.toString());
    }

View Full Code Here

    public void testNegativeMatchesExpr() throws Exception {
        // matches operator is not a supported op type
        String q = query + "b = filter a by name matches 'foo*';" + "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("name"));
        Assert.assertNull(expr);
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        Assert.assertNull(sarg);


        // AND in LHS/RHS
        q = query + "b = filter a by name matches 'foo*' and srcid == 10;" + "store b into 'out';";
        expr = getExpressionForTest(q, Arrays.asList("srcid", "name"));
        sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "expr = leaf-0", sarg.toString());


        q = query + "b = filter a by srcid == 10 and name matches 'foo*';" + "store b into 'out';";
        expr = getExpressionForTest(q, Arrays.asList("srcid", "name"));
        sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "expr = leaf-0", sarg.toString());


        // OR - Nothing should be pushed
        q = query + "b = filter a by name matches 'foo*' or srcid == 10;" + "store b into 'out';";
        expr = getExpressionForTest(q, Arrays.asList("srcid", "name"));
        Assert.assertNull(expr);

View Full Code Here

        //Struct, Map and Bag are not supported
        // TODO: Change the test to use ORCStorage to test OrcStorage.getPredicateFields()
        String q = query + "b = filter a by srcid == 10 and browser#'type' == 'IE';" +
                "store b into 'out';";
        Expression expr = getExpressionForTest(q, Arrays.asList("srcid"));
        SearchArgument sarg = orcStorage.getSearchArgument(expr);
        assertEquals("leaf-0 = (EQUALS srcid 10)\n" +
                "expr = leaf-0", sarg.toString());
    }

View Full Code Here

    if ((sargPushdown == null && serializedPushdown == null)
        || columnNamesString == null) {
      LOG.debug("No ORC pushdown predicate");
      options.searchArgument(null, null);
    } else {
      SearchArgument sarg;
      if (serializedPushdown != null) {
        sarg = SearchArgumentFactory.create
            (Utilities.deserializeExpression(serializedPushdown));
      } else {
        sarg = SearchArgumentFactory.create(sargPushdown);

View Full Code Here

          setIncludedColumns(options, types, context.conf, isOriginal);
          setSearchArgument(options, types, context.conf, isOriginal);
          // only do split pruning if HIVE-8732 has been fixed in the writer
          if (options.getSearchArgument() != null &&
              writerVersion != OrcFile.WriterVersion.ORIGINAL) {
            SearchArgument sarg = options.getSearchArgument();
            List<PredicateLeaf> sargLeaves = sarg.getLeaves();
            List<StripeStatistics> stripeStats = metadata.getStripeStatistics();
            int[] filterColumns = RecordReaderImpl.mapSargColumns(sargLeaves,
                options.getColumnNames(), getRootColumn(isOriginal));


            if (stripeStats != null) {

View Full Code Here

    return vals;
  }


  @Test
  public void testBuilder() throws Exception {
    SearchArgument sarg =
        SearchArgumentFactory.newBuilder()
            .startAnd()
              .lessThan("x", 10)
              .lessThanEquals("y", "hi")
              .equals("z", 1.0)
            .end()
            .build();
    assertEquals("leaf-0 = (LESS_THAN x 10)\n" +
        "leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
        "leaf-2 = (EQUALS z 1.0)\n" +
        "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
    sarg = SearchArgumentFactory.newBuilder()
        .startNot()
           .startOr()
             .isNull("x")
             .between("y", 10, 20)
             .in("z", 1, 2, 3)
             .nullSafeEquals("a", "stinger")
           .end()
        .end()
        .build();
    assertEquals("leaf-0 = (IS_NULL x)\n" +
        "leaf-1 = (BETWEEN y 10 20)\n" +
        "leaf-2 = (IN z 1 2 3)\n" +
        "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
        "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.hive.ql.io.sarg.SearchArgument

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$OrcRecordReader

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator

org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat

org.apache.hadoop.hive.ql.io.orc.TestOrcFile

org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl

org.apache.pig.builtin.OrcStorage

org.apache.pig.builtin.TestOrcStoragePushdown

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.