Package org.apache.hadoop.hive.ql.io.sarg

Examples of org.apache.hadoop.hive.ql.io.sarg.SearchArgument

66
67
68
69
70
71
72
73
74
75
76
                    long offset, long length) throws IOException {
      String serializedPushdown = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
      String columnNamesString =
          conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR);
      String[] columnNames = null;
      SearchArgument sarg = null;
      List<OrcProto.Type> types = file.getTypes();
      if (types.size() == 0) {
        numColumns = 0;
      } else {
        numColumns = types.get(0).getSubtypesCount();
View Full Code Here
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath);
    assertEquals(3500, reader.getNumberOfRows());

    SearchArgument sarg = SearchArgument.FACTORY.newBuilder()
        .startAnd()
          .startNot()
             .lessThan("int1", 300000)
          .end()
          .lessThan("int1", 600000)
View Full Code Here
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
    types.add(builder.build());
    builder.clear().setKind(OrcProto.Type.Kind.INT);
    types.add(builder.build());
    types.add(builder.build());
    types.add(builder.build());
    SearchArgument isNull = SearchArgument.FACTORY.newBuilder()
        .startAnd().isNull("cost").end().build();
    conf.set(OrcInputFormat.SARG_PUSHDOWN, isNull.toKryo());
    conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,
        "url,cost");
    options.include(new boolean[]{true, true, false, true, false});
    OrcInputFormat.setSearchArgument(options, types, conf, false);
    String[] colNames = options.getColumnNames();
    assertEquals(null, colNames[0]);
    assertEquals("url", colNames[1]);
    assertEquals(null, colNames[2]);
    assertEquals("cost", colNames[3]);
    assertEquals(null, colNames[4]);
    SearchArgument arg = options.getSearchArgument();
    List<PredicateLeaf> leaves = arg.getLeaves();
    assertEquals("cost", leaves.get(0).getColumnName());
    assertEquals(PredicateLeaf.Operator.IS_NULL, leaves.get(0).getOperator());
  }
View Full Code Here
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
    writer.close();
    Reader reader = OrcFile.createReader(testFilePath,
        OrcFile.readerOptions(conf).filesystem(fs));
    assertEquals(3500, reader.getNumberOfRows());

    SearchArgument sarg = SearchArgument.FACTORY.newBuilder()
        .startAnd()
          .startNot()
             .lessThan("int1", 300000)
          .end()
          .lessThan("int1", 600000)
View Full Code Here
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
        return types;
    }

    @Override
    public void setPushdownPredicate(Expression expr) throws IOException {
        SearchArgument sArg = getSearchArgument(expr);
        if (sArg != null) {
            log.info("Pushdown predicate expression is " + expr);
            log.info("Pushdown predicate SearchArgument is:\n" + sArg);
            Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
            try {
                p.setProperty(signature + SearchArgsSuffix, sArg.toKryo());
            } catch (Exception e) {
                throw new IOException("Cannot serialize SearchArgument: " + sArg);
            }
        }
    }
View Full Code Here
583
584
585
586
587
588
589
590
        }
        buildSearchArgument(expr, builder);
        if (beginWithAnd) {
            builder.end();
        }
        SearchArgument sArg = builder.build();
        return sArg;
    }
View Full Code Here
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
    types.add(builder.build());
    builder.clear().setKind(OrcProto.Type.Kind.INT);
    types.add(builder.build());
    types.add(builder.build());
    types.add(builder.build());
    SearchArgument isNull = SearchArgument.FACTORY.newBuilder()
        .startAnd().isNull("cost").end().build();
    conf.set(OrcInputFormat.SARG_PUSHDOWN, isNull.toKryo());
    conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,
        "url,cost");
    options.include(new boolean[]{true, true, false, true, false});
    OrcInputFormat.setSearchArgument(options, types, conf, false);
    String[] colNames = options.getColumnNames();
    assertEquals(null, colNames[0]);
    assertEquals("url", colNames[1]);
    assertEquals(null, colNames[2]);
    assertEquals("cost", colNames[3]);
    assertEquals(null, colNames[4]);
    SearchArgument arg = options.getSearchArgument();
    List<PredicateLeaf> leaves = arg.getLeaves();
    assertEquals("cost", leaves.get(0).getColumnName());
    assertEquals(PredicateLeaf.Operator.IS_NULL, leaves.get(0).getOperator());
  }
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    if ((sargPushdown == null && serializedPushdown == null)
        || columnNamesString == null) {
      LOG.debug("No ORC pushdown predicate");
      options.searchArgument(null, null);
    } else {
      SearchArgument sarg;
      if (serializedPushdown != null) {
        sarg = SearchArgument.FACTORY.create
            (Utilities.deserializeExpression(serializedPushdown));
      } else {
        sarg = SearchArgument.FACTORY.create(sargPushdown);
View Full Code Here
752
753
754
755
756
757
758
759
760
761
762
763
        if (deltas.isEmpty()) {
          Reader.Options options = new Reader.Options();
          setIncludedColumns(options, types, context.conf, isOriginal);
          setSearchArgument(options, types, context.conf, isOriginal);
          if (options.getSearchArgument() != null) {
            SearchArgument sarg = options.getSearchArgument();
            List<PredicateLeaf> sargLeaves = sarg.getLeaves();
            List<StripeStatistics> stripeStats = metadata.getStripeStatistics();
            int[] filterColumns = RecordReaderImpl.mapSargColumns(sargLeaves,
                options.getColumnNames(), getRootColumn(isOriginal));

            if (stripeStats != null) {
View Full Code Here
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
    writer.close();
    Reader reader = OrcFile.createReader(testFilePath,
        OrcFile.readerOptions(conf).filesystem(fs));
    assertEquals(3500, reader.getNumberOfRows());

    SearchArgument sarg = SearchArgument.FACTORY.newBuilder()
        .startAnd()
          .startNot()
             .lessThan("int1", 300000)
          .end()
          .lessThan("int1", 600000)
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.io.sarg.SearchArgument

  • org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
  • org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$OrcRecordReader
  • org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator
  • org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat
  • org.apache.hadoop.hive.ql.io.orc.TestOrcFile
  • org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl
  • org.apache.pig.builtin.OrcStorage
  • org.apache.pig.builtin.TestOrcStoragePushdown
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.