Examples of org.apache.pig.newplan.PartitionFilterExtractor

org.apache.pig.newplan.PartitionFilterExtractor
This is a rewrite of {@code PColFilterExtractor}Extracts partition filters for interfaces implementing LoadMetaData

        public void transform(OperatorPlan matched) throws FrontendException {
          subPlan = new OperatorSubPlan( currentPlan );


          setupColNameMaps();


            FilterExtractor filterFinder = new PartitionFilterExtractor(loFilter.getFilterPlan(),
                    getMappedKeys(partitionKeys));
            filterFinder.visit();
            Expression partitionFilter = filterFinder.getPushDownExpression();


            if(partitionFilter != null) {
                // the column names in the filter may be the ones provided by
                // the user in the schema in the load statement - we may need
                // to replace them with partition column names as given by
                // LoadFunc.getSchema()
                updateMappedColNames(partitionFilter);
                try {
                    loadMetadata.setPartitionFilter(partitionFilter);
                } catch (IOException e) {
                    throw new FrontendException( e );
                }
                if(filterFinder.isFilterRemovable()) {
                    currentPlan.removeAndReconnect( loFilter );
                } else {
                    loFilter.setFilterPlan(filterFinder.getFilteredPlan());
                }
            }
        }

View Full Code Here


        if (filterExpr != null) {
            Operator op = newLogicalPlan.getSinks().get(0);
            LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);


            String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
                    .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
            Assert.assertEquals("checking trimmed filter expression:",
                    filterExpr, actual);
        } else {
            Iterator<Operator> it = newLogicalPlan.getOperators();

View Full Code Here

                    throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PartitionFilterExtractor pColExtractor = new PartitionFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();


        if(expPartFilterString == null) {
            Assert.assertEquals("Checking partition column filter:", null,
                    pColExtractor.getPushDownExpression());
        } else  {
            Assert.assertEquals("Checking partition column filter:",
                    expPartFilterString,
                    pColExtractor.getPushDownExpression().toString());
        }


        if (expFilterString == null) {
            Assert.assertTrue("Check that filter can be removed:",
                    pColExtractor.isFilterRemovable());
        } else {
            if (unsupportedExpression) {
                String actual = getTestExpression((LogicalExpression)pColExtractor.getFilteredPlan().getSources().get(0)).toString();
                Assert.assertEquals("checking trimmed filter expression:", expFilterString, actual);
            } else {
                String actual = pColExtractor.getExpression((LogicalExpression)pColExtractor.getFilteredPlan().getSources().get(0)).toString();
                Assert.assertEquals("checking trimmed filter expression:", expFilterString, actual);
            }
        }
        return pColExtractor;
    }

View Full Code Here

    private void negativeTest(String query, List<String> partitionCols) throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PartitionFilterExtractor extractor = new PartitionFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        extractor.visit();
        Assert.assertFalse(extractor.canPushDown());
    }

View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.PartitionFilterExtractor

org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer$PartitionFilterPushDownTransformer

org.apache.pig.test.TestNewPartitionFilterPushDown

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.