Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

org.apache.pig.impl.logicalLayer.PColFilterExtractor
This Visitor works on the filter condition of a LOFilter which immediately follows a LOLoad that interacts with a metadata system (currently OWL) to read table data. The visitor looks for conditions on partition columns in the filter condition and extracts those conditions out of the filter condition. The condition on partition cols will be used to prune partitions of the table.

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            setupColNameMaps();
            PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                    loFilter.getComparisonPlan(), getMappedKeys(partitionKeys));
            pColFilterFinder.visit();
            Expression partitionFilter = pColFilterFinder.getPColCondition();
            if(partitionFilter != null) {
                // the column names in the filter may be the ones provided by
                // the user in the schema in the load statement - we may need
                // to replace them with partition column names as given by
                // LoadFunc.getSchema()
                updateMappedColNames(partitionFilter);
                loadMetadata.setPartitionFilter(partitionFilter);
                if(pColFilterFinder.isFilterRemovable()) {
                    // remove this filter from the plan                  
                    mPlan.removeAndReconnect(loFilter);
                }
            }
        } catch (Exception e) {

View Full Code Here

    
    private PColFilterExtractor test(LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws FrontendException {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
            assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
            assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
            assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (ExpressionOperator) filter.getComparisonPlan().
                                getLeaves().get(0)).
                                toString().toLowerCase();

View Full Code Here

    }
    
    private void negativeTest(LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

    
    private PColFilterExtractor test(LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws FrontendException {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
            assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
            assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
            assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (ExpressionOperator) filter.getComparisonPlan().
                                getLeaves().get(0)).
                                toString().toLowerCase();

View Full Code Here

    }
    
    private void negativeTest(LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            setupColNameMaps();
            PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                    loFilter.getComparisonPlan(), getMappedKeys(partitionKeys));
            pColFilterFinder.visit();
            Expression partitionFilter = pColFilterFinder.getPColCondition();
            if(partitionFilter != null) {
                // the column names in the filter may be the ones provided by
                // the user in the schema in the load statement - we may need
                // to replace them with partition column names as given by
                // LoadFunc.getSchema()
                updateMappedColNames(partitionFilter);
                loadMetadata.setPartitionFilter(partitionFilter);
                if(pColFilterFinder.isFilterRemovable()) {
                    // remove this filter from the plan                  
                    mPlan.removeAndReconnect(loFilter);
                }
            }
        } catch (Exception e) {

View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.PColFilterExtractor

org.apache.pig.Expression

org.apache.pig.Expression.BinaryExpression

org.apache.pig.impl.logicalLayer.optimizer.PartitionFilterOptimizer

org.apache.pig.test.TestPartitionFilterOptimization

org.apache.pig.impl.plan.VisitorException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.