Examples of org.apache.pig.newplan.PColFilterExtractor

org.apache.pig.newplan.PColFilterExtractor
This Visitor works on the filter condition of a LOFilter which immediately follows a LOLoad that interacts with a metadata system (currently OWL) to read table data. The visitor looks for conditions on partition columns in the filter condition and extracts those conditions out of the filter condition. The condition on partition cols will be used to prune partitions of the table.

        @Override
        public void transform(OperatorPlan matched) throws FrontendException {
          subPlan = new OperatorSubPlan( currentPlan );


          setupColNameMaps();
          PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
              loFilter.getFilterPlan(), getMappedKeys( partitionKeys ) );
          pColFilterFinder.visit();
          Expression partitionFilter = pColFilterFinder.getPColCondition();
          if(partitionFilter != null) {
            // the column names in the filter may be the ones provided by
            // the user in the schema in the load statement - we may need
            // to replace them with partition column names as given by
            // LoadFunc.getSchema()
            updateMappedColNames(partitionFilter);
            try {
          loadMetadata.setPartitionFilter(partitionFilter);
        } catch (IOException e) {
          throw new FrontendException( e );
        }
            if(pColFilterFinder.isFilterRemovable()) {  
              currentPlan.removeAndReconnect( loFilter );
            }
          }
        }

View Full Code Here

    private PColFilterExtractor test(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws IOException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
           Assert.assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
           Assert.assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
           Assert.assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                                toString().toLowerCase();
            Assert.assertEquals("checking trimmed filter expression:", expFilterString,

View Full Code Here

    
    private void negativeTest(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) throws VisitorException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
           Assert.assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
        
        String actual = extractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:", 
                "((f5 >= 20) and (f3 == 15))", actual);
    }

View Full Code Here

                null,
                TestLoader.partFilter);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
        
        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().
                getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:", 
                "(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);

View Full Code Here

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
        
        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:", 
                "((age >= 20) and (f3 == 15))", actual);
    }

View Full Code Here

    throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();


        if(expPartFilterString == null) {
            Assert.assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
            Assert.assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }


        if(expFilterString == null) {
            Assert.assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = pColExtractor.getExpression(
                    (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                    toString().toLowerCase();
            Assert.assertEquals("checking trimmed filter expression:", expFilterString,
                    actual);
        }

View Full Code Here

            int expectedErrorCode) throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            Assert.assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

          // original filter plan only if the transformation is successful 
          // (i.e. partition filter can be pushed down) 
          LogicalExpressionPlan filterExpr = loFilter.getFilterPlan();
          LogicalExpressionPlan filterExprCopy = filterExpr.deepCopy();
          
          PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                  filterExprCopy, getMappedKeys( partitionKeys ) );
          pColFilterFinder.visit();
          Expression partitionFilter = pColFilterFinder.getPColCondition();
          
          if(partitionFilter != null) {
            // the column names in the filter may be the ones provided by
            // the user in the schema in the load statement - we may need
            // to replace them with partition column names as given by
            // LoadFunc.getSchema()
            updateMappedColNames(partitionFilter);
            try {
          loadMetadata.setPartitionFilter(partitionFilter);
        } catch (IOException e) {
          throw new FrontendException( e );
        }
            if(pColFilterFinder.isFilterRemovable()) {  
              currentPlan.removeAndReconnect( loFilter );
            } else {
                loFilter.setFilterPlan(filterExprCopy);
            }
          }

View Full Code Here

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);


        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());


        String actual = extractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((f5 >= 20) and (f3 == 15))", actual);
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.pig.newplan.PColFilterExtractor

org.apache.pig.Expression

org.apache.pig.newplan.DepthFirstWalker

org.apache.pig.newplan.logical.expression.BinaryExpression

org.apache.pig.newplan.logical.expression.ConstantExpression

org.apache.pig.newplan.logical.expression.LogicalExpression

org.apache.pig.newplan.logical.expression.ProjectExpression

org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer$PartitionFilterPushDownTransformer

org.apache.pig.test.TestPartitionFilterPushDown

org.apache.pig.impl.logicalLayer.FrontendException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.