Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOFilter


    }

    @Override
    public boolean check(List<LogicalOperator> nodes) throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            List<LogicalOperator> predecessors = (mPlan.getPredecessors(filter) == null ? null
                    : new ArrayList<LogicalOperator>(mPlan
                            .getPredecessors(filter)));

            // if there are no predecessors return false
            if (predecessors == null) {
                return false;
            }

            // if the filter has no predecessors or more than one predecessor
            // return false
            if (predecessors.size() == 0 || predecessors.size() > 1) {
                return false;
            }
               
            LogicalOperator predecessor = predecessors.get(0);

            // if the predecessor is one of LOLoad/LOStore/LOStream/LOLimit
            // return false
            if (predecessor instanceof LOLoad || predecessor instanceof LOStore
                    || predecessor instanceof LOStream
                    || predecessor instanceof LOLimit) {
                return false;
            }
           
            // TODO
            // for now filters cannot be combined
            // remove this check when filters can be combined
            if (predecessor instanceof LOFilter)
                return false;

            // TODO
            // same rule as filters
            if (predecessor instanceof LOSplitOutput) {
                return false;
            }
            if (predecessor instanceof LOSplit) {
                return false;
            }

            UDFFinder udfFinder = new UDFFinder(filter.getComparisonPlan());
            udfFinder.visit();

            // if the filter's inner plan contains any UDF then return false
            if (udfFinder.foundAnyUDF()) {
                return false;
            }

            CastFinder castFinder = new CastFinder(filter.getComparisonPlan());
            castFinder.visit();

            // if the filter's inner plan contains any casts then return false
            if (castFinder.foundAnyCast()) {
                return false;
            }

            List<RequiredFields> filterRequiredFields = filter
                    .getRequiredFields();
            if (filterRequiredFields == null) {
                return false;
            }
            RequiredFields requiredField = filterRequiredFields.get(0);
View Full Code Here


    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            LogicalOperator predecessor = mPlan.getPredecessors(filter).get(0);
            if (mSwap) {
                mPlan.swap(predecessor, filter);
            } else if (mPushBefore) {
                if (mPushBeforeInput == -1) {
View Full Code Here

    public void testQueryFilterNoSchema() {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        List<RequiredFields> filterRelevantFields0 = filter.getRelevantInputs(0, 0);
        assertTrue(filterRelevantFields0.get(0).getFields().size() == 1);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getFields().contains(new Pair<Integer, Integer>(0, 0)));
    }
View Full Code Here

    @Test
    public void testQueryFilterWithSchema() {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        List<RequiredFields> filterRelevantFields0 = filter.getRelevantInputs(0, 0);
        assertTrue(filterRelevantFields0.get(0).getFields().size() == 1);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getFields().contains(new Pair<Integer, Integer>(0, 0)));
    }
View Full Code Here

    @Test
    public void testQueryFilterWithStarNoSchema() {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        RequiredFields filterRelevantFields0 = filter.getRelevantInputs(0, 0).get(0);
        assertTrue(filterRelevantFields0.getNeedAllFields() == false);
        assertTrue(filterRelevantFields0.getNeedNoFields() == false);
        assertTrue(filterRelevantFields0.getFields().size() == 1);
        assertTrue(filterRelevantFields0.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields filterRelevantFields1 = filter.getRelevantInputs(0, 1).get(0);
        assertTrue(filterRelevantFields1.getNeedAllFields() == false);
        assertTrue(filterRelevantFields1.getNeedNoFields() == false);
        assertTrue(filterRelevantFields1.getFields().size() == 1);
        assertTrue(filterRelevantFields1.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

    @Test
    public void testQueryFilterStarWithSchema() {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        RequiredFields filterRelevantFields0 = filter.getRelevantInputs(0, 0).get(0);
        assertTrue(filterRelevantFields0.getNeedAllFields() == false);
        assertTrue(filterRelevantFields0.getNeedNoFields() == false);
        assertTrue(filterRelevantFields0.getFields().size() == 1);
        assertTrue(filterRelevantFields0.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields filterRelevantFields1 = filter.getRelevantInputs(0, 1).get(0);
        assertTrue(filterRelevantFields1.getNeedAllFields() == false);
        assertTrue(filterRelevantFields1.getNeedNoFields() == false);
        assertTrue(filterRelevantFields1.getFields().size() == 1);
        assertTrue(filterRelevantFields1.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

    }

    @Override
    public boolean check(List<LogicalOperator> nodes) throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            List<LogicalOperator> predecessors = (mPlan.getPredecessors(filter) == null ? null
                    : new ArrayList<LogicalOperator>(mPlan
                            .getPredecessors(filter)));

            // if there are no predecessors return false
            if (predecessors == null) {
                return false;
            }

            // if the filter has no predecessors or more than one predecessor
            // return false
            if (predecessors.size() == 0 || predecessors.size() > 1) {
                return false;
            }
               
            LogicalOperator predecessor = predecessors.get(0);

            // if the predecessor is one of LOLoad/LOStore/LOStream/LOLimit
            // return false
            if (predecessor instanceof LOLoad || predecessor instanceof LOStore
                    || predecessor instanceof LOStream
                    || predecessor instanceof LOLimit) {
                return false;
            }
           
            // TODO
            // for now filters cannot be combined
            // remove this check when filters can be combined
            if (predecessor instanceof LOFilter)
                return false;

            // TODO
            // same rule as filters
            if (predecessor instanceof LOSplitOutput) {
                return false;
            }
            if (predecessor instanceof LOSplit) {
                return false;
            }

            UDFFinder udfFinder = new UDFFinder(filter.getComparisonPlan());
            udfFinder.visit();

            // if the filter's inner plan contains any UDF then return false
            if (udfFinder.foundAnyUDF()) {
                return false;
            }

            CastFinder castFinder = new CastFinder(filter.getComparisonPlan());
            castFinder.visit();

            // if the filter's inner plan contains any casts then return false
            if (castFinder.foundAnyCast()) {
                return false;
            }

            List<RequiredFields> filterRequiredFields = filter
                    .getRequiredFields();
            if (filterRequiredFields == null) {
                return false;
            }
            RequiredFields requiredField = filterRequiredFields.get(0);
View Full Code Here

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            LogicalOperator predecessor = mPlan.getPredecessors(filter).get(0);
            if (mSwap) {
                mPlan.swap(predecessor, filter);
            } else if (mPushBefore) {
                if (mPushBeforeInput == -1) {
View Full Code Here

    @Test
    public void testErrNull() throws VisitorException, IOException {
        String query = "filter (load 'a') by $0 is null;";
        LogicalPlan plan = buildPlan(query);
        LOFilter filter = (LOFilter)plan.getLeaves().get(0);
        LogicalPlan innerPlan = filter.getComparisonPlan();
        innerPlan.remove(innerPlan.getRoots().get(0));
        try {
            buildPhysicalPlan(plan);
            fail("Expected error.");
        } catch(VisitorException ve) {
View Full Code Here

    @Test
    public void testErrNull() throws VisitorException, IOException {
        String query = "filter (load 'a') by $0 is null;";
        LogicalPlan plan = buildPlan(query);
        LOFilter filter = (LOFilter)plan.getLeaves().get(0);
        LogicalPlan innerPlan = filter.getComparisonPlan();
        innerPlan.remove(innerPlan.getRoots().get(0));
        try {
            buildPhysicalPlan(plan);
            fail("Expected error.");
        } catch(VisitorException ve) {
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOFilter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.