Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOFilter

1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
            // Create inner plan
            LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();

            // filter
            LOFilter filter1 = new LOFilter(plan);
            filter1.setFilterPlan(innerPlan);

            ProjectExpression project1 = new ProjectExpression(innerPlan, 0, 0, filter1);
            ProjectExpression project2 = new ProjectExpression(innerPlan, 0, 1, filter1);

            AddExpression add1 = new AddExpression(innerPlan, project1, project2);
View Full Code Here
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
            }
            return foreach;
        }

        private CastExpression getCastFromLastFilter(String query) throws FrontendException {
            LOFilter filter = getFilterFromPlan(query);

            LogicalExpressionPlan filterPlan = filter.getFilterPlan();
            return getCastFromExpPlan(filterPlan);
        }
View Full Code Here
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
        }


        private LOFilter getFilterFromPlan(String query) throws FrontendException {
            LogicalPlan plan = createAndProcessLPlan(query);
            LOFilter filter = null;

            for(Operator op : plan.getSinks()){
                if(op instanceof LOFilter){
                    if(filter != null){
                        fail("more than one sink foreach found in plan");
View Full Code Here
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
            checkLastFilterCast(query, "org.apache.pig.builtin.PigStorage");
        }

        private void checkLastFilterCast(String query, String loadFuncStr) throws FrontendException {
            LogicalPlan plan = createAndProcessLPlan(query);
            LOFilter filter = (LOFilter)plan.getSinks().get(0);
            LogicalExpressionPlan filterPlan = filter.getFilterPlan();

            LogicalExpression exOp = (LogicalExpression)filterPlan.getSinks().get(0);

            if(! (exOp instanceof ProjectExpression)) exOp = (LogicalExpression) filterPlan.getSinks().get(1);
View Full Code Here
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
            + "d = foreach c {fil = filter b by f2 == 1; generate flatten(group), SUM(fil.f3);};";

            LOForEach foreach = getForeachFromPlan(query);
            LogicalPlan innerPlan = foreach.getInnerPlan();

            LOFilter filter = null;
            Iterator<Operator> iter = innerPlan.getOperators();
            while(iter.hasNext()){
                Operator op = iter.next();
                if(op instanceof LOFilter)
                    filter = (LOFilter)op;
            }

            LogicalExpressionPlan filterPlan = filter.getFilterPlan();
            CastExpression cast = getCastFromExpPlan(filterPlan);
            assertTrue(cast.getFuncSpec().getClassName().startsWith("org.apache.pig.builtin.PigStorage"));
        }
View Full Code Here
908
909
910
911
912
913
914
915
916
917
918
919
920
       
        LogicalRelationalOperator fil = (LogicalRelationalOperator)
        newLogicalPlan.getSuccessors( newLogicalPlan.getSources().get(0) ).get(0);
        assertEquals( LOFilter.class,
                fil.getClass() );
        LOFilter filter = (LOFilter)fil;
       
        LogicalExpressionPlan filPlan = filter.getFilterPlan();
       
        assertEquals( 1, filPlan.getSources().size() );
        assertEquals( 2, filPlan.size() );
        assertEquals( 1, filPlan.getSinks().size() );
        assertEquals( IsNullExpression.class, filPlan.getSources().get(0).getClass() );
View Full Code Here
944
945
946
947
948
949
950
951
952
953
954
955
956
       
        LogicalRelationalOperator fil = (LogicalRelationalOperator)
        newLogicalPlan.getSuccessors( newLogicalPlan.getSources().get(0) ).get(0);
        assertEquals( LOFilter.class,
                fil.getClass() );
        LOFilter filter = (LOFilter)fil;
       
        LogicalExpressionPlan filPlan = filter.getFilterPlan();
       
        assertEquals( 1, filPlan.getSources().size() );
        assertEquals( 3, filPlan.size() );
        assertEquals( 1, filPlan.getSinks().size() );
        assertEquals( NotExpression.class, filPlan.getSources().get(0).getClass() );
View Full Code Here
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
            String expPartFilterString, String expFilterString)
    throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();

        if(expPartFilterString == null) {
            Assert.assertEquals("Checking partition column filter:", null,
                    pColExtractor.getPColCondition());
        } else  {
            Assert.assertEquals("Checking partition column filter:",
                    expPartFilterString.toLowerCase(),
                    pColExtractor.getPColCondition().toString().toLowerCase());  
        }

        if(expFilterString == null) {
            Assert.assertTrue("Check that filter can be removed:",
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = pColExtractor.getExpression(
                    (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                    toString().toLowerCase();
            Assert.assertEquals("checking trimmed filter expression:", expFilterString,
                    actual);
        }
        return pColExtractor;
View Full Code Here
609
610
611
612
613
614
615
616
617
618
619
620
621
    private void negativeTest(String query, List<String> partitionCols,
            int expectedErrorCode) throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            Assert.assertEquals("Checking if exception has right error code",
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
View Full Code Here
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, null);
            ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
            new EqualExpression(filterPlan, fy, fc);
            LOFilter D = new LOFilter(lp, filterPlan);
            D.neverUseForRealSetSchema(cschema);
            lp.add(D);
            lp.connect(C, D);
        }
       
        // Build a second similar plan to test equality
        // A = load
        LogicalPlan lp1 = new LogicalPlan();
        {
            LogicalSchema aschema = new LogicalSchema();
            aschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
            LOLoad A = new LOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema, lp1, null);
            lp1.add(A);
           
            // B = load
            LogicalSchema bschema = new LogicalSchema();
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
            LOLoad B = new LOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), bschema, lp1, null);
            lp1.add(B);
           
            // C = join
            LogicalSchema cschema = new LogicalSchema();
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
            LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
            new ProjectExpression(aprojplan, 0, 0, null);
            LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
            new ProjectExpression(bprojplan, 1, 0, null);
            MultiMap<Integer, LogicalExpressionPlan> mm =
                new MultiMap<Integer, LogicalExpressionPlan>();
            mm.put(0, aprojplan);
            mm.put(1, bprojplan);
            LOJoin C = new LOJoin(lp1, mm, JOINTYPE.HASH, new boolean[] {true, true});
            C.neverUseForRealSetSchema(cschema);
            lp1.add(C);
            lp1.connect(A, C);
            lp1.connect(B, C);
               
           
            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, null);
            ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
            new EqualExpression(filterPlan, fy, fc);
            LOFilter D = new LOFilter(lp1, filterPlan);
            D.neverUseForRealSetSchema(cschema);
            lp1.add(D);
            lp1.connect(C, D);
               
        }
       
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOFilter

  • org.apache.pig.builtin.TestOrcStoragePushdown
  • org.apache.pig.newplan.logical.rules.FilterAboveForeach
  • org.apache.pig.newplan.logical.rules.FilterConstantCalculator
  • org.apache.pig.newplan.logical.rules.LogicalExpressionSimplifier
  • org.apache.pig.newplan.logical.rules.LogicalExpressionSimplifier$LogicalExpressionSimplifierTransformer
  • org.apache.pig.newplan.logical.rules.MergeFilter
  • org.apache.pig.newplan.logical.rules.MergeFilter$MergeFilterTransformer
  • org.apache.pig.newplan.logical.rules.PushUpFilter
  • org.apache.pig.newplan.logical.rules.PushUpFilter$PushUpFilterTransformer
  • org.apache.pig.newplan.logical.rules.SplitFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.