Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOFilter

172
173
174
175
176
177
178
    Map<String, Operator> getOperators() {
        return operators;
    }

    LOFilter createFilterOp() {
        return new LOFilter( plan );
    }
View Full Code Here
180
181
182
183
184
185
186
    LOLimit createLimitOp() {
        return new LOLimit( plan );
    }

    LOFilter createSampleOp() {
        return new LOFilter( plan, true );
    }
View Full Code Here
223
224
225
226
227
228
229
230
        //  Generate a filter condition.
        LogicalExpression konst = new ConstantExpression( filterPlan, value);
        konst.setLocation( valLoc );
        UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );
        new LessThanExpression( filterPlan, udf, konst );
        LOFilter filter = new LOFilter( plan, true );
        return buildFilterOp( loc, filter, alias, inputAlias, filterPlan );
    }
View Full Code Here
1575
1576
1577
1578
1579
1580
1581
    private long getNextId() {
        return getNextId(scope);
    }

    static LOFilter createNestedFilterOp(LogicalPlan plan) {
        return new LOFilter( plan );
    }
View Full Code Here
265
266
267
268
269
270
271
272
273
274
275
        FileSpec filespec1 = new FileSpec(generateTmpFile(inputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        FileSpec filespec2 = new FileSpec(generateTmpFile(outputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad(filespec1, null, plan, ConfigurationUtil.toConfiguration(dfs.getConfiguration()));
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface) PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null);

        LOFilter filter = new LOFilter(plan);

        plan.add(load);
        plan.add(store);
        plan.add(filter);
View Full Code Here
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, null);
            ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
            new EqualExpression(filterPlan, fy, fc);
            LOFilter D = new LOFilter(lp, filterPlan);
            D.neverUseForRealSetSchema(cschema);
            lp.add(D);
            lp.connect(C, D);
        }

        // Build a second similar plan to test equality
        // A = load
        LogicalPlan lp1 = new LogicalPlan();
        {
            LogicalSchema aschema = new LogicalSchema();
            aschema.addField(new LogicalSchema.LogicalFieldSchema(
                    "x", null, DataType.INTEGER));
            LOLoad A = newLOLoad(new FileSpec("/abc",
                    new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema,
                    lp1, conf);
            lp1.add(A);

            // B = load
            LogicalSchema bschema = new LogicalSchema();
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                    "y", null, DataType.INTEGER));
            LOLoad B = newLOLoad(new FileSpec("/def",
                    new FuncSpec("PigStorage", "\t")), bschema, lp1, conf);
            lp1.add(B);

            // C = join
            LogicalSchema cschema = new LogicalSchema();
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                    "x", null, DataType.INTEGER));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                    "y", null, DataType.INTEGER));
            LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
            new ProjectExpression(aprojplan, 0, 0, null);
            LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
            new ProjectExpression(bprojplan, 1, 0, null);
            MultiMap<Integer, LogicalExpressionPlan> mm =
                    new MultiMap<Integer, LogicalExpressionPlan>();
            mm.put(0, aprojplan);
            mm.put(1, bprojplan);
            LOJoin C = new LOJoin(lp1, mm, JOINTYPE.HASH, new boolean[] { true, true });
            C.neverUseForRealSetSchema(cschema);
            lp1.add(C);
            lp1.connect(A, C);
            lp1.connect(B, C);

            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, null);
            ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
            new EqualExpression(filterPlan, fy, fc);
            LOFilter D = new LOFilter(lp1, filterPlan);
            D.neverUseForRealSetSchema(cschema);
            lp1.add(D);
            lp1.connect(C, D);

        }
View Full Code Here
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 1, null);
        ConstantExpression fc1 = new ConstantExpression(fp1,
                new Integer(0));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp.add(D1);

        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
        ProjectExpression fy2 = new ProjectExpression(fp2, 0, 1, null);
        ConstantExpression fc2 = new ConstantExpression(fp2,
                new Integer(1));
        new EqualExpression(fp2, fy2, fc2);
        LOFilter D2 = new LOFilter(lp, fp2);
        D2.neverUseForRealSetSchema(cschema);
        lp.add(D2);

        assertFalse(D1.isEqual(D2));
    }
View Full Code Here
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
        ConstantExpression fc1 = new ConstantExpression(fp1,
                new Integer(0));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp1, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp1.add(D1);
        lp1.connect(A1, D1);

        LogicalPlan lp2 = new LogicalPlan();
        LOLoad A2 = newLOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
                conf);
        lp2.add(A2);

        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
        ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
        ConstantExpression fc2 = new ConstantExpression(fp2,
                new Integer(0));
        new EqualExpression(fp2, fy2, fc2);
        LOFilter D2 = new LOFilter(lp2, fp2);
        D2.neverUseForRealSetSchema(cschema);
        lp2.add(D2);
        lp2.connect(A2, D2);

        assertTrue(D1.isEqual(D2));
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
        j1.setAlias("C");
        plan.add(j1);
       
        // build an expression with no AND
        LogicalExpressionPlan p3 = new LogicalExpressionPlan();
        LogicalRelationalOperator f1 = new LOFilter(plan, p3);
       
        LogicalExpression lp3 = new ProjectExpression(p3, 0, 2, f1);
        LogicalExpression cont = new ConstantExpression(p3, new Integer(3));
        p3.add(lp3);
        p3.add(cont);      
        LogicalExpression eq = new EqualExpression(p3, lp3, cont);       
       
       
        f1.setAlias("D");
        plan.add(f1);
       
        LogicalRelationalOperator s1 = new LOStore(plan, null, null, null);
        plan.add(s1);      
       
View Full Code Here
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);

        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();
        LOFilter optimizedFilt = (LOFilter) newLogicalPlan.getSuccessors(newLogicalPlan.getSources().get(0)).get(0);

        //expected query is same as original query, optimizer should not combine
        // conditions involving non deterministic udfs
        query = "b = filter (load 'd.txt' as (a0, a1)) by RANDOM() > 0.1 and RANDOM() > 0.1;" +
            "store b into 'empty';";
        LogicalPlan expected = Util.buildLp(pigServer, query);
        LOFilter expectedFilt = (LOFilter) expected.getSuccessors(expected.getSources().get(0)).get(0);
        assertEquals(
                "size of filter expression plans",
                optimizedFilt.getFilterPlan().size(),
                expectedFilt.getFilterPlan().size()
        );

    }
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOFilter

  • org.apache.pig.builtin.TestOrcStoragePushdown
  • org.apache.pig.newplan.logical.rules.FilterAboveForeach
  • org.apache.pig.newplan.logical.rules.FilterConstantCalculator
  • org.apache.pig.newplan.logical.rules.LogicalExpressionSimplifier
  • org.apache.pig.newplan.logical.rules.LogicalExpressionSimplifier$LogicalExpressionSimplifierTransformer
  • org.apache.pig.newplan.logical.rules.MergeFilter
  • org.apache.pig.newplan.logical.rules.MergeFilter$MergeFilterTransformer
  • org.apache.pig.newplan.logical.rules.PushUpFilter
  • org.apache.pig.newplan.logical.rules.PushUpFilter$PushUpFilterTransformer
  • org.apache.pig.newplan.logical.rules.SplitFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.