Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
    // func, and I don't want to mess with that here.
   
    @Test
    public void testJoinDifferentJoinTypes() throws FrontendException {
       LogicalPlan lp = new LogicalPlan();
       LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, null);
       lp.add(A1);
       
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, null);
        lp.add(B1);
       
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
       
        // A = load
        LogicalSchema jaschema2 = new LogicalSchema();
        jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A2 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema2, lp, null);
        lp.add(A2);
       
        // B = load
        LogicalSchema jbschema2 = new LogicalSchema();
        jbschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B2 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema2, lp, null);
        lp.add(B2);
       
        // C = join
        LogicalSchema jcschema2 = new LogicalSchema();
        jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan2 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan2, 0, 0, null);
        LogicalExpressionPlan bprojplan2 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan2, 1, 0, null);
View Full Code Here
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
    }
   
    @Test
    public void testJoinDifferentInner() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
               LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, null);
       lp.add(A1);
       
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, null);
        lp.add(B1);
       
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
       
        // Test different inner status
        // A = load
        LogicalSchema jaschema3 = new LogicalSchema();
        jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A3 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema3, lp, null);
        lp.add(A3);
       
        // B = load
        LogicalSchema jbschema3 = new LogicalSchema();
        jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B3 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema3, lp, null);
        lp.add(B3);
       
        // C = join
        LogicalSchema jcschema3 = new LogicalSchema();
        jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan3 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan3, 0, 0, null);
        LogicalExpressionPlan bprojplan3 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan3, 1, 0, null);
View Full Code Here
129
130
131
132
133
134
135
136
137
        Assert.assertEquals( 0, forEachCount1 - forEachCount2 );
        int outputExprCount2 = getOutputExprCount( newLogicalPlan );
        Assert.assertTrue( outputExprCount1 == outputExprCount2 );
        LOForEach foreach2 = getForEachOperator( newLogicalPlan );
        Assert.assertTrue( foreach2.getAlias().equals( "C" ) );
        LogicalSchema newSchema = foreach2.getSchema();
        Assert.assertTrue(newSchema.getField(0).alias.equals("x"));
        Assert.assertTrue(newSchema.getField(1).alias.equals("y"));
    }
View Full Code Here
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
       
        org.apache.pig.newplan.logical.relational.LogicalPlan newLogicalPlan = migratePlan(plan);
        LogicalRelationalOperator ld =  (LogicalRelationalOperator)newLogicalPlan.getSources().get(0);
        assertEquals( LOLoad.class, ld.getClass() );
        LOLoad load = (LOLoad)ld;
        LogicalSchema ls = load.getSchema();
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
            (LogicalRelationalOperator) newLogicalPlan.getSuccessors(load).get(0);
        assertEquals( LOForEach.class, fe.getClass() );
        LOForEach forEach = (LOForEach)fe;
       
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 2, gen.getOutputPlans().size() );
       
        LogicalExpressionPlan genExp1 = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp1.getSources().size() );
        assertEquals( ProjectExpression.class, genExp1.getSources().get(0).getClass() );
        ProjectExpression prj1  = (ProjectExpression) genExp1.getSources().get(0);
        assertEquals( ls.getField(0).uid, prj1.getFieldSchema().uid );
       
        LogicalExpressionPlan genExp2 = gen.getOutputPlans().get(1);
        assertEquals( UserFuncExpression.class, genExp2.getSources().get(0).getClass() );
        assertEquals( ProjectExpression.class, genExp2.getSinks().get(0).getClass() );
        ProjectExpression prj2 = (ProjectExpression)genExp2.getSinks().get(0);
        assertEquals( ls.getField(1).uid, prj2.getFieldSchema().uid );
        assertEquals( DereferenceExpression.class, genExp2.getPredecessors(prj2).get(0).getClass() );
        assertEquals( 0, (int)((DereferenceExpression)genExp2.getPredecessors(prj2).get(0)).getBagColumns().get(0) );
       
        assertEquals( 1, inputPln.getRoots().size() );
        assertEquals( POProject.class, inputPln.getRoots().get(0).getClass() );
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        Assert.assertTrue(tupleFieldSchema.schema.size()==1);
        Assert.assertTrue(tupleFieldSchema.schema.getField(0).type==DataType.BYTEARRAY);
    }
View Full Code Here
94
95
96
97
98
99
100
101
102
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        Assert.assertTrue(schema.getField(0).type==DataType.BYTEARRAY);
    }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        Assert.assertTrue(tupleFieldSchema.schema==null);
    }
View Full Code Here
687
688
689
690
691
692
693
694
695
696
   
    @Test
    // See PIG-730
    public void testMergeSchemaWithTwoLevelAccess3() throws Exception {
        // Generate two schemas
        LogicalSchema ls1 = Utils.parseSchema("a:{t:(a0:int, a1:int)}");
        LogicalSchema ls2 = Utils.parseSchema("b:{t:(b0:int, b1:int)}");
        LogicalSchema ls3 = LogicalSchema.merge(ls1, ls2, MergeMode.LoadForEach);
        Assert.assertTrue(org.apache.pig.newplan.logical.Util.translateSchema(ls3).toString().equals("{a: {(a0: int,a1: int)}}"));
    }
View Full Code Here
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
        Assert.assertTrue(org.apache.pig.newplan.logical.Util.translateSchema(ls3).toString().equals("{a: {(a0: int,a1: int)}}"));
    }
   
    @Test
    public void testNewNormalNestedMerge1() throws Exception {
        LogicalSchema a = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:bytearray, b1:(b11:int, b12:float), c1:long"));
        LogicalSchema b = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a2:bytearray, b2:(b21:double, b22:long), c2:int"));
       
        LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
        LogicalSchema expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:bytearray, b1:(), c1:long"));
        expected.getField(1).schema = new LogicalSchema();
        Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
       
        mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
        expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:bytearray, b1:(b11:int, b12:float), c1:long"));
View Full Code Here
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
   
   
   
    @Test
    public void testNewNormalNestedMerge2() throws Exception {
        LogicalSchema a = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
        LogicalSchema b = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a2:bytearray, b2:(b21:double, b22:long), c2:chararray"));
       
        LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
        LogicalSchema expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:(a11:chararray, a12:float), b1:(), c1:bytearray"));
        expected.getField(1).schema = new LogicalSchema();
        Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
       
        mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
        expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema

  • org.apache.pig.impl.util.Utils
  • org.apache.pig.newplan.logical.expression.DereferenceExpression
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.expression.UserFuncExpression
  • org.apache.pig.newplan.logical.LogicalPlanMigrationVistor
  • org.apache.pig.newplan.logical.optimizer.ProjectionPatcher$ProjectionRewriter
  • org.apache.pig.newplan.logical.rules.AddForEach$AddForEachTransformer
  • org.apache.pig.newplan.logical.rules.ColumnPruneHelper
  • org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor
  • org.apache.pig.newplan.logical.rules.ColumnPruneVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.