Package org.apache.pig.newplan.logical.relational.LogicalSchema

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
                            projExpr.setAttachedRelationalOp( op );
                        }
                    }
                }
                LogicalExpression root = (LogicalExpression)planCopy.getSources().get( 0 );// get the root of the plan
                LogicalFieldSchema schema;
                try {
                    schema = root.getFieldSchema();
                    if (schema.alias == null) {
                        schema.alias = colAlias;
                    }
View Full Code Here
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
     * @param fs fieldSchema of expecteResArray
     * @throws IOException
     */
    static public void checkQueryOutputsAfterSortRecursive(Iterator<Tuple> actualResultsIt,
            String[] expectedResArray, LogicalSchema schema) throws IOException {
        LogicalFieldSchema fs = new LogicalFieldSchema("tuple", schema, DataType.TUPLE);
        ResourceFieldSchema rfs = new ResourceFieldSchema(fs);

        LoadCaster caster = new Utf8StorageConverter();
        List<Tuple> actualResList = new ArrayList<Tuple>();
        while(actualResultsIt.hasNext()){
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
            new MultiMap<Integer, LogicalExpressionPlan>();
        LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});

        LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
        ProjectExpression x = new ProjectExpression(aprojplan, 0, 0, C);
        x.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
                DataType.INTEGER, 1));
        LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
        ProjectExpression y = new ProjectExpression(bprojplan, 1, 0, C);
        y.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
                DataType.INTEGER, 2));
        mm.put(0, aprojplan);
        mm.put(1, bprojplan);

        C.neverUseForRealSetSchema(cschema);
        // Don't add it to the plan quite yet

        // D = filter
        LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
        LOFilter D = new LOFilter(lp, filterPlan);
        ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
        fy.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
                DataType.INTEGER, 2));
        ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
        new EqualExpression(filterPlan, fy, fc);

        D.neverUseForRealSetSchema(cschema);
View Full Code Here
395
396
397
398
399
400
401
402
403
404
405
                                    && userStarSch.getField(schIdx) != null){

                                //if the project-star field has user specified schema, use the
                                // j'th field for this column
                                LogicalSchema sch = new LogicalSchema();
                                sch.addField(new LogicalFieldSchema(userStarSch.getField(schIdx)));
                                newUserSchema.add(sch);
                            }
                            else{
                                newUserSchema.add(null);
                            }
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
            // DA = filter
            LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
            LOFilter DA = new LOFilter(expected, DAfilterPlan);
            ProjectExpression fx = new ProjectExpression(DAfilterPlan, 0, 0, DA);
            fx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, new Integer(0));
            new EqualExpression(DAfilterPlan, fx, fc0);

            DA.neverUseForRealSetSchema(aschema);
            expected.add(DA);
            expected.connect(A, DA);

            // B = load
            LogicalSchema bschema = new LogicalSchema();
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                "a", null, DataType.BYTEARRAY));
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                "b", null, DataType.BYTEARRAY));
            LOLoad B = newLOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, expected, conf);
            expected.add(B);

            // DB = filter
            LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
            LOFilter DB = new LOFilter(expected, DBfilterPlan);
            ProjectExpression fb = new ProjectExpression(DBfilterPlan, 0, 1, DB);
            fb.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, new Integer(1));
            new EqualExpression(DBfilterPlan, fb, fc1);

            DB.neverUseForRealSetSchema(bschema);
            expected.add(DB);
            expected.connect(B, DB);

            // C = join
            LogicalSchema cschema = new LogicalSchema();
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "D::x", null, DataType.BYTEARRAY));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "D::y", null, DataType.BYTEARRAY));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "null::a", null, DataType.BYTEARRAY));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "null::b", null, DataType.BYTEARRAY));
            cschema.getField(0).uid = 1;
            cschema.getField(1).uid = 2;
            cschema.getField(2).uid = 3;
            cschema.getField(3).uid = 4;
            LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
            MultiMap<Integer, LogicalExpressionPlan> mm =
                new MultiMap<Integer, LogicalExpressionPlan>();
            LOJoin C = new LOJoin(expected, mm, JOINTYPE.HASH, new boolean[] {true, true});

            ProjectExpression x = new ProjectExpression(aprojplan, 0, 0, C);
            x.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
            ProjectExpression y = new ProjectExpression(bprojplan, 1, 0, C);
            y.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            mm.put(0, aprojplan);
            mm.put(1, bprojplan);
            C.neverUseForRealSetSchema(cschema);
            expected.add(C);
            expected.connect(DA, C);
            expected.connect(DB, C);

            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            LOFilter D = new LOFilter(expected, filterPlan);
            ProjectExpression fanotherx = new ProjectExpression(filterPlan, 0, 0, D);
            fanotherx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            ProjectExpression fa = new ProjectExpression(filterPlan, 0, 2, D);
            fa.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
            ProjectExpression fanotherb = new ProjectExpression(filterPlan, 0, 3, D);
            fanotherb.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
            fy.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
            EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
            new AndExpression(filterPlan, eq2, eq4);

            D.neverUseForRealSetSchema(cschema);
            expected.add(D);
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        assertEquals(1, schema.size());
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        assertEquals(DataType.BAG, bagFieldSchema.type);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        assertEquals(1, tupleFieldSchema.schema.size());
        assertEquals(DataType.BYTEARRAY, tupleFieldSchema.schema.getField(0).type);
    }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        assertEquals(1, schema.size());
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        assertEquals(DataType.BAG, bagFieldSchema.type);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        assertNull(tupleFieldSchema.schema);
    }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOSort loSort = (LOSort)lp.getPredecessors(loStore).get(0);
        Operator sortPlanLeaf = loSort.getSortColPlans().get(0).getSources().get(0);
        LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
        assertNull(sortPlanFS);

        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        POSort poSort = (POSort)pp.getPredecessors(poStore).get(0);
View Full Code Here
161
162
163
164
165
166
167
168
169
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOCogroup loCoGroup = (LOCogroup)lp.getPredecessors(loStore).get(0);
        LogicalFieldSchema groupFieldSchema = loCoGroup.getSchema().getField(0);
        assertEquals(DataType.TUPLE, groupFieldSchema.type);
        assertNull(groupFieldSchema.schema);
    }
View Full Code Here
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
            ProjectExpression project = new ProjectExpression(genPlan, i, 0, loGen);
            genPlan.add(project);

            // add casting if necessary by comparing target types
            // to the input schema
            LogicalFieldSchema fs = null ;
            fs = fromSchema.getField(i) ;

            // This only does "shallow checking"

            LogicalFieldSchema outFieldSchema ;

            outFieldSchema = toSchema.getField(i) ;

            if (outFieldSchema.type != fs.type) {
                castNeededCounter++ ;
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

  • org.apache.pig.newplan.logical.expression.BinCondExpression
  • org.apache.pig.newplan.logical.expression.DereferenceExpression
  • org.apache.pig.newplan.logical.expression.MapLookupExpression
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.expression.ScalarExpression
  • org.apache.pig.newplan.logical.expression.UserFuncExpression
  • org.apache.pig.newplan.logical.relational.LOCogroup
  • org.apache.pig.newplan.logical.relational.LOGenerate
  • org.apache.pig.newplan.logical.relational.LOInnerLoad
  • org.apache.pig.newplan.logical.relational.LOUnion
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.