Package org.apache.pig.newplan.logical.relational.LogicalSchema

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
                            projExpr.setAttachedRelationalOp( op );
                        }
                    }
                }
                LogicalExpression root = (LogicalExpression)planCopy.getSources().get( 0 );// get the root of the plan
                LogicalFieldSchema schema;
                try {
                    schema = root.getFieldSchema();
                    if (schema.alias == null) {
                        schema.alias = colAlias;
                    }
View Full Code Here
671
672
673
674
675
676
677
678
679
680
681
                        //see if there is a load function associated with the store
                        FuncSpec funcSpec = rel2InputFuncMap.get(inputStore);
                        addUidLoadFuncToMap(outputUid, funcSpec);
                    }else{
                        //find input uid and corresponding load func
                        LogicalFieldSchema fs = sch.getField(inputColNum);
                        FuncSpec funcSpec = uid2LoadFuncMap.get(fs.uid);
                        addUidLoadFuncToMap(outputUid, funcSpec);
                    }
                }
            }
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
         
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, 0, 0, DA);
          fx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, new Integer(0));
          new EqualExpression(DAfilterPlan, fx, fc0);
         
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
         
          // A = foreach
            LOForEach foreachA = org.apache.pig.newplan.logical.Util.addForEachAfter(expected, DA, 0, new HashSet<Integer>());
            foreachA.setAlias("A");
            foreachA.neverUseForRealSetSchema(aschema);
         
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          LOLoad B = newLOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, expected, conf);
          expected.add(B);
         
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
            LOFilter DB = new LOFilter(expected, DBfilterPlan);
          ProjectExpression fb = new ProjectExpression(DBfilterPlan, 0, 1, DB);
          fb.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, new Integer(1));
          new EqualExpression(DBfilterPlan, fb, fc1);

          DB.neverUseForRealSetSchema(bschema);
          expected.add(DB);
          expected.connect(B, DB);
         
            // B = foreach
            LOForEach foreachB = org.apache.pig.newplan.logical.Util.addForEachAfter(expected, DB, 0, new HashSet<Integer>());
            foreachB.setAlias("B");
            foreachB.neverUseForRealSetSchema(bschema);
           
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "D::x", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "D::y", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "null::a", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "null::b", null, DataType.BYTEARRAY));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
            MultiMap<Integer, LogicalExpressionPlan> mm =
                new MultiMap<Integer, LogicalExpressionPlan>();
            LOJoin C = new LOJoin(expected, mm, JOINTYPE.HASH, new boolean[] {true, true});

          ProjectExpression x = new ProjectExpression(aprojplan, 0, 0, C);
          x.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, 1, 0, C);
          y.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          C.neverUseForRealSetSchema(cschema);
          expected.add(C);
          expected.connect(foreachA, C);
          expected.connect(foreachB, C);
         
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            LOFilter D = new LOFilter(expected, filterPlan);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, 0, 0, D);
          fanotherx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ProjectExpression fa = new ProjectExpression(filterPlan, 0, 2, D);
          fa.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, 0, 3, D);
          fanotherb.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
          fy.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, eq2, eq4);
         
          D.neverUseForRealSetSchema(cschema);
          expected.add(D);
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
                boolean needNewUid = innerLoadsPair.second;

                // pull tuple information from innerload
                if (innerLoads.get(0).getProjection().getFieldSchema().schema!=null &&
                        innerLoads.get(0).getProjection().getFieldSchema().type==DataType.BAG) {
                    LogicalFieldSchema originalTupleFieldSchema = innerLoads.get(0).getProjection().getFieldSchema().schema.getField(0);
                    LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(originalTupleFieldSchema.alias,
                            schema, DataType.TUPLE);
                    if (needNewUid) {
                        newTupleFieldSchema.uid = LogicalExpression.getNextUid();
                    }
                    else {
                        newTupleFieldSchema.uid = originalTupleFieldSchema.uid;
                    }
                    LogicalSchema newTupleSchema = new LogicalSchema();
                    newTupleSchema.addField(newTupleFieldSchema);
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, newTupleSchema, DataType.BAG);
                }
                else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, schema, DataType.BAG);
                }
                if (needNewUid)
                    fieldSchema.uid = LogicalExpression.getNextUid();
                else
                    fieldSchema.uid = innerLoads.get(0).getProjection().getFieldSchema().uid;
            }
            else {
                // InnerLoad and source is not bag
                if(schema == null){
                    // if we get here, it is range or starProject, otherwise, innerLoad will convert schema to non-null
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    } else {
                        fieldSchema = null;
                    }
                }
                else{
                    fieldSchema = schema.getField(0);
                }
            }
            if (fieldSchema!=null)
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
        }
        else {
            if (schema == null) {
                if(isRangeOrStarProject()) {
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    }
                    else {
                        fieldSchema = null;
                    }
                } else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                }

                if (fieldSchema!=null)
                    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            }
            else {
                int index = -1;
                if (!isRangeOrStarProject() && uidOnlyFieldSchema!=null) {
                    long uid = uidOnlyFieldSchema.uid;
                    for (int i=0;i<schema.size();i++) {
                        LogicalFieldSchema fs = schema.getField(i);
                        if (fs.uid==uid) {
                            index = i;
                        }
                    }
                }
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
            if(schema.size() == 0)
                return null;
            funcSpec = uid2LoadFuncMap.get(schema.getField(0).uid);
            if(funcSpec != null) {
                for(int i=1; i<schema.size(); i++){
                    LogicalFieldSchema fs = schema.getField(i);
                    if(! funcSpec.equals(uid2LoadFuncMap.get(fs.uid))){
                        //all uid are not associated with same func spec, there is no
                        // single func spec that represents all the fields
                        funcSpec = null;
                        break;
View Full Code Here
456
457
458
459
460
461
462
463
464
465
466
467
                + split;
            throw new VisitorException(split, msg,2261, PigException.BUG) ;
        }

        for(int i=0; i<inputSch.size(); i++){
            LogicalFieldSchema inField = inputSch.getField(i);
            LogicalFieldSchema outField = outSchema.getField(i);
            if(uid2LoadFuncMap.get(inField.uid) != null){
                addUidLoadFuncToMap(outField.uid, uid2LoadFuncMap.get(inField.uid));
            }
        }
       
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
                return;
        }
       
        //if same non null load func is associated with all fieldschemas
        // asssociate that with the uid of outFS
        LogicalFieldSchema inpFS1 = inputFieldSchemas.get(0);
        FuncSpec funcSpec1 = uid2LoadFuncMap.get(inpFS1.uid);
        boolean allInnerSchemaMatch = false;
        if(funcSpec1 != null){
            boolean allMatch = true;
            allInnerSchemaMatch = true;
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
    throws FrontendException {
        String msg = binOp.toString();
        if (currentRelOp.getAlias()!=null){
            msg = "In alias " + currentRelOp.getAlias() + ", ";
        }
        LogicalFieldSchema lhsFs = binOp.getLhs().getFieldSchema();
        LogicalFieldSchema rhsFs = binOp.getRhs().getFieldSchema();
       
        msg = msg + "incompatible types in " + binOp.getName() + " Operator"
        + " left hand side:" + DataType.findTypeName(lhsFs.type)
        + (lhsFs.schema == null ? "" : " " + lhsFs.schema.toString(false) + " ")
        + " right hand side:" + DataType.findTypeName(rhsFs.type)
View Full Code Here
436
437
438
439
440
441
442
443
     * @param arg
     * @throws FrontendException
     */
    private void insertCast(LogicalExpression exp, byte toType, LogicalExpression arg)
    throws FrontendException {
        LogicalFieldSchema toFs = new LogicalSchema.LogicalFieldSchema(null, null, toType);
        insertCast(exp, toFs, arg);
    }
View Full Code Here
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
            String msg = "Cannot cast to bytearray";
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(cast, msg, errCode, PigException.INPUT) ;
        }
       
        LogicalFieldSchema inFs = cast.getExpression().getFieldSchema();
        LogicalFieldSchema outFs = cast.getFieldSchema();
       
        if(inFs == null){
            //replace null schema with bytearray schema.
            inFs = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
        }
       
        //check if the field schemas are castable
        boolean castable = LogicalFieldSchema.castable(inFs, outFs);
        if(!castable) {
            int errCode = 1052;
            String msg = "Cannot cast "
                           + DataType.findTypeName(inType)
                           + ((DataType.isSchemaType(inType))? " with schema " + inFs.toString(false) : "")
                           + " to "
                           + DataType.findTypeName(outType)
                           + ((DataType.isSchemaType(outType))? " with schema " + outFs.toString(false) : "");
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(cast, msg, errCode, PigException.INPUT) ;
        }
      
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

  • org.apache.pig.newplan.logical.expression.BinCondExpression
  • org.apache.pig.newplan.logical.expression.DereferenceExpression
  • org.apache.pig.newplan.logical.expression.MapLookupExpression
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.expression.ScalarExpression
  • org.apache.pig.newplan.logical.expression.UserFuncExpression
  • org.apache.pig.newplan.logical.relational.LOCogroup
  • org.apache.pig.newplan.logical.relational.LOGenerate
  • org.apache.pig.newplan.logical.relational.LOInnerLoad
  • org.apache.pig.newplan.logical.relational.LOUnion
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.