Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOLoad

1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
       
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
       
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
       
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
       
       
        LogicalSchema jaschema7 = new LogicalSchema();
        jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A7 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema7, lp, null);
        lp.add(A7);
       
        // B = load
        LogicalSchema jbschema7 = new LogicalSchema();
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B7 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema7, lp, null);
        lp.add(B7);
       
        // C = join
        LogicalSchema jcschema7 = new LogicalSchema();
View Full Code Here
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
       
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
       
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
       
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
       
        // Test different different number of join keys
        LogicalSchema jaschema8 = new LogicalSchema();
        jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A8 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema8, lp, null);
        lp.add(A8);
       
        // B = load
        LogicalSchema jbschema8 = new LogicalSchema();
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B8 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema8, lp, null);
        lp.add(B8);
       
        // C = join
        LogicalSchema jcschema8 = new LogicalSchema();
View Full Code Here
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
    public void testRelationalSameOpDifferentPreds() throws FrontendException {
        LogicalPlan lp1 = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, null);
        lp1.add(A1);
       
        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
        ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0), new LogicalFieldSchema(null, null, DataType.INTEGER));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp1, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp1.add(D1);
        lp1.connect(A1, D1);
       
        LogicalPlan lp2 = new LogicalPlan();
        LOLoad A2 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, null);
        lp2.add(A2);
       
        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
        ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        schema.addField(new LogicalSchema.LogicalFieldSchema("age", null, DataType.INTEGER));   
        schema.getField(0).uid = 1;
        schema.getField(1).uid = 2;
        schema.getField(2).uid = 3;
        LogicalRelationalOperator l1 = new LOLoad(null, schema, plan, null);
        l1.setAlias("A");
        plan.add(l1);

        schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("dept", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("salary", null, DataType.FLOAT));   
        schema.getField(0).uid = 4;
        schema.getField(1).uid = 5;
        schema.getField(2).uid = 6;
        LogicalRelationalOperator l2 = new LOLoad(null, schema, plan, null);
        l2.setAlias("B");
        plan.add(l2);
       
        MultiMap<Integer, LogicalExpressionPlan> joinPlans = new MultiMap<Integer, LogicalExpressionPlan>();
       
        LogicalRelationalOperator j1 = new LOJoin(plan, joinPlans, LOJoin.JOINTYPE.HASH, new boolean[]{true, true});
View Full Code Here
351
352
353
354
355
356
357
358
        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }
       
        FileSpec loader = new FileSpec( absolutePath, funcSpec );
        LOLoad op = new LOLoad( loader, schema, plan, ConfigurationUtil.toConfiguration( pigContext.getProperties() ) );
        return buildOp( loc, op, alias, new ArrayList<String>(), null );
    }
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
            for( String path : paths )
                pigContext.addJar( path );
            buildOp( loc, op, null, new ArrayList<String>(), null );
            ((LOStore)operators.get( storeAlias )).setTmpStore(true);
            plan.connect( operators.get( storeAlias ), op );
            LOLoad load = (LOLoad)operators.get( loadAlias );
            plan.connect( op, load );
            return load.getAlias();
        } catch (ParserException e) {
            throw new InvalidCommandException( input, loc, cmd );
        } catch (MalformedURLException e) {
            throw new InvalidPathException( input, loc, e);
        }
View Full Code Here
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }

        FileSpec loader = new FileSpec(absolutePath, funcSpec);
        LOLoad op = new LOLoad(
                loader,
                schema,
                plan,
                ConfigurationUtil.toConfiguration(pigContext.getProperties()),
                loFunc,
                alias + "_" + newOperatorKey());
        op.setTmpLoad(false);
        return buildOp( loc, op, alias, new ArrayList<String>(), null );
    }
View Full Code Here
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
            for( String path : paths )
                pigContext.addJar( path );
            buildOp( loc, op, null, new ArrayList<String>(), null );
            ((LOStore)operators.get( storeAlias )).setTmpStore(true);
            plan.connect( operators.get( storeAlias ), op );
            LOLoad load = (LOLoad)operators.get( loadAlias );
            plan.connect( op, load );
            return load.getAlias();
        } catch (ParserException e) {
            throw new InvalidCommandException( input, loc, cmd );
        } catch (MalformedURLException e) {
            throw new InvalidPathException( input, loc, e);
        }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        schema.addField(new LogicalSchema.LogicalFieldSchema("age", null, DataType.INTEGER));   
        schema.getField(0).uid = 1;
        schema.getField(1).uid = 2;
        schema.getField(2).uid = 3;
        LogicalRelationalOperator l1 = new LOLoad(schema, plan);
        l1.setAlias("A");
        plan.add(l1);

        schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("dept", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("salary", null, DataType.FLOAT));   
        schema.getField(0).uid = 4;
        schema.getField(1).uid = 5;
        schema.getField(2).uid = 6;
        LogicalRelationalOperator l2 = new LOLoad(schema, plan);
        l2.setAlias("B");
        plan.add(l2);
       
        MultiMap<Integer, LogicalExpressionPlan> joinPlans = new MultiMap<Integer, LogicalExpressionPlan>();
       
        LogicalRelationalOperator j1 = new LOJoin(plan, joinPlans, LOJoin.JOINTYPE.HASH, new boolean[]{true, true});
View Full Code Here
51
52
53
54
55
56
57
58
59
    }

    @Override
    protected OperatorPlan buildPattern() {
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator load = new LOLoad(null, plan);
        plan.add(load);
        return plan;
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOLoad

  • org.apache.hadoop.mapreduce.Job
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer
  • org.apache.pig.newplan.logical.rules.LoadTypeCastInserter
  • org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer
  • org.apache.pig.newplan.logical.rules.PredicatePushdownOptimizer
  • org.apache.pig.newplan.logical.rules.TypeCastInserter
  • org.apache.pig.parser.LogicalPlanBuilder
  • org.apache.pig.ResourceSchema
  • org.apache.pig.test.TestBlackAndWhitelistValidator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.