Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOLoad

923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            LOLoad load2 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1a", DataType.INTEGER));
                fsList1.add(new FieldSchema("field2a", DataType.LONG));
                fsList1.add(new FieldSchema(null, DataType.BYTEARRAY));
                fsList1.add(new FieldSchema(null, DataType.CHARARRAY));
                inputSchema1 = new Schema(fsList1);
            }

            // schema for input#2
            Schema inputSchema2 = null;
            {
                List<FieldSchema> fsList2 = new ArrayList<FieldSchema>();
                fsList2.add(new FieldSchema("field1b", DataType.DOUBLE));
                fsList2.add(new FieldSchema(null, DataType.INTEGER));
                fsList2.add(new FieldSchema("field3b", DataType.FLOAT));
                fsList2.add(new FieldSchema("field4b", DataType.CHARARRAY));
                inputSchema2 = new Schema(fsList2);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema2));

            // create union operator
            ArrayList<LogicalRelationalOperator> inputList = new ArrayList<LogicalRelationalOperator>();
            inputList.add(load1);
            inputList.add(load2);
View Full Code Here
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();


            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            LOLoad load2 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1a", DataType.INTEGER));
                fsList1.add(new FieldSchema("field2a", DataType.BYTEARRAY));
                inputSchema1 = new Schema(fsList1);
            }

            // schema for input#2
            Schema inputSchema2 = null;
            {
                List<FieldSchema> fsList2 = new ArrayList<FieldSchema>();
                fsList2.add(new FieldSchema("field1b", DataType.DOUBLE));
                fsList2.add(new FieldSchema("field2b", DataType.DOUBLE));
                inputSchema2 = new Schema(fsList2);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema2));

            // create union operator
            ArrayList<LogicalRelationalOperator> inputList = new ArrayList<LogicalRelationalOperator>();
            inputList.add(load1);
            inputList.add(load2);
View Full Code Here
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> innerList = new ArrayList<FieldSchema>();
                innerList.add(new FieldSchema("innerfield1", DataType.BAG));
                innerList.add(new FieldSchema("innerfield2", DataType.FLOAT));
                Schema innerSchema = new Schema(innerList);

                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.INTEGER));
                fsList1.add(new FieldSchema("field2", DataType.BYTEARRAY));
                fsList1.add(new FieldSchema("field3", innerSchema));
                fsList1.add(new FieldSchema("field4", DataType.BAG));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));

            // create union operator
            LODistinct distinct1 = new LODistinct(plan);

            // wiring
            plan.add(load1);
            plan.add(distinct1);

            plan.connect(load1, distinct1);

            // validate
            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);

            // check end result schema
            LogicalSchema outputSchema = distinct1.getSchema();
            assertTrue(load1.getSchema().isEqual(outputSchema));
        }
View Full Code Here
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );
            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", field1Type));
                fsList1.add(new FieldSchema("field2", field2Type));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));

            // Create inner plan
            LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();
            // filter
            LOFilter filter1 = new LOFilter(plan);
View Full Code Here
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.INTEGER));
                fsList1.add(new FieldSchema("field2", DataType.LONG));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));

            // Create inner plan
            LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();

            // filter
View Full Code Here
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );


            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.LONG));
                fsList1.add(new FieldSchema("field2", DataType.INTEGER));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;

            // Create project inner plan #1
            LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan();
            // Sort
            LOSort sort1 = new LOSort(plan);
View Full Code Here
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 =  newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
                fsList1.add(new FieldSchema("field2", DataType.INTEGER));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;

            LOSort sort1 = new LOSort(plan);


            // Create expression inner plan #1
View Full Code Here
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 =  newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );


            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
                fsList1.add(new FieldSchema("field2", DataType.INTEGER));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;


            // Sort
            LOSort sort1 = new LOSort(plan);
View Full Code Here
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 =  newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
                fsList1.add(new FieldSchema("field2", DataType.INTEGER));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;


            // split
            LOSplit split1 = new LOSplit(plan);
View Full Code Here
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 =  newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );

            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
                fsList1.add(new FieldSchema("field2", DataType.INTEGER));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;

            // split
            LOSplit split1 = new LOSplit(plan);

            // output1
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOLoad

  • org.apache.hadoop.mapreduce.Job
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer
  • org.apache.pig.newplan.logical.rules.LoadTypeCastInserter
  • org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer
  • org.apache.pig.newplan.logical.rules.PredicatePushdownOptimizer
  • org.apache.pig.newplan.logical.rules.TypeCastInserter
  • org.apache.pig.parser.LogicalPlanBuilder
  • org.apache.pig.ResourceSchema
  • org.apache.pig.test.TestBlackAndWhitelistValidator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.