Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.FileSpec

1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
       
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
       
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
       
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
       
       
        LogicalSchema jaschema7 = new LogicalSchema();
        jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A7 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema7, lp, null);
        lp.add(A7);
       
        // B = load
        LogicalSchema jbschema7 = new LogicalSchema();
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B7 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema7, lp, null);
        lp.add(B7);
       
        // C = join
        LogicalSchema jcschema7 = new LogicalSchema();
View Full Code Here
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
       
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
       
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
       
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
       
        // Test different different number of join keys
        LogicalSchema jaschema8 = new LogicalSchema();
        jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A8 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema8, lp, null);
        lp.add(A8);
       
        // B = load
        LogicalSchema jbschema8 = new LogicalSchema();
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B8 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema8, lp, null);
        lp.add(B8);
       
        // C = join
        LogicalSchema jcschema8 = new LogicalSchema();
View Full Code Here
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
    public void testRelationalSameOpDifferentPreds() throws FrontendException {
        LogicalPlan lp1 = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, null);
        lp1.add(A1);
       
        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
        ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0), new LogicalFieldSchema(null, null, DataType.INTEGER));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp1, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp1.add(D1);
        lp1.connect(A1, D1);
       
        LogicalPlan lp2 = new LogicalPlan();
        LOLoad A2 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, null);
        lp2.add(A2);
       
        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
        ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
View Full Code Here
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
                    int errCode = 2025;
                    String msg = "Expected leaf of reduce plan to " +
                        "always be POStore. Found " + mpLeaf.getClass().getSimpleName();
                    throw new MRCompilerException(msg, errCode, PigException.BUG);
                }
                FileSpec oldSpec = ((POStore)mpLeaf).getSFile();
                boolean oldIsTmpStore = ((POStore)mpLeaf).isTmpStore();
               
                FileSpec fSpec = getTempFileSpec();
                ((POStore)mpLeaf).setSFile(fSpec);
                ((POStore)mpLeaf).setIsTmpStore(true);
                mr.setReduceDone(true);
                MapReduceOper limitAdjustMROp = getMROp();
                POLoad ld = getLoad();
View Full Code Here
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
            this.oldSpec = oldSpec;
        }

        @Override
        public void visitStore(POStore sto) throws VisitorException {
            FileSpec spec = sto.getSFile();
            if (oldSpec.equals(spec)) {
                sto.setSFile(newSpec);
            }          
        }
View Full Code Here
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
                         + "_" + Long.toString(System.currentTimeMillis())
                         + "_" + i;
                     replicatedPath.add(replFiles[i].getFileName() + "#"
                             + symlink);
                 }
                 newReplFiles[i] = new FileSpec(symlink,
                         (replFiles[i] == null ? null : replFiles[i].getFuncSpec()));              
             }
            
             join.setReplFiles(newReplFiles);
            
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
284
285
286
       
    private LogicalPlan genNewLoadStorePlan(String inputFile,
                                            String outputFile, DataStorage dfs)
                                        throws Throwable {
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
       
        plan.add(load) ;
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
       
        for(MapReduceOper mrOp: mrOpList) {
            for(PhysicalOperator scalar: mrOp.scalars) {               
                MapReduceOper mro = phyToMROpMap.get(scalar);
                if (scalar instanceof POStore) {                                    
                    FileSpec oldSpec = ((POStore)scalar).getSFile();
                    MapReduceOper mro2 = seen.get(oldSpec);
                    boolean hasSeen = false;
                    if (mro2 != null) {
                        hasSeen = true;
                        mro = mro2;
                    }
                    if (!hasSeen
                            && combinable
                            && (mro.reducePlan.isEmpty() ? hasTooManyInputFiles(mro, conf)
                                    : (mro.requestedParallelism >= fileConcatenationThreshold))) {
                        PhysicalPlan pl = mro.reducePlan.isEmpty() ? mro.mapPlan : mro.reducePlan;
                        FileSpec newSpec = getTempFileSpec();
                       
                        // replace oldSpec in mro with newSpec
                        new FindStoreNameVisitor(pl, newSpec, oldSpec).visit();
                       
                        POStore newSto = getStore();
View Full Code Here
588
589
590
591
592
593
594
595
596
597
598
            if (!mro.isMapDone()) {
                mro.setMapDoneSingle(true);
                curMROp = mro;
            }
            else if(mro.isMapDone() && !mro.isReduceDone()){
                FileSpec fSpec = getTempFileSpec();
               
                POStore st = getStore();
                st.setSFile(fSpec);
                mro.reducePlan.addAsLeaf(st);
                mro.setReduceDone(true);
View Full Code Here
630
631
632
633
634
635
636
637
638
639
640
        if(leaves!=null && leaves.size()>0)
            leaf = leaves.get(0);

        for (MapReduceOper mmro : mergedPlans) {
            mmro.setReduceDone(true);
            FileSpec fileSpec = getTempFileSpec();
            POLoad ld = getLoad();
            ld.setLFile(fileSpec);
            POStore str = getStore();
            str.setSFile(fileSpec);
            mmro.reducePlan.addAsLeaf(str);
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.FileSpec

  • com.netflix.lipstick.MRPlanCalculator
  • org.apache.pig.backend.executionengine.util.ExecTools
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler$JoinDistributedCacheVisitor
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$FindStoreNameVisitor
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NoopStoreRemover$PhysicalRemover
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.