Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.UdfCacheShipFilesVisitor

574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
                    log.info(BIG_JOB_LOG_MSG);
                    // Search to see if we have any UDF/LoadFunc/StoreFunc that need to pack things into the
                    // distributed cache.
                    List<String> cacheFiles = new ArrayList<String>();
                    List<String> shipFiles = new ArrayList<String>();
                    UdfCacheShipFilesVisitor mapUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.mapPlan);
                    mapUdfCacheFileVisitor.visit();
                    cacheFiles.addAll(mapUdfCacheFileVisitor.getCacheFiles());
                    shipFiles.addAll(mapUdfCacheFileVisitor.getShipFiles());

                    UdfCacheShipFilesVisitor reduceUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.reducePlan);
                    reduceUdfCacheFileVisitor.visit();
                    cacheFiles.addAll(reduceUdfCacheFileVisitor.getCacheFiles());
                    shipFiles.addAll(reduceUdfCacheFileVisitor.getShipFiles());

                    setupDistributedCache(pigContext, conf, cacheFiles.toArray(new String[]{}), false);

                    // Setup the DistributedCache for this job
                    List<URL> allJars = new ArrayList<URL>();
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    }
   
    private void assertPlanContains(MapReduceOper mro, String[] expectedFiles, int size) throws VisitorException {
        List<String> cacheFiles = new ArrayList<String>();
        List<String> shipFiles = new ArrayList<String>();
        UdfCacheShipFilesVisitor mapUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.mapPlan);
        mapUdfCacheFileVisitor.visit();
        cacheFiles.addAll(mapUdfCacheFileVisitor.getCacheFiles());
        shipFiles.addAll(mapUdfCacheFileVisitor.getShipFiles());

        UdfCacheShipFilesVisitor reduceUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.reducePlan);
        reduceUdfCacheFileVisitor.visit();
        cacheFiles.addAll(reduceUdfCacheFileVisitor.getCacheFiles());
        shipFiles.addAll(reduceUdfCacheFileVisitor.getShipFiles());

        Assert.assertEquals(shipFiles.size(), size);
        assertContains(shipFiles, expectedFiles);
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
    }

    @Override
    public void visitTezOp(TezOperator tezOp) throws VisitorException {
        if(!tezOp.plan.isEmpty()) {
            UdfCacheShipFilesVisitor udfCacheFileVisitor = new UdfCacheShipFilesVisitor(tezOp.plan);
            udfCacheFileVisitor.visit();
            cacheFiles.addAll(udfCacheFileVisitor.getCacheFiles());
            shipFiles.addAll(udfCacheFileVisitor.getShipFiles());
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.UdfCacheShipFilesVisitor

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOUserFuncVisitor
  • org.apache.pig.LoadFunc
  • org.apache.pig.test.TestLoaderStorerShipCacheFilesMR
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.