Examples of visit()


Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer.visit()

        LogicalPlan lp = planTester.buildPlan("store D into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();

        assertTrue(so.getNumMRUseSecondaryKey() == 1);
        assertTrue(so.getNumSortRemoved() == 0);
        assertTrue(so.getDistinctChanged() == 1);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizerMR.visit()

            throws Exception, VisitorException {
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizerMR so = new SecondaryKeyOptimizerMR(mrPlan);
        so.visit();
        return so;
    }

}
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.UDFFinishVisitor.visit()

        failedJobs += runPipeline(stores.toArray(new POStore[0]), pc);
       
        stats.accumulateStats();

        UDFFinishVisitor finisher = new UDFFinishVisitor(php, new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(php));
        finisher.visit();

        for (FileSpec spec: failedStores) {
            log.info("Failed to produce result in: \""+spec.getFileName()+"\"");
        }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.EndOfAllInputSetter.visit()

     
        // check whether stream operator is present
        // after MultiQueryOptimizer because it can shift streams from
        // map to reduce, etc.
        EndOfAllInputSetter checker = new EndOfAllInputSetter(plan);
        checker.visit();
       
        boolean isAccum =
            "true".equalsIgnoreCase(pc.getProperties().getProperty("opt.accumulator","true"));
        if (isAccum) {
            AccumulatorOptimizer accum = new AccumulatorOptimizer(plan);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRIntermediateDataVisitor.visit()

        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
       
        // Find all the intermediate data stores. The plan will be destroyed during compile/execution
        // so this needs to be done before.
        MRIntermediateDataVisitor intermediateVisitor = new MRIntermediateDataVisitor(mrp);
        intermediateVisitor.visit();
       
        List<Job> failedJobs = new LinkedList<Job>();
        List<NativeMapReduceOper> failedNativeMR = new LinkedList<NativeMapReduceOper>();
        List<Job> completeFailedJobsInThisRun = new LinkedList<Job>();
        List<Job> succJobs = new LinkedList<Job>();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRPrinter.visit()

        MROperPlan mrp = compile(php, pc);

        if (format.equals("text")) {
            MRPrinter printer = new MRPrinter(ps, mrp);
            printer.setVerbose(verbose);
            printer.visit();
        } else {
            ps.println("#--------------------------------------------------");
            ps.println("# Map Reduce Plan                                  ");
            ps.println("#--------------------------------------------------");
           
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRStreamHandler.visit()

        POPackageAnnotator pkgAnnotator = new POPackageAnnotator(plan);
        pkgAnnotator.visit();
       
        // check whether stream operator is present
        MRStreamHandler checker = new MRStreamHandler(plan);
        checker.visit();
       
        // optimize joins
        LastInputStreamingOptimizer liso =
            new MRCompiler.LastInputStreamingOptimizer(plan, lastInputChunkSize);
        liso.visit();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator.visit()

            skOptimizer.visit();
        }
       
        // optimize key - value handling in package
        POPackageAnnotator pkgAnnotator = new POPackageAnnotator(plan);
        pkgAnnotator.visit();
       
        // optimize joins
        LastInputStreamingOptimizer liso =
            new MRCompiler.LastInputStreamingOptimizer(plan, lastInputChunkSize);
        liso.visit();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.ScalarPhyFinder.visit()

    private void processUDFs(PhysicalPlan plan) throws VisitorException{
        if(plan!=null){
            //Process Scalars (UDF with referencedOperators)
            ScalarPhyFinder scalarPhyFinder = new ScalarPhyFinder(plan);
            scalarPhyFinder.visit();
            curMROp.scalars.addAll(scalarPhyFinder.getScalars());
           
            //Process UDFs
            udfFinder.setPlan(plan);
            udfFinder.visit();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.XMLMRPrinter.visit()

            printer.setVerbose(verbose);
            printer.visit();
        } else if (format.equals("xml")) {
            try {
                XMLMRPrinter printer = new XMLMRPrinter(ps, mrp);
                printer.visit();
                printer.closePlan();
            } catch (ParserConfigurationException e) {
                e.printStackTrace();
            } catch (TransformerException e) {
                e.printStackTrace();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.