Examples of visit()

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOUserFuncVisitor.visit()

39
40
41
42
43
44
45
46
47
48
49
        assertPlanContains(tezPlanContainer.getRoots().get(0).getTezOperPlan(), expectedFiles, size);
    }

    private void assertPlanContains(TezOperPlan plan, String[] expectedFiles, int size) throws VisitorException {
        TezPOUserFuncVisitor udfVisitor = new TezPOUserFuncVisitor(plan);
        udfVisitor.visit();

        List<String> shipFiles = new ArrayList<String>();
        shipFiles.addAll(udfVisitor.getShipFiles());

        Assert.assertEquals(shipFiles.size(), size);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPlanContainerPrinter.visit()

315
316
317
318
319
320
321
322
323
324
325
        TezPlanContainer tezPlanContainer = compile(php, pc);

        if (format.equals("text")) {
            TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, tezPlanContainer);
            printer.setVerbose(verbose);
            printer.visit();
        } else {
            // TODO: add support for other file format
            throw new IOException("Non-text output of explain is not supported.");
        }
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezGraphPrinter.visit()

47
48
49
50
51
52
53
54
55
56
57
    public void visitTezPlanContainerNode(TezPlanContainerNode tezPlanContainerNode) throws VisitorException {
        mStream.println("#--------------------------------------------------");
        mStream.println("# TEZ DAG plan: " + tezPlanContainerNode.getOperatorKey());
        mStream.println("#--------------------------------------------------");
        TezGraphPrinter graphPrinter = new TezGraphPrinter(tezPlanContainerNode.getTezOperPlan());
        graphPrinter.visit();
        mStream.print(graphPrinter.toString());
        TezPrinter printer = new TezPrinter(mStream, tezPlanContainerNode.getTezOperPlan());
        printer.setVerbose(isVerbose);
        printer.visit();
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.AccumulatorOptimizer.visit()

372
373
374
375
376
377
378
379
380
381
382
        // Run AccumulatorOptimizer on Tez plan
        boolean isAccum = conf.getBoolean(PigConfiguration.OPT_ACCUMULATOR, true);
        if (isAccum) {
            AccumulatorOptimizer accum = new AccumulatorOptimizer(tezPlan);
            accum.visit();
        }

        // Use VertexGroup in Tez
        boolean isUnionOpt = conf.getBoolean(PigConfiguration.TEZ_OPT_UNION, true);
        if (isUnionOpt) {
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.CombinerOptimizer.visit()

348
349
350
351
352
353
354
355
356
357
358
        boolean nocombiner = conf.getBoolean(PigConfiguration.PROP_NO_COMBINER, false);
        if (!pc.inIllustrator && !nocombiner)  {
            boolean doMapAgg = Boolean.parseBoolean(pc.getProperties().getProperty(
                    PigConfiguration.PROP_EXEC_MAP_PARTAGG, "false"));
            CombinerOptimizer co = new CombinerOptimizer(tezPlan, doMapAgg);
            co.visit();
            co.getMessageCollector().logMessages(MessageType.Warning, aggregateWarning, log);
        }

        // Run optimizer to make use of secondary sort key when possible for nested foreach
        // order by and distinct. Should be done before AccumulatorOptimizer
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.LoaderProcessor.visit()

387
388
389
390
391
392
393
394
395
396
397
    }

    public static void processLoadAndParallelism(TezOperPlan tezPlan, PigContext pc) throws VisitorException {
        if (!pc.inExplain && !pc.inDumpSchema) {
            LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
            loaderStorer.visit();

            ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
            parallelismSetter.visit();
            tezPlan.setEstimatedParallelism(parallelismSetter.getEstimatedTotalParallelism());
        }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.MultiQueryOptimizerTez.visit()

365
366
367
368
369
370
371
372
373
374
375
        boolean isMultiQuery = conf.getBoolean(PigConfiguration.OPT_MULTIQUERY, true);
        if (isMultiQuery) {
            // reduces the number of TezOpers in the Tez plan generated
            // by multi-query (multi-store) script.
            MultiQueryOptimizerTez mqOptimizer = new MultiQueryOptimizerTez(tezPlan);
            mqOptimizer.visit();
        }

        // Run AccumulatorOptimizer on Tez plan
        boolean isAccum = conf.getBoolean(PigConfiguration.OPT_ACCUMULATOR, true);
        if (isAccum) {
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.NoopFilterRemover.visit()

340
341
342
343
344
345
346
347
348
349
350
    private void optimize(TezOperPlan tezPlan, PigContext pc) throws VisitorException {
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
        boolean aggregateWarning = conf.getBoolean("aggregate.warning", false);

        NoopFilterRemover filter = new NoopFilterRemover(tezPlan);
        filter.visit();

        // Run CombinerOptimizer on Tez plan
        boolean nocombiner = conf.getBoolean(PigConfiguration.PROP_NO_COMBINER, false);
        if (!pc.inIllustrator && !nocombiner)  {
            boolean doMapAgg = Boolean.parseBoolean(pc.getProperties().getProperty(
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.ParallelismSetter.visit()

390
391
392
393
394
395
396
397
398
399
400
        if (!pc.inExplain && !pc.inDumpSchema) {
            LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
            loaderStorer.visit();

            ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
            parallelismSetter.visit();
            tezPlan.setEstimatedParallelism(parallelismSetter.getEstimatedTotalParallelism());
        }
    }

    @Override
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.SecondaryKeyOptimizerTez.visit()

357
358
359
360
361
362
363
364
365
366
367
        // Run optimizer to make use of secondary sort key when possible for nested foreach
        // order by and distinct. Should be done before AccumulatorOptimizer
        boolean noSecKeySort = conf.getBoolean(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, false);
        if (!pc.inIllustrator && !noSecKeySort)  {
            SecondaryKeyOptimizerTez skOptimizer = new SecondaryKeyOptimizerTez(tezPlan);
            skOptimizer.visit();
        }

        boolean isMultiQuery = conf.getBoolean(PigConfiguration.OPT_MULTIQUERY, true);
        if (isMultiQuery) {
            // reduces the number of TezOpers in the Tez plan generated
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.