Package org.apache.pig.tools.pigstats.tez

Examples of org.apache.pig.tools.pigstats.tez.TezScriptState

590
591
592
593
594
595
596
597
598
599
600
601
        // set various parallelism into the job conf for later analysis, PIG-2779
        payloadConf.setInt(PigImplConstants.REDUCER_DEFAULT_PARALLELISM, pc.defaultParallel);
        payloadConf.setInt(PigImplConstants.REDUCER_REQUESTED_PARALLELISM, tezOp.getRequestedParallelism());
        payloadConf.setInt(PigImplConstants.REDUCER_ESTIMATED_PARALLELISM, tezOp.getEstimatedParallelism());

        TezScriptState ss = TezScriptState.get();
        ss.addVertexSettingsToConf(dag.getName(), tezOp, payloadConf);

        // Take our assembled configuration and create a vertex
        UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
        procDesc.setUserPayload(userPayload);
View Full Code Here
34
35
36
37
38
39
40
41
42
        this.launcher = new TezLauncher();
    }

    @Override
    public ScriptState instantiateScriptState() {
        TezScriptState ss = new TezScriptState(UUID.randomUUID().toString());
        ss.setPigContext(pigContext);
        return ss;
    }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
    private static SessionInfo createSession(Configuration conf,
            Map<String, LocalResource> requestedAMResources, Credentials creds,
            TezJobConfig tezJobConf) throws TezException, IOException,
            InterruptedException {
        TezConfiguration amConf = MRToTezHelper.getDAGAMConfFromMRConf(conf);
        TezScriptState ss = TezScriptState.get();
        ss.addDAGSettingsToConf(amConf);
        adjustAMConfig(amConf, tezJobConf);
        String jobName = conf.get(PigContext.JOB_NAME, "pig");
        TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
        tezClient.start();
        TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus();
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
        return comp.getTezPlan();
    }

    private DAG getTezDAG(TezOperPlan tezPlan, PigContext pc) {
        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);
        DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString());
        return tezDag;
    }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
        return comp.getTezPlan();
    }

    private DAG getTezDAG(TezOperPlan tezPlan, PigContext pc) {
        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);
        DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString());
        return tezDag;
    }
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
302
303
        TezCompiler comp = new TezCompiler(pp, pc);
        TezOperPlan tezPlan = comp.compile();
        TezLauncher.processLoadAndParallelism(tezPlan, pc);

        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);

        TezJobCompiler jobComp = new TezJobCompiler(pc, new Configuration());
        DAG dag = jobComp.buildDAG(tezPlanNode, new HashMap<String, LocalResource>());
        return new Pair<TezOperPlan, DAG>(tezPlan, dag);
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.tez.TezScriptState

  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.backend.hadoop.executionengine.tez.TezExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.tez.TezSessionManager
  • org.apache.pig.tez.TestGroupConstParallelTez
  • org.apache.pig.tez.TestJobSubmissionTez
  • org.apache.pig.tez.TestTezJobControlCompiler
  • org.apache.pig.tools.pigstats.JobStats
  • java.util.BitSet
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.