Package org.apache.pig.backend.hadoop.executionengine

Examples of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

177
178
179
180
181
182
183
184
185
186
187
        switch (execType) {
            case LOCAL:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            HExecutionEngine eng = pigContext.getExecutionEngine();
            if(pigContext.getExecType() != ExecType.LOCAL){
                try {
                    if(val<=0)
                        val = pigContext.defaultParallel;
                    if (val<=0)
                        val = eng.getJobConf().getNumReduceTasks();
                    if (val<=0)
                        val = 1;
                } catch (Exception e) {
                    int errCode = 6015;
                    String msg = "Problem getting the default number of reduces from the Job Client.";
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
163
        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);

        ScriptState.get().addWorkflowAdjacenciesToConf(mrp, conf);
       
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
    }

    /* Create a Mock PigContext object with methods stubbed out to
        let the BasicP2LClient.createPlan() method complete */
    public void addMockPigContext(BasicP2LClient client) {
        HExecutionEngine exec_engine = mock(HExecutionEngine.class);
        when(exec_engine.getLogToPhyMap()).thenReturn(new HashMap<Operator, PhysicalOperator>());

        PigContext ctx = mock(PigContext.class);
        when(ctx.getExecutionEngine()).thenReturn(exec_engine);
        when(ctx.getProperties()).thenReturn(new Properties());
        client.setPigContext(ctx);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    private P2jPlanGenerator getOpPlanGenerator(LipstickPigServer lps) throws Exception {
        return new P2jPlanGenerator(lps.getLP(null));
    }

    private Map<PhysicalOperator, Operator> getP2lMap(LipstickPigServer lps) throws Exception {
        HExecutionEngine he = new HExecutionEngine(lps.getPigContext());
        he.compile(getLogicalPlan(lps), null);

        Map<Operator, PhysicalOperator> l2pMap = he.getLogToPhyMap();
        Map<PhysicalOperator, Operator> p2lMap = Maps.newHashMap();
        for (Entry<Operator, PhysicalOperator> i : l2pMap.entrySet()) {
            p2lMap.put(i.getValue(), i.getKey());
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

  • com.netflix.lipstick.MRPlanCalculatorTest
  • com.netflix.lipstick.pigtolipstick.BasicP2LClientTest
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.hdfs.DistributedFileSystem
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.JobConf
  • org.apache.pig.backend.executionengine.ExecLogicalPlan
  • org.apache.pig.backend.hadoop.datastorage.HDataStorage
  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.