Package org.apache.pig.tools.pigstats.mapreduce

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

187
188
189
190
191
192
193
194
195
196
197
198
199
        if (mro == null) {
            LOG.warn("unable to get MR oper for job: " + job.toString());
            return null;
        }
        MRJobStats js = mroJobMap.get(mro);

        JobID jobId = job.getAssignedJobID();
        js.setId(jobId);
        js.setAlias(mro);
        js.setConf(job.getJobConf());
        return js;
    }
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
        js.setConf(job.getJobConf());
        return js;
    }

    MRJobStats addMRJobStatsForNative(NativeMapReduceOper mr) {
        MRJobStats js = mroJobMap.get(mr);
        js.setId(new JobID(mr.getJobId(), NativeMapReduceOper.getJobNumber()));
        js.setAlias(mr);

        return js;
    }
View Full Code Here
281
282
283
284
285
286
287
288
289
290
291
    void mapMROperToJob(MapReduceOper mro, Job job) {
        if (mro == null) {
            LOG.warn("null MR operator");
        } else {
            MRJobStats js = mroJobMap.get(mro);
            if (js == null) {
                LOG.warn("null job stats for mro: " + mro.getOperatorKey());
            } else {
                jobMroMap.put(job, mro);
            }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
            mroJobMap = new HashMap<MapReduceOper, MRJobStats>();
        }

        @Override
        public void visitMROp(MapReduceOper mr) throws VisitorException {
            MRJobStats js = new MRJobStats(
                    mr.getOperatorKey().toString(), jobPlan);
            jobPlan.add(js);
            List<MapReduceOper> preds = getPlan().getPredecessors(mr);
            if (preds != null) {
                for (MapReduceOper pred : preds) {
                    MRJobStats jpred = mroJobMap.get(pred);
                    if (!jobPlan.isConnected(jpred, js)) {
                        jobPlan.connect(jpred, js);
                    }
                }
            }
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
198
199
        if (mro == null) {
            LOG.warn("unable to get MR oper for job: " + job.toString());
            return null;
        }
        MRJobStats js = mroJobMap.get(mro);

        JobID jobId = job.getAssignedJobID();
        js.setId(jobId);
        js.setAlias(mro);
        js.setConf(job.getJobConf());
        return js;
    }
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
        js.setConf(job.getJobConf());
        return js;
    }

    MRJobStats addMRJobStatsForNative(NativeMapReduceOper mr) {
        MRJobStats js = mroJobMap.get(mr);
        js.setId(new JobID(mr.getJobId(), NativeMapReduceOper.getJobNumber()));
        js.setAlias(mr);

        return js;
    }
View Full Code Here
281
282
283
284
285
286
287
288
289
290
291
    void mapMROperToJob(MapReduceOper mro, Job job) {
        if (mro == null) {
            LOG.warn("null MR operator");
        } else {
            MRJobStats js = mroJobMap.get(mro);
            if (js == null) {
                LOG.warn("null job stats for mro: " + mro.getOperatorKey());
            } else {
                jobMroMap.put(job, mro);
            }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
            mroJobMap = new HashMap<MapReduceOper, MRJobStats>();
        }

        @Override
        public void visitMROp(MapReduceOper mr) throws VisitorException {
            MRJobStats js = new MRJobStats(
                    mr.getOperatorKey().toString(), jobPlan);
            jobPlan.add(js);
            List<MapReduceOper> preds = getPlan().getPredecessors(mr);
            if (preds != null) {
                for (MapReduceOper pred : preds) {
                    MRJobStats jpred = mroJobMap.get(pred);
                    if (!jobPlan.isConnected(jpred, js)) {
                        jobPlan.connect(jpred, js);
                    }
                }
            }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
            Util.checkQueryOutputsAfterSort(iter, expectedRes);
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(1, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSources().get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(1, js.getNumberReduces());
        }

    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    MRJobStats createJobStats(String name, JobGraph plan) {
        try {
            Constructor<MRJobStats> con = MRJobStats.class.getDeclaredConstructor(String.class, JobGraph.class);
            con.setAccessible(true);
            MRJobStats jobStats = (MRJobStats) con.newInstance(name, plan);
            return jobStats;
        } catch (Exception e) {
            return null;
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

  • org.apache.hadoop.mapred.Counters.Counter
  • org.apache.hadoop.mapred.TaskReport
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.impl.io.FileSpec
  • org.apache.pig.test.TestCounters
  • org.apache.pig.test.TestEmptyInputDir
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestGroupConstParallel
  • org.apache.pig.test.TestGroupConstParallelMR
  • org.apache.pig.test.TestMRJobStats
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.