Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
An operator model for a Map Reduce job. Acts as a host to the plans that will execute in map, reduce and optionally combine phases. These will be embedded in the MROperPlan in order to capture the dependencies amongst jobs.

    JobControlCompiler getJobControlCompiler() {
        return jcc;
    }


    MRJobStats addMRJobStats(Job job) {
        MapReduceOper mro = jobMroMap.get(job);


        if (mro == null) {
            LOG.warn("unable to get MR oper for job: " + job.toString());
            return null;
        }

View Full Code Here

        if(lrFound != pkg.getNumInps()) {
            // we did not find the LocalRearrange(s) in the map plan
            // let's look in the predecessors
            List<MapReduceOper> preds = this.mPlan.getPredecessors(mr);
            for (Iterator<MapReduceOper> it = preds.iterator(); it.hasNext();) {
                MapReduceOper mrOper = it.next();
                if (mrOper.isLimitOnly() && !mPlan.getPredecessors(mrOper).get(0).isGlobalSort())
                    mrOper = this.mPlan.getPredecessors(mrOper).get(0);
                lrFound += patchPackage(mrOper.reducePlan, pkg);
                if(lrFound == pkg.getNumInps()) {
                    break;
                }

View Full Code Here

     * @param jobMroMap the map that maps {@link Job}s to {@link MapReduceOper}s
     */
    public static void updateJobMroMap(Map<Job, MapReduceOper> jobMroMap) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        for (Map.Entry<Job, MapReduceOper> entry : jobMroMap.entrySet()) {
            MapReduceOper mro = entry.getValue();
            ps.mapMROperToJob(mro, entry.getKey());
        }
    }

View Full Code Here

    JobControlCompiler getJobControlCompiler() {
        return jcc;
    }


    MRJobStats addMRJobStats(Job job) {
        MapReduceOper mro = jobMroMap.get(job);


        if (mro == null) {
            LOG.warn("unable to get MR oper for job: " + job.toString());
            return null;
        }

View Full Code Here

      "c = limit b 10;" +
      "store c into 'output';";


      PhysicalPlan pp = Util.buildPp(pigServer, query);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;


      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here


      LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
        la.visit();
        la.adjust();


      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;


      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here

        "b = load 'input2';" +
        "c = join a by $0, b by $0;" + "store c into 'output';";


        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);


        assertEquals(2, mrOper.UDFs.size());
        assertEquals(2, mrOper.UDFs.size());
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));

View Full Code Here

        "d = order c by a0;" +
        "e = foreach d generate a1+a2;" +
        "store e into 'output';";
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here


        LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
        la.visit();
        la.adjust();


        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        int count = 1;


        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }
        assertEquals(4, count);


        MapReduceOper op = mrPlan.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here

            "c = cogroup a by $0, b by $0 using 'merge';" +
            "store c into 'output';";


        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);


        assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName()));
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName()));
        assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

com.twitter.ambrose.pig.AmbrosePigProgressNotificationListener

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator

org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion

org.apache.pig.impl.plan.OperatorKey

org.apache.pig.pen.LocalMapReduceSimulator

org.apache.pig.test.TestFRJoin2

org.apache.pig.test.TestJobControlCompiler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.