Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

org.apache.hadoop.hive.ql.plan.ReduceWork
ReduceWork represents all the information used to run a reduce task on the cluster. It is first used when the query planner breaks the logical plan into tasks and used throughout physical optimization to track reduce-side operator plans, schema info about key/value pairs, etc ExecDriver will serialize the contents of this class and make sure it is distributed on the cluster. The ExecReducer will ultimately deserialize this class on the data nodes and setup it's operator pipeline accordingly. This class is also used in the explain command any property with the appropriate annotation will be displayed in the explain output.

      }
    }
  }


  private void checkMRReducer(String taskName, MapredWork mrWrk) throws SemanticException {
    ReduceWork rWrk = mrWrk.getReduceWork();
    if ( rWrk == null) {
      return;
    }
    Operator<? extends OperatorDesc> reducer = rWrk.getReducer();
    if ( reducer instanceof JoinOperator ) {
      BaseWork prntWork = mrWrk.getMapWork();
      checkForCrossProduct(taskName, reducer,
          new ExtractReduceSinkInfo(null).analyze(prntWork));
    }

View Full Code Here

    return true;
  }


  private JoinOperator getJoinOp(MapRedTask task) throws SemanticException {
    MapWork mWork = task.getWork().getMapWork();
    ReduceWork rWork = task.getWork().getReduceWork();
    if (rWork == null) {
      return null;
    }
    Operator<? extends OperatorDesc> reducerOp = rWork.getReducer();
    if (reducerOp instanceof JoinOperator) {
      /* Is any operator present, which prevents the conversion */
      Map<String, Operator<? extends OperatorDesc>> aliasToWork = mWork.getAliasToWork();
      for (Operator<? extends OperatorDesc> op : aliasToWork.values()) {
        if (!checkOperatorOKMapJoinConversion(op)) {

View Full Code Here

        getMapJoinOperator(newTask, newWork, smbJoinOp, joinTree, bigTablePosition);


    // The reducer needs to be restored - Consider a query like:
    // select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key;
    // The reducer contains a groupby, which needs to be restored.
    ReduceWork rWork = newWork.getReduceWork();


    // create the local work for this plan
    MapJoinProcessor.genLocalWorkForMapJoin(newWork, newMapJoinOp, bigTablePosition);


    // restore the reducer

View Full Code Here


      // need to add this branch to the key + value info
      assert operator instanceof ReduceSinkOperator
        && followingWork instanceof ReduceWork;
      ReduceSinkOperator rs = (ReduceSinkOperator) operator;
      ReduceWork rWork = (ReduceWork) followingWork;
      GenMapRedUtils.setKeyAndValueDesc(rWork, rs);


      // remember which parent belongs to which tag
      rWork.getTagToInput().put(rs.getConf().getTag(), work.getName());


      // remember the output name of the reduce sink
      rs.getConf().setOutputName(rWork.getName());


      if (!context.connectedReduceSinks.contains(rs)) {
        // add dependency between the two work items
        TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE);
        tezWork.connect(work, rWork, edgeProp);

View Full Code Here

    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        opProcCtx.getOpTaskMap();
    Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();


    opTaskMap.put(reducer, currTask);
    plan.setReduceWork(new ReduceWork());
    plan.getReduceWork().setReducer(reducer);
    ReduceSinkDesc desc = op.getConf();


    plan.getReduceWork().setNumReduceTasks(desc.getNumReducers());

View Full Code Here

    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        opProcCtx.getOpTaskMap();


    opTaskMap.put(reducer, unionTask);


    plan.setReduceWork(new ReduceWork());
    plan.getReduceWork().setReducer(reducer);
    plan.getReduceWork().setReducer(reducer);
    ReduceSinkDesc desc = op.getConf();


    plan.getReduceWork().setNumReduceTasks(desc.getNumReducers());

View Full Code Here

    Task<? extends Serializable> childTask = TaskFactory.get(childPlan, parseCtx
        .getConf());
    Operator<? extends OperatorDesc> reducer = cRS.getChildOperators().get(0);


    // Add the reducer
    ReduceWork rWork = new ReduceWork();
    childPlan.setReduceWork(rWork);
    rWork.setReducer(reducer);
    ReduceSinkDesc desc = cRS.getConf();
    childPlan.getReduceWork().setNumReduceTasks(new Integer(desc.getNumReducers()));


    opProcCtx.getOpTaskMap().put(reducer, childTask);

View Full Code Here

        } else {
          // There are separate configuration parameters to control whether to
          // merge for a map-only job
          // or for a map-reduce job
          if (currTask.getWork() instanceof MapredWork) {  
            ReduceWork reduceWork = ((MapredWork) currTask.getWork()).getReduceWork();
            boolean mergeMapOnly =
              hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null;
            boolean mergeMapRed =
              hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) &&
              reduceWork != null;

View Full Code Here


    BaseWork w = work.getAllWork().get(1);
    assertTrue(w instanceof ReduceWork);
    assertTrue(work.getParents(w).contains(work.getAllWork().get(0)));


    ReduceWork rw = (ReduceWork)w;


    // need to make sure names are set for tez to connect things right
    assertNotNull(w.getName());


    // map work should start with our ts op
    assertSame(rw.getReducer(),fs);


    // should have severed the ties
    assertEquals(fs.getParentOperators().size(),0);
  }

View Full Code Here

        .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")),
        Utilities.makeList(getStringColumn("value")), outputColumns, true,
        -1, 1, -1));


    addMapWork(mr, src, "a", op1);
    ReduceWork rWork = new ReduceWork();
    rWork.setNumReduceTasks(Integer.valueOf(1));
    rWork.setKeyDesc(op1.getConf().getKeySerializeInfo());
    rWork.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
    mr.setReduceWork(rWork);


    // reduce side work
    Operator<FileSinkDesc> op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator
        + "mapredplan1.out"), Utilities.defaultTd, false));


    Operator<ExtractDesc> op2 = OperatorFactory.get(new ExtractDesc(
        getStringColumn(Utilities.ReduceField.VALUE.toString())), op3);


    rWork.setReducer(op2);
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.hive.ql.plan.ReduceWork

org.apache.hadoop.hive.ql.DriverContext

org.apache.hadoop.hive.ql.exec.mr.ExecDriver

org.apache.hadoop.hive.ql.exec.mr.ExecReducer

org.apache.hadoop.hive.ql.exec.mr.MapRedTask

org.apache.hadoop.hive.ql.exec.TestExecDriver

org.apache.hadoop.hive.ql.exec.tez.TestTezTask

org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils

org.apache.hadoop.hive.ql.optimizer.physical.CommonJoinTaskDispatcher

org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck

org.apache.hadoop.hive.ql.optimizer.physical.SamplingOptimizer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.