Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.MapredWork


    GenMRUnionCtx uCtxTask = ctx.getUnionTask(union);
    Task<? extends Serializable> uTask = null;

    Operator<? extends Serializable> parent = union.getParentOperators().get(pos);
    mapredWork uPlan = null;

    // union is encountered for the first time
    if (uCtxTask == null) {
      uCtxTask = new GenMRUnionCtx();
      uPlan = GenMapRedUtils.getMapRedWork();
      uTask = TaskFactory.get(uPlan, parseCtx.getConf());
      uCtxTask.setUTask(uTask);
      ctx.setUnionTask(union, uCtxTask);
    }
    else {
      uTask = uCtxTask.getUTask();
      uPlan = (mapredWork)uTask.getWork();
    }

    // If there is a mapjoin at position 'pos'
    if (uPrsCtx.getMapJoinSubq(pos)) {
      GenMRMapJoinCtx mjCtx = ctx.getMapJoinCtx(ctx.getCurrMapJoinOp());
      String taskTmpDir = mjCtx.getTaskTmpDir();
      if (uPlan.getPathToAliases().get(taskTmpDir) == null) {
        uPlan.getPathToAliases().put(taskTmpDir, new ArrayList<String>());
        uPlan.getPathToAliases().get(taskTmpDir).add(taskTmpDir);
        uPlan.getPathToPartitionInfo().put(taskTmpDir, new partitionDesc(mjCtx.getTTDesc(), null));
        uPlan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp());
      }

      for (Task t : currTask.getParentTasks())
        t.addDependentTask(uTask);
      try {
View Full Code Here


      // Generate the hiveConfArgs after potentially adding the jars
      String hiveConfArgs = ExecDriver.generateCmdLine(conf);
      File scratchDir = new File(conf.getVar(HiveConf.ConfVars.SCRATCHDIR));
     
      mapredWork plan = getWork();

      File planFile = File.createTempFile("plan", ".xml", scratchDir);
      LOG.info("Generating plan file " + planFile.toString());
      FileOutputStream out = new FileOutputStream(planFile);
      Utilities.serializeMapRedWork(plan, out);
View Full Code Here

    return true;
  }

  @Override
  public boolean hasReduce() {
    mapredWork w = getWork();
    return w.getReducer() != null;
  }
View Full Code Here

      int pos = getPositionParent(mapJoin, stack);

      Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos));
      Task<? extends Serializable> currTask    = mapredCtx.getCurrTask();
      mapredWork currPlan = (mapredWork) currTask.getWork();
      Operator<? extends Serializable> currTopOp   = mapredCtx.getCurrTopOp();
      String currAliasId = mapredCtx.getCurrAliasId();
      Operator<? extends Serializable> reducer = mapJoin;
      HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);
     
      ctx.setCurrTopOp(currTopOp);
      ctx.setCurrAliasId(currAliasId);
      ctx.setCurrTask(currTask);
     
      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert currPlan.getReducer() == null;
        GenMapRedUtils.initMapJoinPlan(mapJoin, ctx, false, false, false, pos);
      }
      // The current plan can be thrown away after being merged with the original plan
      else {
        GenMapRedUtils.joinPlan(mapJoin, null, opMapTask, ctx, pos, false, false, false);
View Full Code Here

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
        Object... nodeOutputs) throws SemanticException {
      MapJoinOperator mapJoin = (MapJoinOperator)nd;
      GenMRProcContext opProcCtx = (GenMRProcContext)procCtx;
     
      mapredWork cplan = GenMapRedUtils.getMapRedWork();
      ParseContext parseCtx = opProcCtx.getParseCtx();
      Task<? extends Serializable> redTask  = TaskFactory.get(cplan, parseCtx.getConf());
      Task<? extends Serializable> currTask = opProcCtx.getCurrTask();

      // find the branch on which this processor was invoked
      int pos = getPositionParent(mapJoin, stack);
      boolean local = (pos == ((mapJoinDesc)mapJoin.getConf()).getPosBigTable()) ? false : true;
     
      GenMapRedUtils.splitTasks(mapJoin, currTask, redTask, opProcCtx, false, local, pos);

      currTask = opProcCtx.getCurrTask();
      HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = opProcCtx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(mapJoin);
     
      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert cplan.getReducer() == null;
        opTaskMap.put(mapJoin, currTask);
        opProcCtx.setCurrMapJoinOp(null);
      }
      // The current plan can be thrown away after being merged with the original plan
      else {
View Full Code Here

          + Arrays.asList(((URLClassLoader)Thread.currentThread().getContextClassLoader()).getURLs()));
    } catch (Exception e) {
      l4j.info("cannot get classpath: " + e.getMessage());
    }
    jc = job;
    mapredWork gWork = Utilities.getMapRedWork(job);
    reducer = gWork.getReducer();
    reducer.setParentOperators(null); // clear out any parents as reducer is the root
    isTagged = gWork.getNeedsTagging();
    try {
      keyTableDesc = gWork.getKeyDesc();
      inputKeyDeserializer = (SerDe)ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), null);
      inputKeyDeserializer.initialize(null, keyTableDesc.getProperties());
      keyObjectInspector = inputKeyDeserializer.getObjectInspector();
      valueTableDesc = new tableDesc[gWork.getTagToValueDesc().size()];
      for(int tag=0; tag<gWork.getTagToValueDesc().size(); tag++) {
        // We should initialize the SerDe with the TypeInfo when available.
        valueTableDesc[tag] = gWork.getTagToValueDesc().get(tag);
        inputValueDeserializer[tag] = (SerDe)ReflectionUtils.newInstance(valueTableDesc[tag].getDeserializerClass(), null);
        inputValueDeserializer[tag].initialize(null, valueTableDesc[tag].getProperties());
        valueObjectInspector[tag] = inputValueDeserializer[tag].getObjectInspector();
       
        ArrayList<ObjectInspector> ois = new ArrayList<ObjectInspector>();
View Full Code Here

      l4j.info("cannot get classpath: " + e.getMessage());
    }
    try {
      jc = job;
      // create map and fetch operators
      mapredWork mrwork = Utilities.getMapRedWork(job);
      mo = new MapOperator();
      mo.setConf(mrwork);
      // initialize map operator
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
View Full Code Here

      if (mjCtx == null) {
        mjCtx = new GenMRMapJoinCtx();
        ctx.setMapJoinCtx(mapJoin, mjCtx);
      }
     
      mapredWork mjPlan = GenMapRedUtils.getMapRedWork();
      Task<? extends Serializable> mjTask = TaskFactory.get(mjPlan, parseCtx.getConf());
     
      tableDesc tt_desc =
        PlanUtils.getIntermediateFileTableDesc(
            PlanUtils.getFieldSchemasFromRowSchema(mapJoin.getSchema(), "temporarycol"));
View Full Code Here

      int pos = getPositionParent(mapJoin, stack);

      Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos));
      Task<? extends Serializable> currTask    = mapredCtx.getCurrTask();
      mapredWork currPlan = (mapredWork) currTask.getWork();
      String currAliasId = mapredCtx.getCurrAliasId();
      Operator<? extends Serializable> reducer = mapJoin;
      HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);
     
      ctx.setCurrTask(currTask);
     
      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert currPlan.getReducer() == null;
        GenMapRedUtils.initMapJoinPlan(mapJoin, ctx, true, false, false, pos);
      }
      // The current plan can be thrown away after being merged with the original plan
      else {
        GenMapRedUtils.joinPlan(mapJoin, currTask, opMapTask, ctx, pos, false, true, false);
View Full Code Here

      int pos = getPositionParent(mapJoin, stack);

      Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos));
      Task<? extends Serializable> currTask    = mapredCtx.getCurrTask();
      mapredWork currPlan = (mapredWork) currTask.getWork();
      Operator<? extends Serializable> reducer = mapJoin;
      HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);
     
      // union result cannot be a map table
      boolean local = (pos == ((mapJoinDesc)mapJoin.getConf()).getPosBigTable()) ? false : true;
      if (local)
        throw new SemanticException(ErrorMsg.INVALID_MAPJOIN_TABLE.getMsg());
     
      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert currPlan.getReducer() == null;
        ctx.setCurrMapJoinOp(mapJoin);
        GenMapRedUtils.initMapJoinPlan(mapJoin, ctx, true, true, false, pos);
        ctx.setCurrUnionOp(null);
      }
      // The current plan can be thrown away after being merged with the original plan
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.plan.MapredWork

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.