Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.MoveWork

207
208
209
210
211
212
213
214
215
216
217
218
                                        (ts.partSpec != null) ? ts.partSpec :
                                        new HashMap<String, String> (),
                                        isOverWrite);

    if(rTask != null) {
      rTask.addDependentTask(TaskFactory.get(new moveWork(getInputs(), getOutputs(), loadTableWork, null, true), this.conf));
    } else {
      rTask = TaskFactory.get(new moveWork(getInputs(), getOutputs(), loadTableWork, null, true), this.conf);
    }

    rootTasks.add(rTask);
  }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
    cplan.getPathToAliases().put(fsConf.getDirName(), aliases);
    cplan.getAliasToWork().put(fsConf.getDirName(), ts_op);   
    cplan.getPathToPartitionInfo().put(fsConf.getDirName(), new partitionDesc(fsConf.getTableInfo(), null));
    cplan.setNumReduceTasks(-1);
   
    moveWork dummyMv = new moveWork(null, null, null, new loadFileDesc(fsOp.getConf().getDirName(), finalName, true, null, null), false);
    Task<? extends Serializable> dummyMergeTask = TaskFactory.get(dummyMv, ctx.getConf());
    List<Serializable> listWorks = new ArrayList<Serializable>();
    listWorks.add(dummyMv);
    listWorks.add(mergeTask.getWork());
    ConditionalWork cndWork = new ConditionalWork(listWorks);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
  }
  private Task<? extends Serializable> findMoveTask(List<Task<? extends Serializable>> mvTasks, FileSinkOperator fsOp) {
    // find the move task
    for (Task<? extends Serializable> mvTsk : mvTasks) {
      moveWork mvWork = (moveWork)mvTsk.getWork();
      String srcDir = null;
      if (mvWork.getLoadFileWork() != null)
        srcDir = mvWork.getLoadFileWork().getSourceDir();
      else if (mvWork.getLoadTableWork() != null)
        srcDir = mvWork.getLoadTableWork().getSourceDir();
     
      if ((srcDir != null) && (srcDir.equalsIgnoreCase(fsOp.getConf().getDirName())))
        return mvTsk;
    }
    
View Full Code Here
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
    } else {
      // First we generate the move work as this needs to be made dependent on all
      // the tasks that have a file sink operation
      List<moveWork>  mv = new ArrayList<moveWork>();
      for (loadTableDesc ltd : loadTableWork)
        mvTask.add(TaskFactory.get(new moveWork(null, null, ltd, null, false), this.conf));

      boolean oneLoadFile = true;
      for (loadFileDesc lfd : loadFileWork) {
        if ( qb.isCTAS() ) {
          assert(oneLoadFile); // should not have more than 1 load file for CTAS
          // make the movetask's destination directory the table's destination.
          String location = qb.getTableDesc().getLocation();
          if ( location == null ) {
            // get the table's default location
            location = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
            assert(location.length() > 0 );
            if ( location.charAt(location.length()-1) != '/' ) {
              location += '/';
            }
            location += qb.getTableDesc().getTableName().toLowerCase();
          }
          lfd.setTargetDir(location);
          oneLoadFile = false;
        }
        mvTask.add(TaskFactory.get(new moveWork(null, null, null, lfd, false), this.conf));
      }
    }

    // generate map reduce plans
    GenMRProcContext procCtx =
View Full Code Here
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
    else {
      // First we generate the move work as this needs to be made dependent on all
      // the tasks that have a file sink operation
      List<moveWork>  mv = new ArrayList<moveWork>();
      for (loadTableDesc ltd : loadTableWork)
        mvTask.add(TaskFactory.get(new moveWork(ltd, null, false), this.conf));
      for (loadFileDesc lfd : loadFileWork)
        mvTask.add(TaskFactory.get(new moveWork(null, lfd, false), this.conf));
    }

    // generate map reduce plans
    GenMRProcContext procCtx =
      new GenMRProcContext(
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    cplan.getPathToAliases().put(fsConf.getDirName(), aliases);
    cplan.getAliasToWork().put(fsConf.getDirName(), ts_op);   
    cplan.getPathToPartitionInfo().put(fsConf.getDirName(), new partitionDesc(fsConf.getTableInfo(), null));
    cplan.setNumReduceTasks(-1);
   
    moveWork dummyMv = new moveWork(null, new loadFileDesc(fsOp.getConf().getDirName(), finalName, true, null, null), false);
    Task<? extends Serializable> dummyMergeTask = TaskFactory.get(dummyMv, ctx.getConf());
    List<Serializable> listWorks = new ArrayList<Serializable>();
    listWorks.add(dummyMv);
    listWorks.add(mergeTask.getWork());
    ConditionalWork cndWork = new ConditionalWork(listWorks);
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
  }
  private Task<? extends Serializable> findMoveTask(List<Task<? extends Serializable>> mvTasks, FileSinkOperator fsOp) {
    // find the move task
    for (Task<? extends Serializable> mvTsk : mvTasks) {
      moveWork mvWork = (moveWork)mvTsk.getWork();
      String srcDir = null;
      if (mvWork.getLoadFileWork() != null)
        srcDir = mvWork.getLoadFileWork().getSourceDir();
      else if (mvWork.getLoadTableWork() != null)
        srcDir = mvWork.getLoadTableWork().getSourceDir();
     
      if ((srcDir != null) && (srcDir.equalsIgnoreCase(fsOp.getConf().getDirName())))
        return mvTsk;
    }
    
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
218
                                        (ts.partSpec != null) ? ts.partSpec :
                                        new HashMap<String, String> (),
                                        isOverWrite);

    if(rTask != null) {
      rTask.addDependentTask(TaskFactory.get(new moveWork(loadTableWork, null, true), this.conf));
    } else {
      rTask = TaskFactory.get(new moveWork(loadTableWork, null, true), this.conf);
    }

    rootTasks.add(rTask);
  }
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
    loadTableWork.add(new loadTableDesc(fromURI.toString(), Utilities.getTableDesc(ts.tableHandle),
                                        (ts.partSpec != null) ? ts.partSpec : new HashMap<String, String> (),
                                        isOverWrite));

    if(rTask != null) {
      rTask.addDependentTask(TaskFactory.get(new moveWork(loadTableWork, loadFileWork), this.conf));
    } else {
      rTask = TaskFactory.get(new moveWork(loadTableWork, loadFileWork), this.conf);
    }

    rootTasks.add(rTask);
  }
View Full Code Here
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
    else {
      // First we generate the move work as this needs to be made dependent on all
      // the tasks that have a file sink operation
      List<moveWork>  mv = new ArrayList<moveWork>();
      for (loadTableDesc ltd : loadTableWork)
        mvTask.add(TaskFactory.get(new moveWork(ltd, null, false), this.conf));
      for (loadFileDesc lfd : loadFileWork)
        mvTask.add(TaskFactory.get(new moveWork(null, lfd, false), this.conf));
    }

    // generate map reduce plans
    GenMRProcContext procCtx =
      new GenMRProcContext(
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.plan.MoveWork

  • org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils
  • org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1
  • org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer
  • org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer
  • org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer
  • org.apache.hadoop.hive.ql.parse.MapReduceCompiler
  • org.apache.hadoop.hive.ql.parse.SemanticAnalyzer
  • org.apache.hadoop.hive.ql.parse.TaskCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.