Package org.apache.hadoop.hive.ql.exec

Examples of org.apache.hadoop.hive.ql.exec.ConditionalTask

252
253
254
255
256
257
258
259
260
261
    // since it is unknown if the merge MR will be triggered at execution time.

    MoveWork dummyMv = new MoveWork(null, null, null,
        new LoadFileDesc(fsConf.getDirName(), finalName, true, null, null), false);

    ConditionalTask cndTsk = createCondTask(conf, currTask, dummyMv, cplan,
        fsConf.getDirName());

    LinkMoveTask(ctx, newOutput, cndTsk);
  }
View Full Code Here
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
      // use CombineHiveInputFormat for map-only merging
    }
    cplan.setInputformat("org.apache.hadoop.hive.ql.io.CombineHiveInputFormat");
    // NOTE: we should gather stats in MR1 rather than MR2 at merge job since we don't
    // know if merge MR2 will be triggered at execution time
    ConditionalTask cndTsk = createCondTask(ctx.getConf(), ctx.getCurrTask(), dummyMv, cplan,
        fsInputDesc.getDirName());

    // keep the dynamic partition context in conditional task resolver context
    ConditionalResolverMergeFilesCtx mrCtx =
      (ConditionalResolverMergeFilesCtx) cndTsk.getResolverCtx();
    mrCtx.setDPCtx(fsInputDesc.getDynPartCtx());

    //
    // 3. add the moveTask as the children of the conditional task
    //
View Full Code Here
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
    List<Task<? extends Serializable>> listTasks = new ArrayList<Task<? extends Serializable>>();
    listTasks.add(moveTask);
    listTasks.add(mergeTask);

    ConditionalTask cndTsk = (ConditionalTask) TaskFactory.get(cndWork, conf);
    cndTsk.setListTasks(listTasks);

    // create resolver
    cndTsk.setResolver(new ConditionalResolverMergeFiles());
    ConditionalResolverMergeFilesCtx mrCtx =
      new ConditionalResolverMergeFilesCtx(listTasks, inputPath);
    cndTsk.setResolverCtx(mrCtx);

    // make the conditional task as the child of the current leaf task
    currTask.addDependentTask(cndTsk);

    return cndTsk;
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    List<Serializable> listWorks = new ArrayList<Serializable>();
    listWorks.add(dummyMv);
    listWorks.add(mergeTask.getWork());
    ConditionalWork cndWork = new ConditionalWork(listWorks);
   
    ConditionalTask cndTsk = (ConditionalTask)TaskFactory.get(cndWork, ctx.getConf());
    List<Task<? extends Serializable>> listTasks = new ArrayList<Task<? extends Serializable>>();
    listTasks.add(dummyMergeTask);
    listTasks.add(mergeTask);
    cndTsk.setListTasks(listTasks);
   
    cndTsk.setResolver(new ConditionalResolverMergeFiles());
    cndTsk.setResolverCtx(new ConditionalResolverMergeFilesCtx(listTasks, fsOp.getConf().getDirName()));
   
    currTask.addDependentTask(cndTsk);
   
    List<Task<? extends Serializable>> mvTasks = ctx.getMvTask();
    Task<? extends Serializable> mvTask = findMoveTask(mvTasks, newOutput);
   
    if (mvTask != null)
      cndTsk.addDependentTask(mvTask);
  }
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
      }
      if (task instanceof ConditionalTask) {
        org.apache.hadoop.hive.ql.plan.api.Adjacency listEntry = new org.apache.hadoop.hive.ql.plan.api.Adjacency();
        listEntry.setAdjacencyType(AdjacencyType.DISJUNCTIVE);
        listEntry.setNode(task.getId());
        ConditionalTask t = (ConditionalTask)task;
       
        for (Task<? extends Serializable> listTask: t.getListTasks()) {
          if (t.getChildTasks() != null) {
            org.apache.hadoop.hive.ql.plan.api.Adjacency childEntry = new org.apache.hadoop.hive.ql.plan.api.Adjacency();
            childEntry.setAdjacencyType(AdjacencyType.DISJUNCTIVE);
            childEntry.setNode(listTask.getId());
            // done processing the task
            for (Task<? extends Serializable> childTask: t.getChildTasks()) {
              childEntry.addToChildren(childTask.getId());
              if (!tasksVisited.contains(childTask)) {
                tasksToVisit.add(childTask);
              }
            }
View Full Code Here
317
318
319
320
321
322
323
324
325
326
327
328
            done.add(task.getId() + "_REDUCE");
          }
        }
      }
      else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask)task;
        for (Task<? extends Serializable> listTask: cTask.getListTasks()) {
          if (!tasksVisited.contains(listTask)) {
            tasksToVisit.add(listTask);
          }
        }
      }
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
      if (task instanceof ConditionalTask) {
        org.apache.hadoop.hive.ql.plan.api.Adjacency listEntry =
          new org.apache.hadoop.hive.ql.plan.api.Adjacency();
        listEntry.setAdjacencyType(AdjacencyType.DISJUNCTIVE);
        listEntry.setNode(task.getId());
        ConditionalTask t = (ConditionalTask) task;

        for (Task<? extends Serializable> listTask : t.getListTasks()) {
          if (t.getChildTasks() != null) {
            org.apache.hadoop.hive.ql.plan.api.Adjacency childEntry =
              new org.apache.hadoop.hive.ql.plan.api.Adjacency();
            childEntry.setAdjacencyType(AdjacencyType.DISJUNCTIVE);
            childEntry.setNode(listTask.getId());
            // done processing the task
            for (Task<? extends Serializable> childTask : t.getChildTasks()) {
              childEntry.addToChildren(childTask.getId());
              if (!tasksVisited.contains(childTask)) {
                tasksToVisit.add(childTask);
              }
            }
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
402
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;
        for (Task<? extends Serializable> listTask : cTask.getListTasks()) {
          if (!tasksVisited.contains(listTask)) {
            tasksToVisit.add(listTask);
          }
        }
      }
View Full Code Here
331
332
333
334
335
336
337
338
339
340
    // since it is unknown if the merge MR will be triggered at execution time.

    MoveWork dummyMv = new MoveWork(null, null, null,
        new LoadFileDesc(fsConf.getFinalDirName(), finalName, true, null, null), false);

    ConditionalTask cndTsk = createCondTask(conf, currTask, dummyMv, cplan,
        fsConf.getFinalDirName());

    linkMoveTask(ctx, newOutput, cndTsk);
  }
View Full Code Here
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      // use CombineHiveInputFormat for map-only merging
    }
    cplan.setInputformat("org.apache.hadoop.hive.ql.io.CombineHiveInputFormat");
    // NOTE: we should gather stats in MR1 rather than MR2 at merge job since we don't
    // know if merge MR2 will be triggered at execution time
    ConditionalTask cndTsk = createCondTask(ctx.getConf(), ctx.getCurrTask(), dummyMv, cplan,
        fsInputDesc.getFinalDirName());

    // keep the dynamic partition context in conditional task resolver context
    ConditionalResolverMergeFilesCtx mrCtx =
      (ConditionalResolverMergeFilesCtx) cndTsk.getResolverCtx();
    mrCtx.setDPCtx(fsInputDesc.getDynPartCtx());

    //
    // 3. add the moveTask as the children of the conditional task
    //
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.exec.ConditionalTask

  • org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils
  • org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1
  • org.apache.hadoop.hive.ql.optimizer.physical.CommonJoinResolver$CommonJoinTaskDispatcher
  • org.apache.hadoop.hive.ql.optimizer.physical.CommonJoinTaskDispatcher
  • org.apache.hadoop.hive.ql.optimizer.physical.GenMRSkewJoinProcessor
  • org.apache.hadoop.hive.ql.optimizer.physical.SortMergeJoinTaskDispatcher
  • org.apache.hadoop.hive.ql.QueryPlan
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.