Package org.apache.hadoop.hive.ql.io.rcfile.merge

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
    mergeWork.resolveConcatenateMerge(db.getConf());
    mergeWork.setMapperCannotSpanPartns(true);
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
    taskExec.setQueryPlan(this.getQueryPlan());
View Full Code Here
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
      if (!hasDynamicPartitions
          && !isSkewedStoredAsDirs(fsInputDesc)) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions, fsInputDesc.getDynPartCtx());
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
      if (hasDynamicPartitions
          || isSkewedStoredAsDirs(fsInputDesc)) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }
      work.setListBucketingCtx(fsInputDesc.getLbCtx());

      return work;
    }

    throw new SemanticException("createRCFileMergeTask called on non-RCFile table");
View Full Code Here
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here
396
397
398
399
400
401
402
403
404
405
406
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
    mergeWork.resolveConcatenateMerge(db.getConf());
    mergeWork.setMapperCannotSpanPartns(true);
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
    taskExec.setQueryPlan(this.getQueryPlan());
View Full Code Here
460
461
462
463
464
465
466
467
468
469
470
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions, fsInputDesc.getDynPartCtx());
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hive.ql.exec.DDLTask
  • org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils
  • org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1
  • org.apache.hadoop.hive.ql.plan.PartitionDesc
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.