Examples of InputSplitShim

Examples of org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim

387
388
389
390
391
392
393
394
395
396
      CombineFileSplit[] splits = (CombineFileSplit[]) super.getSplits(job, numSplits);

      InputSplitShim[] isplits = new InputSplitShim[splits.length];
      for (int pos = 0; pos < splits.length; pos++) {
        isplits[pos] = new InputSplitShim(splits[pos]);
      }

      return isplits;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim

394
395
396
397
398
399
400
      return isplits;
    }

    public InputSplitShim getInputSplitShim() throws IOException {
      return new InputSplitShim();
    }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim

391
392
393
394
395
396
397
398
399
400
      CombineFileSplit[] splits = (CombineFileSplit[]) super.getSplits(job, numSplits);

      InputSplitShim[] isplits = new InputSplitShim[splits.length];
      for (int pos = 0; pos < splits.length; pos++) {
        isplits[pos] = new InputSplitShim(splits[pos]);
      }

      return isplits;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim

398
399
400
401
402
403
404
      return isplits;
    }

    public InputSplitShim getInputSplitShim() throws IOException {
      return new InputSplitShim();
    }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
      long targetSize = splitSample.getTargetSize(totalSize);
      int startIndex = splitSample.getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
      long targetSize = (long) (totalSize * nameToSamples.get(entry.getKey()).getPercent() / 100D);
      int startIndex = nameToSamples.get(entry.getKey()).getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      long targetSize = (long) (totalSize * nameToSamples.get(entry.getKey()).getPercent() / 100D);
      int startIndex = nameToSamples.get(entry.getKey()).getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
      long targetSize = (long) (totalSize * nameToSamples.get(entry.getKey()).getPercent() / 100D);
      int startIndex = nameToSamples.get(entry.getKey()).getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      long targetSize = (long) (totalSize * nameToSamples.get(entry.getKey()).getPercent() / 100D);
      int startIndex = nameToSamples.get(entry.getKey()).getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim

533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
      long targetSize = splitSample.getTargetSize(totalSize);
      int startIndex = splitSample.getSeedNum() % splitList.size();
      long size = 0;
      for (int i = 0; i < splitList.size(); i++) {
        InputSplitShim split = splitList.get((startIndex + i) % splitList.size());
        retLists.add(split);
        long splitgLength = split.getLength();
        if (size + splitgLength >= targetSize) {
          LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits");
          if (size + splitgLength > targetSize) {
            split.shrinkSplit(targetSize - size);
          }
          break;
        }
        size += splitgLength;
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.