Examples of org.apache.hadoop.mapred.split.TezMapredSplitsGrouper

org.apache.hadoop.mapred.split.TezMapredSplitsGrouper
A Helper that provides grouping logic to group InputSplits using various parameters. A {@link TezGroupedSplit} is usedto wrap the real InputSplits in a group.

        ArrayListMultimap.<Integer, InputSplit>create(bucketToInitialSplitMap);
    
    Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
    for (int bucketId : bucketSplitMap.keySet()) {
      Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
      TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();


      InputSplit[] groupedSplits = grouper.getGroupedSplits(conf, 
          inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
          HiveInputFormat.class.getName());
      LOG.info("Original split size is " + 
          inputSplitCollection.toArray(new InputSplit[0]).length + 
          " grouped split size is " + groupedSplits.length);

View Full Code Here

    if (conf.getBoolean(GROUP_SPLITS, true)) {
      estimateBucketSizes();
      Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
      for (int bucketId : bucketSplitMap.keySet()) {
        Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
        TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();


        InputSplit[] groupedSplits = grouper.getGroupedSplits(conf, 
            inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
            HiveInputFormat.class.getName());
        LOG.info("Original split size is " + 
            inputSplitCollection.toArray(new InputSplit[0]).length + 
            " grouped split size is " + groupedSplits.length);

View Full Code Here

    if (conf.getBoolean(GROUP_SPLITS, true)) {
      estimateBucketSizes();
      Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
      for (int bucketId : bucketSplitMap.keySet()) {
        Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
        TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();


        InputSplit[] groupedSplits = grouper.getGroupedSplits(conf, 
            inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
            HiveInputFormat.class.getName());
        LOG.info("Original split size is " + 
            inputSplitCollection.toArray(new InputSplit[0]).length + 
            " grouped split size is " + groupedSplits.length);

View Full Code Here

        ArrayListMultimap.<Integer, InputSplit>create(bucketToInitialSplitMap);
    
    Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
    for (int bucketId : bucketSplitMap.keySet()) {
      Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
      TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();


      InputSplit[] groupedSplits = grouper.getGroupedSplits(conf, 
          inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
          HiveInputFormat.class.getName());
      LOG.info("Original split size is " + 
          inputSplitCollection.toArray(new InputSplit[0]).length + 
          " grouped split size is " + groupedSplits.length);

View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.split.TezMapredSplitsGrouper

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.tez.dag.api.TezUncheckedException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.