Examples of TaskLocationHint

org.apache.tez.dag.api.TaskLocationHint
Describes the placements hints for tasks. The system will make a best-effort attempt to run the tasks close to the specified locations.
org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint


    List<TaskLocationHint> locationHints =
        new ArrayList<TaskLocationHint>(splits.length);
    for (int i = 0; i < splits.length; ++i) {
      locationHints.add(
          new TaskLocationHint(new HashSet<String>(
              Arrays.asList(splits[i].getLocations())), null));
    }


    return new InputSplitInfoDisk(
        JobSubmissionFiles.getJobSplitFile(inputSplitDir),

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

    Iterable<TaskLocationHint> iterable = Iterables.transform(Arrays.asList(newFormatSplits),
        new Function<org.apache.hadoop.mapreduce.InputSplit, TaskLocationHint>() {
          @Override
          public TaskLocationHint apply(org.apache.hadoop.mapreduce.InputSplit input) {
            try {
              return new TaskLocationHint(new HashSet<String>(Arrays.asList(input.getLocations())),
                  null);
            } catch (IOException e) {
              throw new RuntimeException(e);
            } catch (InterruptedException e) {
              throw new RuntimeException(e);

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

    Iterable<TaskLocationHint> iterable = Iterables.transform(Arrays.asList(oldFormatSplits),
        new Function<org.apache.hadoop.mapred.InputSplit, TaskLocationHint>() {
          @Override
          public TaskLocationHint apply(org.apache.hadoop.mapred.InputSplit input) {
            try {
              return new TaskLocationHint(new HashSet<String>(Arrays.asList(input.getLocations())),
                  null);
            } catch (IOException e) {
              throw new RuntimeException(e);
            }
          }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

        for (org.apache.hadoop.mapreduce.InputSplit split : splits) {
          String rack = 
              ((org.apache.hadoop.mapreduce.split.TezGroupedSplit) split).getRack();
          if (rack == null) {
            if (split.getLocations() != null) {
              locationHints.add(new TaskLocationHint(new HashSet<String>(Arrays
                  .asList(split.getLocations())), null));
            } else {
              locationHints.add(new TaskLocationHint(null, null));
            }
          } else {
            locationHints.add(new TaskLocationHint(null, 
                Collections.singleton(rack)));
          }
        }
        inputSplitInfo = new InputSplitInfoMem(splits, locationHints, splits.length, null, conf);
      } else {
        LOG.info("Grouping mapred api input splits");
        org.apache.hadoop.mapred.InputSplit[] splits = MRHelpers
            .generateOldSplits(jobConf, realInputFormatName, numTasks);
        List<TaskLocationHint> locationHints = Lists
            .newArrayListWithCapacity(splits.length);
        for (org.apache.hadoop.mapred.InputSplit split : splits) {
          String rack = 
              ((org.apache.hadoop.mapred.split.TezGroupedSplit) split).getRack();
          if (rack == null) {
            if (split.getLocations() != null) {
              locationHints.add(new TaskLocationHint(new HashSet<String>(Arrays
                  .asList(split.getLocations())), null));
            } else {
              locationHints.add(new TaskLocationHint(null, null));
            }
          } else {
            locationHints.add(new TaskLocationHint(null, 
                Collections.singleton(rack)));
          }
        }
        inputSplitInfo = new InputSplitInfoMem(splits, locationHints, splits.length, null, conf);
      }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint


    List<TaskLocationHint> locationHints =
        new ArrayList<TaskLocationHint>(splits.length);
    for (int i = 0; i < splits.length; ++i) {
      locationHints.add(
          new TaskLocationHint(new HashSet<String>(
              Arrays.asList(splits[i].getLocations())), null));
    }


    return new InputSplitInfoDisk(
        JobSubmissionFiles.getJobSplitFile(inputSplitDir),

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint


    List<TaskLocationHint> locationHints =
        new ArrayList<TaskLocationHint>(splits.length);
    for (int i = 0; i < splits.length; ++i) {
      locationHints.add(
          new TaskLocationHint(new HashSet<String>(
              Arrays.asList(splits[i].getLocations())), null));
    }


    return new InputSplitInfoDisk(
        JobSubmissionFiles.getJobSplitFile(inputSplitDir),

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint


    List<TaskLocationHint> locationHints = Lists
        .newArrayListWithCapacity(newSplits.length);
    for (org.apache.hadoop.mapreduce.InputSplit newSplit : newSplits) {
      splitsBuilder.addSplits(createSplitProto(newSplit, serializationFactory));
      locationHints.add(new TaskLocationHint(new HashSet<String>(Arrays
          .asList(newSplit.getLocations())), null));
    }
    return new InputSplitInfoMem(splitsBuilder.build(), locationHints,
        newSplits.length);
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint


    List<TaskLocationHint> locationHints = Lists
        .newArrayListWithCapacity(oldSplits.length);
    for (org.apache.hadoop.mapred.InputSplit oldSplit : oldSplits) {
      splitsBuilder.addSplits(createSplitProto(oldSplit));
      locationHints.add(new TaskLocationHint(new HashSet<String>(Arrays
          .asList(oldSplit.getLocations())), null));
    }
    return new InputSplitInfoMem(splitsBuilder.build(), locationHints,
        oldSplits.length);
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

  
  private List<TaskLocationHint> createTaskLocationHints(int numTasks) {
    List<TaskLocationHint> locationHints = Lists
        .newArrayListWithCapacity(numTasks);
    for (int i = 0; i < numTasks; i++) {
      TaskLocationHint taskLocationHint = new TaskLocationHint(
          Sets.newSet("host" + i), null);
      locationHints.add(taskLocationHint);
    }
    return locationHints;
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint

    EventHandler eventHandler = mock(EventHandler.class);
    Set<String> hosts = new TreeSet<String>();
    hosts.add("host1");
    hosts.add("host2");
    hosts.add("host3");
    TaskLocationHint locationHint = new TaskLocationHint(hosts, null);


    TezTaskID taskID = new TezTaskID(
        new TezVertexID(new TezDAGID("1", 1, 1), 1), 1);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
        mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.