Examples of org.apache.tez.dag.api.TaskLocationHint

org.apache.tez.dag.api.TaskLocationHint
Describes the placements hints for tasks. The system will make a best-effort attempt to run the tasks close to the specified locations.

            new Path(jobSubmitDir));
    int splitsCount = splitsInfo.length;
    List<TaskLocationHint> locationHints =
        new ArrayList<TaskLocationHint>(splitsCount);
    for (int i = 0; i < splitsCount; ++i) {
      TaskLocationHint locationHint =
          TaskLocationHint.createTaskLocationHint(
              new HashSet<String>(
                  Arrays.asList(splitsInfo[i].getLocations())), null
          );
      locationHints.add(locationHint);

View Full Code Here


  private List<TaskLocationHint> createTaskLocationHints(int numTasks) {
    List<TaskLocationHint> locationHints = Lists
        .newArrayListWithCapacity(numTasks);
    for (int i = 0; i < numTasks; i++) {
      TaskLocationHint taskLocationHint = TaskLocationHint.createTaskLocationHint(
          Sets.newSet("host" + i), null);
      locationHints.add(taskLocationHint);
    }
    return locationHints;
  }

View Full Code Here


  private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(
    TezTaskAttemptID taID, TaskAttempt ta, Resource capability,
    String[] hosts, String[] racks, Priority priority,
    ContainerContext containerContext) {
    TaskLocationHint locationHint = null;
    if (hosts != null || racks != null) {
      Set<String> hostsSet = Sets.newHashSet(hosts);
      Set<String> racksSet = Sets.newHashSet(racks);
      locationHint = TaskLocationHint.createTaskLocationHint(hostsSet, racksSet);
    }

View Full Code Here

      // start only the ready 1-1 tasks
      tasksToStart = Lists.newLinkedList();
      for (int i=0; i<taskIsStarted.length; ++i) {
        if (!taskIsStarted[i] && oneToOneSrcTasksDoneCount[i] == numOneToOneEdges) {
          taskIsStarted[i] = true;
          TaskLocationHint locationHint = null;
          if (oneToOneLocationHints[i] != null) {
            locationHint = TaskLocationHint.createTaskLocationHint(oneToOneLocationHints[i].getId());
          }
          LOG.info("Starting task " + i + " for vertex: "
              + getContext().getVertexName() + " with location: "
              + ((locationHint != null) ? locationHint.getAffinitizedContainer() : "null"));
          tasksToStart.add(new TaskWithLocationHint(new Integer(i), locationHint));
        }
      }
    }

View Full Code Here


      String[] requestHosts = new String[0];


      // Compute node/rack location request even if re-scheduled.
      Set<String> racks = new HashSet<String>();
      TaskLocationHint locationHint = ta.getTaskLocationHint();
      if (locationHint != null) {
        if (locationHint.getRacks() != null) {
          racks.addAll(locationHint.getRacks());
        }
        if (locationHint.getHosts() != null) {
          for (String host : locationHint.getHosts()) {
            racks.add(RackResolver.resolve(host).getNetworkLocation());
          }
          requestHosts = ta.resolveHosts(locationHint.getHosts()
              .toArray(new String[locationHint.getHosts().size()]));
        }
      }


      ta.taskHosts.addAll(Arrays.asList(requestHosts));
      ta.taskRacks = racks;

View Full Code Here

    }
  }


  private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
    TaskAttempt taskAttempt = event.getTaskAttempt();
    TaskLocationHint locationHint = event.getLocationHint();
    String hosts[] = null;
    String racks[] = null;
    if (locationHint != null) {
      if (locationHint.getAffinitizedContainer() != null) {
        taskScheduler.allocateTask(taskAttempt,
            event.getCapability(),
            locationHint.getAffinitizedContainer(),
            event.getPriority(),
            event.getContainerContext(),
            event);
        return;
      } else {
        hosts = (locationHint.getHosts() != null) ? locationHint
            .getHosts().toArray(
                new String[locationHint.getHosts().size()]) : null;
        racks = (locationHint.getRacks() != null) ? locationHint.getRacks()
            .toArray(new String[locationHint.getRacks().size()]) : null;
      }
    }
    
    taskScheduler.allocateTask(taskAttempt,
                               event.getCapability(),

View Full Code Here

      for (TaskWithLocationHint task : tasksToSchedule) {
        if (numTasks <= task.getTaskIndex().intValue()) {
          throw new TezUncheckedException(
              "Invalid taskId: " + task.getTaskIndex() + " for vertex: " + logIdentifier);
        }
        TaskLocationHint locationHint = task.getTaskLocationHint();
        if (locationHint != null) {
          if (taskLocationHints == null) {
            taskLocationHints = new TaskLocationHint[numTasks];
          }
          taskLocationHints[task.getTaskIndex().intValue()] = locationHint;

View Full Code Here

    EventHandler eventHandler = mock(EventHandler.class);
    Set<String> hosts = new TreeSet<String>();
    hosts.add("host1");
    hosts.add("host2");
    hosts.add("host3");
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(hosts, null);


    TezTaskID taskID = TezTaskID.getInstance(
        TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
        mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),

View Full Code Here


    EventHandler eventHandler = mock(EventHandler.class);
    String hosts[] = new String[] { "127.0.0.1", "host2", "host3" };
    Set<String> resolved = new TreeSet<String>(
        Arrays.asList(new String[]{ "host1", "host2", "host3" }));
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(
        new TreeSet<String>(Arrays.asList(hosts)), null);


    TezTaskID taskID = TezTaskID.getInstance(
        TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,

View Full Code Here


    Configuration taskConf = new Configuration();
    taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
    taskConf.setBoolean("fs.file.impl.disable.cache", true);


    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(
        new HashSet<String>(Arrays.asList(new String[]{"127.0.0.1"})), null);
    Resource resource = Resource.newInstance(1024, 1);


    AppContext mockAppContext = mock(AppContext.class);
    doReturn(new ClusterInfo()).when(mockAppContext).getClusterInfo();

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.dag.api.TaskLocationHint

org.apache.tez.dag.app.dag.impl.TaskAttemptImpl$ScheduleTaskattemptTransition

org.apache.tez.dag.app.dag.impl.TestTaskAttempt

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.dag.app.dag.impl.VertexImpl

org.apache.tez.dag.app.rm.TaskSchedulerEventHandler

org.apache.tez.dag.app.rm.TestContainerReuse

org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager

org.apache.tez.mapreduce.client.YARNRunner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.