Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

Package org.apache.tez.runtime.library.shuffle.common

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

org.apache.tez.runtime.library.shuffle.common.InputHost
Represents a Host with respect to the MapReduce ShuffleHandler. srcPhysicalIndex / partition is part of this since that only knows how to serve ine partition at a time.

          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - numRunningFetchers.get();
            int count = 0;
            while (pendingHosts.peek() != null) {
              InputHost inputHost = null;
              try {
                inputHost = pendingHosts.take();
              } catch (InterruptedException e) {
                if (isShutdown.get()) {
                  LOG.info("Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                  break;
                } else {
                  throw e;
                }
              }
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier());
                Fetcher fetcher = constructFetcherForHost(inputHost);
                numRunningFetchers.incrementAndGet();
                if (isShutdown.get()) {
                  LOG.info("hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                }
                ListenableFuture<FetchResult> future = fetcherExecutor
                    .submit(fetcher);
                Futures.addCallback(future, fetchFutureCallback);
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Skipping host: " + inputHost.getIdentifier()
                      + " since it has no inputs to process");
                }
              }
            }
          } finally {

View Full Code Here

    
    @Override
    public void onSuccess(FetchResult result) {
      Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
      if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
        InputHost inputHost = knownSrcHosts.get(InputHost.createIdentifier(result.getHost(), result.getPort()));
        assert inputHost != null;
        for (InputAttemptIdentifier input : pendingInputs) {
          inputHost.addKnownInput(input);
        }
        pendingHosts.add(inputHost);
      }
      doBookKeepingForFetcherComplete();
    }

View Full Code Here

  
  /////////////////// Methods for InputEventHandler
  
  public void addKnownInput(String hostName, int port,
      InputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex) {
    InputHost host = knownSrcHosts.get(hostName);
    if (host == null) {
      host = new InputHost(hostName, port, inputContext.getApplicationId(), srcPhysicalIndex);
      InputHost old = knownSrcHosts.putIfAbsent(hostName, host);
      if (old != null) {
        host = old;
      }
    }
    if (LOG.isDebugEnabled()) {

View Full Code Here

          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - numRunningFetchers.get();
            int count = 0;
            while (pendingHosts.peek() != null) {
              InputHost inputHost = null;
              try {
                inputHost = pendingHosts.take();
              } catch (InterruptedException e) {
                if (isShutdown.get()) {
                  LOG.info("Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                  break;
                } else {
                  throw e;
                }
              }
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getHost());
                Fetcher fetcher = constructFetcherForHost(inputHost);
                numRunningFetchers.incrementAndGet();
                if (isShutdown.get()) {
                  LOG.info("hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                }
                ListenableFuture<FetchResult> future = fetcherExecutor
                    .submit(fetcher);
                Futures.addCallback(future, fetchFutureCallback);
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Skipping host: " + inputHost.getHost()
                      + " since it has no inputs to process");
                }
              }
            }
          } finally {

View Full Code Here

    
    @Override
    public void onSuccess(FetchResult result) {
      Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
      if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
        InputHost inputHost = knownSrcHosts.get(result.getHost());
        assert inputHost != null;
        for (InputAttemptIdentifier input : pendingInputs) {
          inputHost.addKnownInput(input);
        }
        pendingHosts.add(inputHost);
      }
      doBookKeepingForFetcherComplete();
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.runtime.library.shuffle.common.InputHost

org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleManager

org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleManager$FetchFutureCallback

org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleManager$RunBroadcastShuffleCallable

org.apache.tez.runtime.library.shuffle.common.impl.ShuffleManager

org.apache.tez.runtime.library.shuffle.common.impl.ShuffleManager$FetchFutureCallback

org.apache.tez.runtime.library.shuffle.common.impl.ShuffleManager$RunShuffleCallable

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.