Package org.apache.flink.core.io

Examples of org.apache.flink.core.io.LocatableInputSplit


    if (host == null) {
     
      synchronized (this.unassigned) {
        Iterator<LocatableInputSplit> iter = this.unassigned.iterator();
        if (iter.hasNext()) {
          LocatableInputSplit next = iter.next();
          iter.remove();
         
          if (LOG.isInfoEnabled()) {
            LOG.info("Assigning split to null host (random assignment).");
          }
         
          remoteAssignments++;
          return next;
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug("No more unassigned input splits remaining.");
          }
          return null;
        }
      }
    }
   
    host = host.toLowerCase(Locale.US);
   
    // for any non-null host, we take the list of non-null splits
    List<LocatableInputSplit> localSplits = this.localPerHost.get(host);
   
    // if we have no list for this host yet, create one
    if (localSplits == null) {
      localSplits = new ArrayList<LocatableInputSplit>(16);
     
      // lock the list, to be sure that others have to wait for that host's local list
      synchronized (localSplits) {
        List<LocatableInputSplit> prior = this.localPerHost.putIfAbsent(host, localSplits);
       
        // if someone else beat us in the case to create this list, then we do not populate this one, but
        // simply work with that other list
        if (prior == null) {
          // we are the first, we populate
         
          // first, copy the remaining splits to release the lock on the set early
          // because that is shared among threads
          LocatableInputSplit[] remaining;
          synchronized (this.unassigned) {
            remaining = (LocatableInputSplit[]) this.unassigned.toArray(new LocatableInputSplit[this.unassigned.size()]);
          }
         
          for (LocatableInputSplit is : remaining) {
            if (isLocal(host, is.getHostnames())) {
              localSplits.add(is);
            }
          }
        }
        else {
          // someone else was faster
          localSplits = prior;
        }
      }
    }
   
   
    // at this point, we have a list of local splits (possibly empty)
    // we need to make sure no one else operates in the current list (that protects against
    // list creation races) and that the unassigned set is consistent
    // NOTE: we need to obtain the locks in this order, strictly!!!
    synchronized (localSplits) {
      int size = localSplits.size();
      if (size > 0) {
        synchronized (this.unassigned) {
          do {
            --size;
            LocatableInputSplit split = localSplits.remove(size);
            if (this.unassigned.remove(split)) {
             
              if (LOG.isInfoEnabled()) {
                LOG.info("Assigning local split to host " + host);
              }
             
              localAssignments++;
              return split;
            }
          } while (size > 0);
        }
      }
    }
   
    // we did not find a local split, return any
    synchronized (this.unassigned) {
      Iterator<LocatableInputSplit> iter = this.unassigned.iterator();
      if (iter.hasNext()) {
        LocatableInputSplit next = iter.next();
        iter.remove();
       
        if (LOG.isInfoEnabled()) {
          LOG.info("Assigning remote split to host " + host);
        }
View Full Code Here


      };
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, hosts[i%3]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
View Full Code Here

      final int NUM_SPLITS = 50;
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, "testhost"));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
View Full Code Here

      final int NUM_SPLITS = 10 * hosts.length;
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
View Full Code Here

      final int NUM_SPLITS = 10 * hosts.length;
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
View Full Code Here

      };
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, hosts[i%3]));
      }
     
      final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
     
      final AtomicInteger splitsRetrieved = new AtomicInteger(0);
      final AtomicInteger sumOfIds = new AtomicInteger(0);
     
      Runnable retriever = new Runnable() {
       
        @Override
        public void run() {
          LocatableInputSplit split;
          while ((split = ia.getNextInputSplit(null)) != null) {
            splitsRetrieved.incrementAndGet();
            sumOfIds.addAndGet(split.getSplitNumber());
          }
        }
      };
     
      // create the threads
View Full Code Here

      final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, "testhost"));
      }
     
      final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
     
      final AtomicInteger splitsRetrieved = new AtomicInteger(0);
      final AtomicInteger sumOfIds = new AtomicInteger(0);
     
      Runnable retriever = new Runnable() {
       
        @Override
        public void run() {
          LocatableInputSplit split;
          while ((split = ia.getNextInputSplit("testhost")) != null) {
            splitsRetrieved.incrementAndGet();
            sumOfIds.addAndGet(split.getSplitNumber());
          }
        }
      };
     
      // create the threads
View Full Code Here

      final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
     
      // load some splits
      Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new LocatableInputSplit(i, hosts[i%hosts.length]));
      }
     
      final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
     
      final AtomicInteger splitsRetrieved = new AtomicInteger(0);
      final AtomicInteger sumOfIds = new AtomicInteger(0);
     
      Runnable retriever = new Runnable() {
       
        @Override
        public void run() {
          final String threadHost = hosts[(int) (Math.random() * hosts.length)];
         
          LocatableInputSplit split;
          while ((split = ia.getNextInputSplit(threadHost)) != null) {
            splitsRetrieved.incrementAndGet();
            sumOfIds.addAndGet(split.getSplitNumber());
          }
        }
      };
     
      // create the threads
View Full Code Here

TOP

Related Classes of org.apache.flink.core.io.LocatableInputSplit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.