Examples of AllocatedSlot


Examples of org.apache.flink.runtime.instance.AllocatedSlot

        ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0]);
        setVertexState(vertex, ExecutionState.CANCELING);
       
        TaskOperationProtocol taskManager = mock(TaskOperationProtocol.class);
        Instance instance = getInstance(taskManager);
        AllocatedSlot slot = instance.allocateSlot(new JobID());
       
        vertex.deployToSlot(slot);
        fail("Method should throw an exception");
      }
      catch (IllegalStateException e) {}
     
     
      // fail while canceling
      {
        ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0]);
       
        TaskOperationProtocol taskManager = mock(TaskOperationProtocol.class);
        Instance instance = getInstance(taskManager);
        AllocatedSlot slot = instance.allocateSlot(new JobID());
       
        setVertexResource(vertex, slot);
        setVertexState(vertex, ExecutionState.CANCELING);
       
        Exception failureCause = new Exception("test exception");
       
        vertex.fail(failureCause);
        assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
        assertEquals(failureCause, vertex.getFailureCause());
       
        assertTrue(slot.isReleased());
      }
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

      scheduler.newInstanceAvailable(getRandomInstance(1));
      scheduler.newInstanceAvailable(getRandomInstance(2));
      assertEquals(5, scheduler.getNumberOfAvailableSlots());
     
      // schedule something into all slots
      AllocatedSlot s1 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
      AllocatedSlot s2 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
      AllocatedSlot s3 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
      AllocatedSlot s4 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
      AllocatedSlot s5 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
     
      // the slots should all be different
      assertTrue(areAllDistinct(s1, s2, s3, s4, s5));
     
      try {
        scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
        fail("Scheduler accepted scheduling request without available resource.");
      }
      catch (NoResourceAvailableException e) {
        // pass!
      }
     
      // release some slots again
      s3.releaseSlot();
      s4.releaseSlot();
      assertEquals(2, scheduler.getNumberOfAvailableSlots());
     
      // now we can schedule some more slots
      AllocatedSlot s6 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
      AllocatedSlot s7 = scheduler.scheduleImmediately(new ScheduledUnit(getDummyTask()));
     
      assertTrue(areAllDistinct(s1, s2, s3, s4, s5, s6, s7));
     
      // release all
     
      s1.releaseSlot();
      s2.releaseSlot();
      s5.releaseSlot();
      s6.releaseSlot();
      s7.releaseSlot();
     
      assertEquals(5, scheduler.getNumberOfAvailableSlots());
     
      // check that slots that are released twice (accidentally) do not mess things up
     
      s1.releaseSlot();
      s2.releaseSlot();
      s5.releaseSlot();
      s6.releaseSlot();
      s7.releaseSlot();
     
      assertEquals(5, scheduler.getNumberOfAvailableSlots());
    }
    catch (Exception e) {
      e.printStackTrace();
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

                while (toRelease.isEmpty()) {
                  toRelease.wait();
                }
               
                Iterator<AllocatedSlot> iter = toRelease.iterator();
                AllocatedSlot next = iter.next();
                iter.remove();
               
                next.releaseSlot();
                recycled++;
              }
            }
          }
          catch (Throwable t) {
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

      scheduler.newInstanceAvailable(i1);
      scheduler.newInstanceAvailable(i2);
      scheduler.newInstanceAvailable(i3);
     
      // schedule something on an arbitrary instance
      AllocatedSlot s1 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Collections.<Instance>emptyList())));
     
      // figure out how we use the location hints
      Instance first = s1.getInstance();
      Instance second = first != i1 ? i1 : i2;
      Instance third = first == i3 ? i2 : i3;
     
      // something that needs to go to the first instance again
      AllocatedSlot s2 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Collections.singletonList(s1.getInstance()))));
      assertEquals(first, s2.getInstance());

      // first or second --> second, because first is full
      AllocatedSlot s3 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Arrays.asList(first, second))));
      assertEquals(second, s3.getInstance());
     
      // first or third --> third (because first is full)
      AllocatedSlot s4 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Arrays.asList(first, third))));
      AllocatedSlot s5 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Arrays.asList(first, third))));
      assertEquals(third, s4.getInstance());
      assertEquals(third, s5.getInstance());
     
      // first or third --> second, because all others are full
      AllocatedSlot s6 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Arrays.asList(first, third))));
      assertEquals(second, s6.getInstance());
     
      // release something on the first and second instance
      s2.releaseSlot();
      s6.releaseSlot();
     
      AllocatedSlot s7 = scheduler.scheduleImmediately(new ScheduledUnit(getTestVertex(Arrays.asList(first, third))));
      assertEquals(first, s7.getInstance());
     
      assertEquals(1, scheduler.getNumberOfUnconstrainedAssignments());
      assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
      assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
    }
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

        }
        else {
          slotFromGroup = assignment.getSlotForTask(vertex, constraint);
        }
       
        AllocatedSlot newSlot = null;
       
        // the following needs to make sure any allocated slot is released in case of an error
        try {
         
          // check whether the slot from the group is already what we want
          if (slotFromGroup != null) {
            // local (or unconstrained in the current group)
            if (slotFromGroup.getLocality() != Locality.NON_LOCAL) {
              updateLocalityCounters(slotFromGroup.getLocality());
              return slotFromGroup;
            }
          }
         
          final Iterable<Instance> locations = (constraint == null || constraint.isUnassigned()) ?
              vertex.getPreferredLocations() : Collections.singleton(constraint.getLocation());
         
          // get a new slot, since we could not place it into the group, or we could not place it locally
          newSlot = getFreeSlotForTask(vertex, locations);
         
          SubSlot toUse;
         
          if (newSlot == null) {
            if (slotFromGroup == null) {
              // both null
              if (constraint == null || constraint.isUnassigned()) {
                throw new NoResourceAvailableException(getNumberOfAvailableInstances(), getTotalNumberOfSlots());
              } else {
                throw new NoResourceAvailableException("Could not allocate a slot on instance " +
                      constraint.getLocation() + ", as required by the co-location constraint.");
              }
            } else {
              // got a non-local from the group, and no new one
              toUse = slotFromGroup;
            }
          }
          else if (slotFromGroup == null || newSlot.getLocality() == Locality.LOCAL) {
            // new slot is preferable
            if (slotFromGroup != null) {
              slotFromGroup.releaseSlot();
            }
           
            if (constraint == null) {
              toUse = assignment.addNewSlotWithTask(newSlot, vertex);
            } else {
              toUse = assignment.addNewSlotWithTask(newSlot, vertex, constraint);
            }
          }
          else {
            // both are available and usable. neither is local
            newSlot.releaseSlot();
            toUse = slotFromGroup;
          }
         
          // assign to the co-location hint, if we have one and it is unassigned
          // if it was assigned before and the new one is not local, it is a fail
          if (constraint != null) {
            if (constraint.isUnassigned() || toUse.getLocality() == Locality.LOCAL) {
              constraint.setSharedSlot(toUse.getSharedSlot());
            } else {
              // the fail
              throw new NoResourceAvailableException("Could not allocate a slot on instance " +
                  constraint.getLocation() + ", as required by the co-location constraint.");
            }
          }
         
          updateLocalityCounters(toUse.getLocality());
          return toUse;
        }
        catch (NoResourceAvailableException e) {
          throw e;
        }
        catch (Throwable t) {
          if (slotFromGroup != null) {
            slotFromGroup.releaseSlot();
          }
          if (newSlot != null) {
            newSlot.releaseSlot();
          }
         
          ExceptionUtils.rethrow(t, "An error occurred while allocating a slot in a sharing group");
        }
      }
   
      // 2) === schedule without hints and sharing ===
     
      AllocatedSlot slot = getFreeSlotForTask(vertex, vertex.getPreferredLocations());
      if (slot != null) {
        updateLocalityCounters(slot.getLocality());
        return slot;
      }
      else {
        // no resource available now, so queue the request
        if (queueIfNoResource) {
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

          LOG.debug("Unconstrained assignment: " + vertex.getSimpleName() + " --> " + instanceToUse);
        }
      }
     
      try {
        AllocatedSlot slot = instanceToUse.allocateSlot(vertex.getJobId());
       
        // if the instance has further available slots, re-add it to the set of available resources.
        if (instanceToUse.hasResourcesAvailable()) {
          this.instancesWithAvailableResources.add(instanceToUse);
        }
       
        if (slot != null) {
          slot.setLocality(locality);
          return slot;
        }
      }
      catch (InstanceDiedException e) {
        // the instance died it has not yet been propagated to this scheduler
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

      if (queued != null) {
        ScheduledUnit task = queued.getTask();
        ExecutionVertex vertex = task.getTaskToExecute().getVertex();
       
        try {
          AllocatedSlot newSlot = instance.allocateSlot(vertex.getJobId());
          if (newSlot != null) {
           
            // success, remove from the task queue and notify the future
            taskQueue.poll();
            if (queued.getFuture() != null) {
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

        fail();
      } catch (IllegalStateException e) {
        // expected
      }
     
      final AllocatedSlot slot1 = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
      final AllocatedSlot slot2 = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
     
      future.setSlot(slot1);
      try {
        future.setSlot(slot2);
        fail();
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

    try {
     
      // action before the slot
      {
        final AtomicInteger invocations = new AtomicInteger();
        final AllocatedSlot thisSlot = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
       
        SlotAllocationFuture future = new SlotAllocationFuture();
       
        future.setFutureAction(new SlotAllocationFutureAction() {
          @Override
          public void slotAllocated(AllocatedSlot slot) {
            assertEquals(thisSlot, slot);
            invocations.incrementAndGet();
          }
        });
       
        future.setSlot(thisSlot);
       
        assertEquals(1, invocations.get());
      }
     
      // slot before action
      {
        final AtomicInteger invocations = new AtomicInteger();
        final AllocatedSlot thisSlot = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
       
        SlotAllocationFuture future = new SlotAllocationFuture();
        future.setSlot(thisSlot);
       
        future.setFutureAction(new SlotAllocationFutureAction() {
View Full Code Here

Examples of org.apache.flink.runtime.instance.AllocatedSlot

      // sync before setting the slot
      {
        final AtomicInteger invocations = new AtomicInteger();
        final AtomicBoolean error = new AtomicBoolean();
       
        final AllocatedSlot thisSlot = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
       
        final SlotAllocationFuture future = new SlotAllocationFuture();
       
       
        Runnable r = new Runnable() {
          @Override
          public void run() {
            try {
              AllocatedSlot syncSlot = future.waitTillAllocated();
              if (syncSlot == null || syncSlot != thisSlot) {
                error.set(true);
                return;
              }
              invocations.incrementAndGet();
            }
            catch (Throwable t) {
              error.set(true);
            }
          }
        };
       
        Thread syncer = new Thread(r);
        syncer.start();
       
        // wait, and give the sync thread a chance to sync
        Thread.sleep(10);
        future.setSlot(thisSlot);
       
        syncer.join();
       
        assertFalse(error.get());
        assertEquals(1, invocations.get());
      }
     
      // setting slot before syncing
      {
        final AllocatedSlot thisSlot = new AllocatedSlot(new JobID(), SchedulerTestUtils.getRandomInstance(1), 0);
        final SlotAllocationFuture future = new SlotAllocationFuture();

        future.setSlot(thisSlot);
       
        AllocatedSlot retrieved = future.waitTillAllocated();
       
        assertNotNull(retrieved);
        assertEquals(thisSlot, retrieved);
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.