Examples of eu.stratosphere.nephele.instance.InstanceRequestMap

Package eu.stratosphere.nephele.instance

Examples of eu.stratosphere.nephele.instance.InstanceRequestMap

eu.stratosphere.nephele.instance.InstanceRequestMap
An instance request map specifies the required types of instances to run a specific job and the respective number thereof. For each instance type it is possible to specify the minimum number of instances required to run the job. If the {@link InstanceManager} cannot manage to provide at least this minimum numbers of instances for the given type,the job will be rejected.
In addition, is it also possible to specify the optimal number of instances for a particular instance type. The {@link InstanceManager} will try to provide this optimal number of instances, but will also start the job with lessinstances.
This class is not thread-safe.

      LibraryCacheManager.register(jobID, new String[0]);


      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);


      // test all methods of ExecutionGraph
      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      final ExecutionStage executionStage = eg.getCurrentExecutionStage();
      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);
      assertEquals(1, instanceRequestMap.size());
      assertEquals(1, (int) instanceRequestMap.getMaximumNumberOfInstances(INSTANCE_MANAGER
          .getInstanceTypeByName(DEFAULT_INSTANCE_TYPE_NAME)));


      assertEquals(jobID, eg.getJobID());
      assertEquals(0, eg.getIndexOfCurrentExecutionStage());
      assertEquals(1, eg.getNumberOfInputVertices());

View Full Code Here


      // now convert job graph to execution graph
      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);


      // test instance types in ExecutionGraph
      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      final ExecutionStage executionStage = eg.getCurrentExecutionStage();
      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);
      assertEquals(1, instanceRequestMap.size());
      assertEquals(1,
        (int) instanceRequestMap.getMaximumNumberOfInstances(INSTANCE_MANAGER.getDefaultInstanceType()));


      // stage0
      ExecutionStage es = eg.getStage(0);
      ExecutionGroupVertex egv0 = null; // input1
      ExecutionGroupVertex egv1 = null; // output1

View Full Code Here

      LibraryCacheManager.register(jobID, new String[0]);


      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);


      // test instance types in ExecutionGraph
      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      final ExecutionStage executionStage = eg.getCurrentExecutionStage();
      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);
      assertEquals(1, instanceRequestMap.size());
      assertEquals(2,
        (int) instanceRequestMap.getMaximumNumberOfInstances(INSTANCE_MANAGER.getDefaultInstanceType()));


      // stage0
      final ExecutionStage es = eg.getStage(0);
      ExecutionGroupVertex egv0 = null; // input1
      ExecutionGroupVertex egv1 = null; // input2

View Full Code Here


      // now convert job graph to execution graph
      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);


      // test instance types in ExecutionGraph
      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      ExecutionStage executionStage = eg.getCurrentExecutionStage();
      assertNotNull(executionStage);
      assertEquals(0, executionStage.getStageNumber());
      
      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);
      assertEquals(1, instanceRequestMap.size());
      assertEquals(8,
        (int) instanceRequestMap.getMaximumNumberOfInstances(INSTANCE_MANAGER
          .getInstanceTypeByName(DEFAULT_INSTANCE_TYPE_NAME)));
      // Fake transition to next stage by triggering execution state changes manually
      final Iterator<ExecutionVertex> it = new ExecutionGraphIterator(eg, eg.getIndexOfCurrentExecutionStage(),
        true, true);


      while (it.hasNext()) {
        final ExecutionVertex ev = it.next();
        ev.updateExecutionState(ExecutionState.SCHEDULED);
        ev.updateExecutionState(ExecutionState.ASSIGNED);
        ev.updateExecutionState(ExecutionState.READY);
        ev.updateExecutionState(ExecutionState.STARTING);
        ev.updateExecutionState(ExecutionState.RUNNING);
        ev.updateExecutionState(ExecutionState.FINISHING);
        ev.updateExecutionState(ExecutionState.FINISHED);
      }
      instanceRequestMap.clear();
    } catch (GraphConversionException e) {
      fail(e.getMessage());
    } catch (JobGraphDefinitionException e) {
      fail(e.getMessage());
    } catch (IOException e) {

View Full Code Here

      // Check number of vertices in stage
      final ExecutionStage stage = eg.getStage(0);
      assertEquals(5, stage.getNumberOfStageMembers());


      // Check number of required instances
      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      stage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);


      // First, we expect all required instances to be of the same type
      assertEquals(1, instanceRequestMap.size());


      final int numberOfRequiredInstances = instanceRequestMap.getMinimumNumberOfInstances(INSTANCE_MANAGER
        .getDefaultInstanceType());
      assertEquals(degreeOfParallelism, numberOfRequiredInstances);


    } catch (GraphConversionException e) {
      fail(e.getMessage());

View Full Code Here

      
      
      // allocate something
      JobID jobID = new JobID();
      Configuration conf = new Configuration();
      InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      instanceRequestMap.setNumberOfInstances(cm.getDefaultInstanceType(), 2);
      cm.requestInstance(jobID, conf, instanceRequestMap, null);
      
      ClusterManagerTestUtils.waitForInstances(jobID, testInstanceListener, 3, 1000);
      
      List<AllocatedResource> allocatedResources = testInstanceListener.getAllocatedResourcesForJob(jobID);
      assertEquals(2, allocatedResources.size());
      
      Iterator<AllocatedResource> it = allocatedResources.iterator();
      Set<AllocationID> allocationIDs = new HashSet<AllocationID>();
      while (it.hasNext()) {
        AllocatedResource allocatedResource = it.next();
        if (ConfigConstants.DEFAULT_INSTANCE_TYPE.equals(allocatedResource.getInstance().getType().getIdentifier())) {
          fail("Allocated unexpected instance of type "
            + allocatedResource.getInstance().getType().getIdentifier());
        }


        if (allocationIDs.contains(allocatedResource.getAllocationID())) {
          fail("Discovered allocation ID " + allocatedResource.getAllocationID() + " at least twice");
        } else {
          allocationIDs.add(allocatedResource.getAllocationID());
        }
      }


      // Try to allocate more resources which must result in an error
      try {
        InstanceRequestMap instancem = new InstanceRequestMap();
        instancem.setNumberOfInstances(cm.getDefaultInstanceType(), 1);
        cm.requestInstance(jobID, conf, instancem, null);


        fail("ClusterManager allowed to request more instances than actually available");


      } catch (InstanceException ie) {
        // Exception is expected and correct behavior here
      }


      // Release all allocated resources
      it = allocatedResources.iterator();
      while (it.hasNext()) {
        final AllocatedResource allocatedResource = it.next();
        cm.releaseAllocatedResource(jobID, conf, allocatedResource);
      }
      
      // Now further allocations should be possible
      
      InstanceRequestMap instancem = new InstanceRequestMap();
      instancem.setNumberOfInstances(cm.getDefaultInstanceType(), 1);
      cm.requestInstance(jobID, conf, instancem, null);
      
      
      cm.shutdown();
    }

View Full Code Here

      
      // request some instances
      JobID jobID = new JobID();
      Configuration conf = new Configuration();


      InstanceRequestMap instancem = new InstanceRequestMap();
      instancem.setNumberOfInstances(cm.getDefaultInstanceType(), 1);
      cm.requestInstance(jobID, conf, instancem, null);
      
      ClusterManagerTestUtils.waitForInstances(jobID, testInstanceListener, 1, 1000);
      assertEquals(1, testInstanceListener.getNumberOfAllocatedResourcesForJob(jobID));

View Full Code Here

        .getMapOfAvailableInstanceTypes();


    final Iterator<ExecutionStage> stageIt = executionGraph.iterator();
    while (stageIt.hasNext()) {


      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      final ExecutionStage stage = stageIt.next();
      stage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);


      // Iterator over required Instances
      final Iterator<Map.Entry<InstanceType, Integer>> it = instanceRequestMap.getMinimumIterator();
      while (it.hasNext()) {


        final Map.Entry<InstanceType, Integer> entry = it.next();


        final InstanceTypeDescription descr = availableInstances.get(entry.getKey());

View Full Code Here

        .getMapOfAvailableInstanceTypes();


    final Iterator<ExecutionStage> stageIt = executionGraph.iterator();
    while (stageIt.hasNext()) {


      final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      final ExecutionStage stage = stageIt.next();
      stage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);


      // Iterator over required Instances
      final Iterator<Map.Entry<InstanceType, Integer>> it = instanceRequestMap.getMinimumIterator();
      while (it.hasNext()) {


        final Map.Entry<InstanceType, Integer> entry = it.next();


        final InstanceTypeDescription descr = availableInstances.get(entry.getKey());

View Full Code Here

   *         thrown if the given execution graph is already processing its final stage
   */
  protected void requestInstances(final ExecutionStage executionStage) throws InstanceException {


    final ExecutionGraph executionGraph = executionStage.getExecutionGraph();
    final InstanceRequestMap instanceRequestMap = new InstanceRequestMap();


    synchronized (executionStage) {


      executionStage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);


      final Iterator<Map.Entry<InstanceType, Integer>> it = instanceRequestMap.getMinimumIterator();
      LOG.info("Requesting the following instances for job " + executionGraph.getJobID());
      while (it.hasNext()) {
        final Map.Entry<InstanceType, Integer> entry = it.next();
        LOG.info(" " + entry.getKey() + " [" + entry.getValue().intValue() + ", "
          + instanceRequestMap.getMaximumNumberOfInstances(entry.getKey()) + "]");
      }


      if (instanceRequestMap.isEmpty()) {
        return;
      }


      this.instanceManager.requestInstance(executionGraph.getJobID(), executionGraph.getJobConfiguration(),
        instanceRequestMap, null);

View Full Code Here

TOP

Related Classes of eu.stratosphere.nephele.instance.InstanceRequestMap

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

eu.stratosphere.nephele.instance.cluster.ClusterManagerTest

eu.stratosphere.nephele.jobmanager.scheduler.AbstractScheduler

eu.stratosphere.nephele.jobmanager.scheduler.local.LocalScheduler

eu.stratosphere.nephele.jobmanager.scheduler.queue.QueueScheduler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.