Examples of org.apache.hadoop.yarn.server.MiniYARNCluster

org.apache.hadoop.yarn.server.MiniYARNCluster
Embedded Yarn minicluster for testcases that need to interact with a cluster.
In a real cluster, resource request matching is done using the hostname, and by default Yarn minicluster works in the exact same way as a real cluster.
If a testcase needs to use multiple nodes and exercise resource request matching to a specific node, then the property {@YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} should be settrue in the configuration used to initialize the minicluster.
With this property set to true, the matching will be done using the hostname:port of the namenodes. In such case, the AM must do resource request using hostname:port as the location.

    LOG.info("Starting up YARN cluster");
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, 
        FifoScheduler.class, ResourceScheduler.class);
    if (yarnCluster == null) {
      yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getName(),
          1, 1, 1);
      yarnCluster.init(conf);
      yarnCluster.start();
      URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
      if (url == null) {

View Full Code Here

    LOG.info("Starting up YARN cluster");
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, 
        FifoScheduler.class, ResourceScheduler.class);
    if (yarnCluster == null) {
      yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getName(),
          1, 1, 1);
      yarnCluster.init(conf);
      yarnCluster.start();
      URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
      if (url == null) {

View Full Code Here

  
  @Before
  public void setup() throws YarnRemoteException {
    // start minicluster
    conf = new YarnConfiguration();
    yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
    yarnCluster.init(conf);
    yarnCluster.start();


    // start rm client
    yarnClient = new YarnClientImpl();

View Full Code Here

    conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);


    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);


    cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
  }

View Full Code Here

  }


  @Test
  public void testRMWebAppRedirect() throws YarnException,
      InterruptedException, IOException {
    cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);


    cluster.init(conf);
    cluster.start();
    getAdminService(0).transitionToActive(req);

View Full Code Here

    config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
    config.set("yarn.nodemanager.vmem-check-enabled", "false");
    config.set("yarn.scheduler.minimum-allocation-mb", "128");
    config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");


    cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
    cluster.init(config);
    cluster.start();


    runnerService = createTwillRunnerService(folder);
    runnerService.startAndWait();

View Full Code Here

      yarnCluster.stop();
      FileUtil.fullyDelete(localFSDirBase);
      localFSDirBase.mkdirs();
    }
    LOG.info("Starting up YARN cluster");
    yarnCluster = new MiniYARNCluster(TestDiskFailures.class.getName(),
        1, numLocalDirs, numLogDirs);
    yarnCluster.init(conf);
    yarnCluster.start();


    NodeManager nm = yarnCluster.getNodeManager(0);

View Full Code Here


  @BeforeClass
  public static void setup() throws InterruptedException, IOException {
    LOG.info("Starting up YARN cluster");
    if (yarnCluster == null) {
      yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getName());
      yarnCluster.init(conf);
      yarnCluster.start();
    }
    try {
      Thread.sleep(2000);

View Full Code Here

    config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
    config.set("yarn.nodemanager.vmem-check-enabled", "false");
    config.set("yarn.scheduler.minimum-allocation-mb", "128");
    config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");


    cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
    cluster.init(config);
    cluster.start();


    runnerService = createTwillRunnerService(folder);
    runnerService.startAndWait();

View Full Code Here


  /**
   * Initialize the MiniYARNCluster for the integration test.
   */
  private void initYarnCluster() {
    cluster = new MiniYARNCluster(TestYarnJob.class.getName(), 1, 1, 1);
    cluster.init(new ImmutableClassesGiraphConfiguration(conf));
    cluster.start();
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.yarn.server.MiniYARNCluster

com.cloudera.kitten.TestKittenDistributedShell

com.mongodb.hadoop.testutils.BaseHadoopTest

com.yahoo.storm.yarn.TestIntegration

org.apache.giraph.yarn.TestYarnJob

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.Path

org.apache.hadoop.util.Shell.ShellCommandExecutor

org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest

org.apache.hadoop.yarn.applications.distributedshell.TestDistributedShell

org.apache.hadoop.yarn.applications.unmanagedamlauncher.TestUnmanagedAMLauncher

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.