Package org.apache.hadoop.mapreduce.v2

Examples of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

331
332
333
334
335
336
337
338
339
340
341
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
26
27
28
29
30
31
32
33
34
     System.setProperty("hadoop.log.dir", "/tmp/logs");
     YarnConfiguration clusterConf = new YarnConfiguration();
     clusterConf.addResource(dfsConf);
     clusterConf.set("yarn.nodemanager.vmem-check-enabled", "false");
     clusterConf.set("yarn.nodemanager.pmem-check-enabled", "false");
    cluster = new MiniMRYarnCluster("simpleprocessor");
    cluster.init(clusterConf);
    cluster.start();
  }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    builder.numDataNodes(numSlaves);
    builder.format(true);
    builder.racks(null);
    dfsCluster = builder.build();
   
    mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs);
    mrCluster.init(conf);
    mrCluster.start();

    NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads();
   
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);

    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier,
        noOfNMs);
    job.getConfiguration().set("minimrclientcluster.caller.name",
        identifier);
    job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
        noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
        this.getClass().getName());
    int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1);
    oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    stop();
    miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs);
    miniMRYarnCluster.init(oldConf);
    miniMRYarnCluster.start();
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    fs = FileSystem.get(cconf);
    Assert.assertTrue(fs instanceof DistributedFileSystem);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;

    JobClient client = null;
    MiniMRYarnCluster miniMRYarnCluster = null;
    try {
      // This will test RPC to the NameNode only.
      // could we test Client-DataNode connections?
      Path filePath = new Path("/dir");

      Assert.assertFalse(directDfs.exists(filePath));
      Assert.assertFalse(dfs.exists(filePath));

      directDfs.mkdirs(filePath);
      Assert.assertTrue(directDfs.exists(filePath));
      Assert.assertTrue(dfs.exists(filePath));

      // This will test RPC to a Resource Manager
      fs = FileSystem.get(sconf);
      JobConf jobConf = new JobConf();
      FileSystem.setDefaultUri(jobConf, fs.getUri().toString());
      miniMRYarnCluster = initAndStartMiniMRYarnCluster(jobConf);
      JobConf jconf = new JobConf(miniMRYarnCluster.getConfig());
      jconf.set("hadoop.rpc.socket.factory.class.default",
                "org.apache.hadoop.ipc.DummySocketFactory");
      jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
      String rmAddress = jconf.get("yarn.resourcemanager.address");
      String[] split = rmAddress.split(":");
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
      shutdownDFSCluster(cluster);
    }
  }

  private MiniMRYarnCluster initAndStartMiniMRYarnCluster(JobConf jobConf) {
    MiniMRYarnCluster miniMRYarnCluster;
    miniMRYarnCluster = new MiniMRYarnCluster(this.getClass().getName(), 1);
    miniMRYarnCluster.init(jobConf);
    miniMRYarnCluster.start();
    return miniMRYarnCluster;
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
        this.getClass().getName());
    int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1);
    oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    stop();
    miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs);
    miniMRYarnCluster.init(oldConf);
    miniMRYarnCluster.start();
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);

    MiniMRYarnCluster miniMRYarnCluster =
        new MiniMRYarnCluster(identifier, noOfNMs);
    job.getConfiguration().set("minimrclientcluster.caller.name",
        identifier);
    job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
        noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileContext
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.ipc.TestMRCJCSocketFactory
  • org.apache.hadoop.ipc.TestSocketFactory
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.MiniMRClientClusterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.