Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MiniMRCluster$JobTrackerRunner

127
128
129
130
131
132
133
134
135
136
137
      e.printStackTrace();
      fail("Fail to startup HDFS cluster.");     
    }
    // Startup MR Cluster
    try {
      mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri()
          .toString(), 1);
    } catch(Exception e) {
      fail("Fail to startup Map/reduce cluster.");
    }
    // Startup collector
View Full Code Here
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    ////

    // Allow the user to override FS URI for this map-reduce cluster to use.
    mrCluster = new MiniMRCluster(servers,
      FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
      null, null, new JobConf(this.conf));
    JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
    if (jobConf == null) {
      jobConf = mrCluster.createJobConf();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
      // Builds and starts the mini dfs and mapreduce clusters
      System.setProperty("hadoop.log.dir", ".");
      m_dfs = new MiniDFSCluster(config, dataNodes, true, null);

      m_fileSys = m_dfs.getFileSystem();
      m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);

      // Create the configuration hadoop-site.xml file
      File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
      conf_dir.mkdirs();
      File conf_file = new File(conf_dir, "hadoop-site.xml");
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
    qSkipSet = new HashSet<String>();

    if (miniMr) {
      dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
      FileSystem fs = dfs.getFileSystem();
      mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
    }

    initConf();

    testFiles = conf.get("test.data.files").replace('\\', '/')
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
            Configuration config = new Configuration();
           
            // Builds and starts the mini dfs and mapreduce clusters
            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
            m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);
           
            // Create the configuration hadoop-site.xml file
            File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
            conf_dir.mkdirs();
            File conf_file = new File(conf_dir, "hadoop-site.xml");
View Full Code Here
692
693
694
695
696
697
698
699
700
701
702
            fileSystem.setPermission(new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------"));
            String nnURI = fileSystem.getUri().toString();
            int numDirs = 1;
            String[] racks = null;
            String[] hosts = null;
            mrCluster = new MiniMRCluster(0, 0, taskTrackers, nnURI, numDirs, racks, hosts, null, conf);
            JobConf jobConf = mrCluster.createJobConf();
            System.setProperty(OOZIE_TEST_JOB_TRACKER, jobConf.get("mapred.job.tracker"));
            System.setProperty(OOZIE_TEST_NAME_NODE, jobConf.get("fs.default.name"));
            ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
            Runtime.getRuntime().addShutdownHook(new Thread() {
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    conf.set("yarn.scheduler.capacity.root.queues", "default");
    conf.set("yarn.scheduler.capacity.root.default.capacity", "100");

    FileSystem fs = FileSystem.get(conf);
    System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
      new JobConf(conf));
    mrConf = mrCluster.createJobConf();
    fs.mkdirs(warehousedir);

    initializeSetup();
View Full Code Here
80
81
82
83
84
85
86
87
88
    fs = FileSystem.get(conf);
    System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
    // LocalJobRunner does not work with mapreduce OutputCommitter. So need
    // to use MiniMRCluster. MAPREDUCE-2350
    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
      new JobConf(conf));
    mrConf = mrCluster.createJobConf();
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    fs = FileSystem.get(conf);
    System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
        .toString(), "/logs").getAbsolutePath());
    // LocalJobRunner does not work with mapreduce OutputCommitter. So need
    // to use MiniMRCluster. MAPREDUCE-2350
    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
        new JobConf(conf));
    mrConf = mrCluster.createJobConf();

    if (isServerRunning) {
      return;
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
      // Builds and starts the mini dfs and mapreduce clusters
      System.setProperty("hadoop.log.dir", ".");
      m_dfs = new MiniDFSCluster(config, dataNodes, true, null);

      m_fileSys = m_dfs.getFileSystem();
      m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);

      // Create the configuration hadoop-site.xml file
      File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
      conf_dir.mkdirs();
      File conf_file = new File(conf_dir, "hadoop-site.xml");
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.MiniMRCluster$JobTrackerRunner

  • org.apache.blur.mapreduce.lib.BlurOutputFormatTest
  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.extraction.demux.TestDemux
  • org.apache.hadoop.cli.TestMRCLI
  • org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy
  • org.apache.hadoop.contrib.index.mapred.TestIndexUpdater
  • org.apache.hadoop.filecache.TestMRWithDistributedCache
  • org.apache.hadoop.fs.TestCopyFiles
  • org.apache.hadoop.fs.TestHarFileSystem
  • org.apache.hadoop.hbase.HBaseTestingUtility
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.