Examples of org.apache.hadoop.mapred.MiniMRCluster

org.apache.hadoop.mapred.MiniMRCluster
This class creates a single-process Map-Reduce cluster for junit testing. One thread is created for each server.

            // Builds and starts the mini dfs and mapreduce clusters
            System.setProperty("hadoop.log.dir", ".");
            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);


            m_fileSys = m_dfs.getFileSystem();
            m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);


            // Create the configuration hadoop-site.xml file
            File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
            conf_dir.mkdirs();
            File conf_file = new File(conf_dir, "hadoop-site.xml");

View Full Code Here

        Configuration conf = new Configuration(true);
        fs = FileSystem.get(conf);
        System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
        // LocalJobRunner does not work with mapreduce OutputCommitter. So need
        // to use MiniMRCluster. MAPREDUCE-2350
        mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
            new JobConf(conf));
        mrConf = mrCluster.createJobConf();
    }

View Full Code Here

    qSkipSet = new HashSet<String>();


    if (miniMr) {
      dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
      FileSystem fs = dfs.getFileSystem();
      mr = new MiniMRCluster(4, getHdfsUriString(fs.getUri().toString()), 1);
    }


    initConf();


    // Use the current directory if it is not specified

View Full Code Here

    if (options.localFS) {
      Path localFSRootPath = new Path(System.getProperty("test.build.data",
          "build/test/data/work-dir"));
      fileSys = localFSRootPath.getFileSystem(conf);
      rootPath = new Path(localFSRootPath, options.rootPath);
      mr = new MiniMRCluster(options.taskTrackers, "file:///", 3);
    } else {
      dfs = new MiniDFSCluster(conf, options.dataNodes, true, null);
      fileSys = dfs.getFileSystem();
      rootPath = new Path(options.rootPath);
      mr = new MiniMRCluster(options.taskTrackers, fileSys.getUri().toString(),
          1);
    }
    conf = getJobConf("TestBasicTableIOFormat");
    srcPath = new Path(rootPath, options.srcPath);
    fwdIndexRootPath = new Path(rootPath, options.fwdIndexRootPath);

View Full Code Here

        fs = FileSystem.get(conf);
        System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
                .toString(), "/logs").getAbsolutePath());
        // LocalJobRunner does not work with mapreduce OutputCommitter. So need
        // to use MiniMRCluster. MAPREDUCE-2350
        mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
                new JobConf(conf));
        mrConf = mrCluster.createJobConf();


        if (isServerRunning) {
            return;

View Full Code Here

        // LocalJobRunner does not work with mapreduce OutputCommitter. So need
        // to use MiniMRCluster. MAPREDUCE-2350
        Configuration conf = new Configuration(true);
        FileSystem fs = FileSystem.get(conf);
        System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
        mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
            new JobConf(conf));
        mrConf = mrCluster.createJobConf();
        fs.mkdirs(warehousedir);


        initializeSetup();

View Full Code Here


            jobConf.setInt("mapred.submit.replication", 1);
            //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history");
            System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());


            mrCluster = new MiniMRCluster(jobTrackerPort,
                taskTrackerPort,
                numTaskTrackers,
                getFileSystem().getUri().toString(),
                numTaskTrackers,
                null,

View Full Code Here

    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    ////


    // Allow the user to override FS URI for this map-reduce cluster to use.
    mrCluster = new MiniMRCluster(servers,
      FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
      null, null, new JobConf(this.conf));
    JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
    if (jobConf == null) {
      jobConf = mrCluster.createJobConf();

View Full Code Here

    LOG.info("Starting mini mapreduce cluster...");
    // These are needed for the new and improved Map/Reduce framework
    Configuration c = getConfiguration();
    System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir"));
    c.set("mapred.output.dir", c.get("hadoop.tmp.dir"));
    mrCluster = new MiniMRCluster(servers,
      FileSystem.get(c).getUri().toString(), 1);
    LOG.info("Mini mapreduce cluster started");
    c.set("mapred.job.tracker",
        mrCluster.createJobConf().get("mapred.job.tracker"));
  }

View Full Code Here

    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    ////


    // Allow the user to override FS URI for this map-reduce cluster to use.
    mrCluster = new MiniMRCluster(servers,
      FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
      null, null, new JobConf(this.conf));
    JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
    if (jobConf == null) {
      jobConf = mrCluster.createJobConf();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.MiniMRCluster

org.apache.blur.mapreduce.lib.BlurOutputFormatTest

org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder

org.apache.hadoop.chukwa.extraction.demux.TestDemux

org.apache.hadoop.cli.TestMRCLI

org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy

org.apache.hadoop.contrib.index.mapred.TestIndexUpdater

org.apache.hadoop.filecache.TestMRWithDistributedCache

org.apache.hadoop.fs.TestCopyFiles

org.apache.hadoop.fs.TestHarFileSystem

org.apache.hadoop.hbase.HBaseTestingUtility

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.