Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MiniMRCluster

1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
    if (logDir == null) {
      logDir = tmpDir;
    }
    System.setProperty("hadoop.log.dir", logDir);
    c.set("mapred.output.dir", tmpDir);
    mrCluster = new MiniMRCluster(servers,
      FileSystem.get(conf).getUri().toString(), 1);
    LOG.info("Mini mapreduce cluster started");
    JobConf mrClusterJobConf = mrCluster.createJobConf();
    c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
    /* this for mrv2 support */
 
View Full Code Here
69
70
71
72
73
74
75
76
77
    try {
      localFs = FileSystem.getLocal(conf);
    } catch (IOException io) {
      throw new RuntimeException("problem getting local fs", io);
    }
    mr = new MiniMRCluster(1, "file:///", 1);
    jobConf = mr.createJobConf();
    BufferStore.init(128, 128);
  }
View Full Code Here
227
228
229
230
231
232
233
234
235
236
237
    qSkipSet = new HashSet<String>();

    if (miniMr) {
      dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
      FileSystem fs = dfs.getFileSystem();
      mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
    }

    initConf();

    testFiles = conf.get("test.data.files").replace('\\', '/')
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
      // Builds and starts the mini dfs and mapreduce clusters
      System.setProperty("hadoop.log.dir", ".");
      m_dfs = new MiniDFSCluster(config, dataNodes, true, null);

      m_fileSys = m_dfs.getFileSystem();
      m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);

      // Create the configuration hadoop-site.xml file
      File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
      conf_dir.mkdirs();
      File conf_file = new File(conf_dir, "hadoop-site.xml");
View Full Code Here
650
651
652
653
654
655
656
657
658
659
660
    LOG.info("Starting mini mapreduce cluster...");
    // These are needed for the new and improved Map/Reduce framework
    Configuration c = getConfiguration();
    System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir"));
    c.set("mapred.output.dir", c.get("hadoop.tmp.dir"));
    mrCluster = new MiniMRCluster(servers,
      FileSystem.get(c).getUri().toString(), 1);
    LOG.info("Mini mapreduce cluster started");
    c.set("mapred.job.tracker",
        mrCluster.createJobConf().get("mapred.job.tracker"));
  }
View Full Code Here
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
    if (logDir == null) {
      logDir = tmpDir;
    }
    System.setProperty("hadoop.log.dir", logDir);
    c.set("mapred.output.dir", tmpDir);
    mrCluster = new MiniMRCluster(servers,
      FileSystem.get(conf).getUri().toString(), 1);
    LOG.info("Mini mapreduce cluster started");
    JobConf mrClusterJobConf = mrCluster.createJobConf();
    c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
    /* this for mrv2 support */
 
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    runTestOnTable(new HTable(conf, MULTI_REGION_TABLE_NAME));
  }

  private void runTestOnTable(HTable table)
  throws IOException, InterruptedException, ClassNotFoundException {
    MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);

    Job job = null;
    try {
      LOG.info("Before map/reduce startup");
      job = new Job(conf, "process column contents");
      job.setNumReduceTasks(1);
      Scan scan = new Scan();
      scan.addFamily(INPUT_FAMILY);
      TableMapReduceUtil.initTableMapperJob(
        Bytes.toString(table.getTableName()), scan,
        ProcessContentsMapper.class, ImmutableBytesWritable.class,
        Put.class, job);
      TableMapReduceUtil.initTableReducerJob(
        Bytes.toString(table.getTableName()),
        IdentityTableReducer.class, job);
      FileOutputFormat.setOutputPath(job, new Path("test"));     
      LOG.info("Started " + Bytes.toString(table.getTableName()));
      job.waitForCompletion(true);
      LOG.info("After map/reduce completion");

      // verify map-reduce results
      verify(Bytes.toString(table.getTableName()));
    } finally {
      mrCluster.shutdown();
      if (job != null) {
        FileUtil.fullyDelete(
          new File(job.getConfiguration().get("hadoop.tmp.dir")));
      }
    }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    b.close();
    return result.toString();
  }

  public void testWithLocal() throws Exception {
    MiniMRCluster mr = null;
    try {
      mr = new MiniMRCluster(2, "file:///", 3);
      Configuration conf = mr.createJobConf();
      runWordCount(conf);
    } finally {
      if (mr != null) { mr.shutdown(); }
    }
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
            // Builds and starts the mini dfs and mapreduce clusters
            Configuration config = new Configuration();
            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
            m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);

            // Write the necessary config info to hadoop-site.xml
            m_conf = m_mr.createJobConf();
            m_conf.setInt("mapred.submit.replication", 2);
            m_conf.set("dfs.datanode.address", "0.0.0.0:0");
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
      final int numSlaves = 2;
      Configuration conf = new Configuration();
      dfs = new MiniDFSCluster(conf, numSlaves, true, null);
      fs = dfs.getFileSystem();
     
      mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1);
      writeInputFile(fs, inputPath);
      map = StreamUtil.makeJavaCommand(UlimitApp.class, new String[]{}); 
      runProgram(SET_MEMORY_LIMIT);
      FileUtil.fullyDelete(fs, outputPath);
      assertFalse("output not cleaned up", fs.exists(outputPath));
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.MiniMRCluster

  • org.apache.blur.mapreduce.lib.BlurOutputFormatTest
  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.extraction.demux.TestDemux
  • org.apache.hadoop.cli.TestMRCLI
  • org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy
  • org.apache.hadoop.contrib.index.mapred.TestIndexUpdater
  • org.apache.hadoop.filecache.TestMRWithDistributedCache
  • org.apache.hadoop.fs.TestCopyFiles
  • org.apache.hadoop.fs.TestHarFileSystem
  • org.apache.hadoop.hbase.HBaseTestingUtility
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.