Package org.apache.hadoop.mapreduce.v2

Examples of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

69
70
71
72
73
74
75
76
77
78
79
            Configuration config = new Configuration();
            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
            m_dfs_conf = m_dfs.getConfiguration(0);
           
            m_mr = new MiniMRYarnCluster("PigMiniCluster");
            m_mr.init(new Configuration());
            //m_mr.init(m_dfs_conf);
            m_mr.start();
           
            // Write the necessary config info to hadoop-site.xml
View Full Code Here
358
359
360
361
362
363
364
365
366
367
368
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
        + " not found. Not running test.");
      return;
    }

    if (mrCluster == null) {
      mrCluster = new MiniMRYarnCluster(getClass().getSimpleName());
      mrCluster.init(new Configuration());
      mrCluster.start();
    }
    // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
    // workaround the absent public discache.
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);

    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
        .getName(), noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
            m_dfs_conf = m_dfs.getConfiguration(0);

            //Create user home directory
            m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());

            m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers);
            m_mr.init(m_dfs_conf);
            m_mr.start();

            // Write the necessary config info to hadoop-site.xml
            m_mr_conf = new Configuration(m_mr.getConfig());
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
                }
            }
            hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE));

            // Build mini YARN cluster
            m_mr = new MiniMRYarnCluster("PigMiniCluster", 2);
            m_mr.init(m_dfs_conf);
            m_mr.start();
            m_mr_conf = m_mr.getConfig();
            m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                    System.getProperty("java.class.path"));
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
            m_dfs_conf = m_dfs.getConfiguration(0);
           
            //Create user home directory
            m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());

            m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers);
            m_mr.init(m_dfs_conf);
            m_mr.start();

            // Write the necessary config info to hadoop-site.xml
            m_mr_conf = new Configuration(m_mr.getConfig());
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
        + " not found. Not running test.");
      return;
    }

    if (mrCluster == null) {
      mrCluster = new MiniMRYarnCluster(getClass().getName());
      mrCluster.init(new Configuration());
      mrCluster.start();
    }
    // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
    // workaround the absent public discache.
View Full Code Here
301
302
303
304
305
306
307
308
309
310
311
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileContext
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.ipc.TestMRCJCSocketFactory
  • org.apache.hadoop.ipc.TestSocketFactory
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.MiniMRClientClusterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.