* The contents of this file are subject to the terms

* of the Common Development and Distribution License

* (the License). You may not use this file except in

* compliance with the License.

*

* You can obtain a copy of the License at

* http://www.sun.com/cddl/cddl.html or

* install_dir/legal/LICENSE

* See the License for the specific language governing

* permission and limitations under the License.

*

* When distributing Covered Code, include this CDDL

* Header Notice in each file and include the License file

* at install_dir/legal/LICENSE.

* If applicable, add the following below the CDDL Header,

* with the fields enclosed by brackets [] replaced by

* your own identifying information:

* "Portions Copyrighted [year] [name of copyright owner]"

*

* Copyright 2009 Sun Microsystems Inc. All Rights Reserved

*/

* Harness hook for the hadoop hadoop benchmark.

*

* @author Damien Cooke

*/

   

   

    static Logger logger = Logger.getLogger(GridMix.class.getName());

    int totalRunningTimeInSecs = 0;   

    private String hadoopHome;

    private String gridmixHome;

    private String hadoopConfPath;

    private String hadoopVersion;

    private String benchmarkName;

    private Calendar startTime;

    private Calendar endTime;

    private CommandHandle benchHandle = null;

    private ArrayList<String> slaveList;

    private ArrayList<String> masterList;

    private String runDir;

    private String runID;

    private ParamRepository params;

    private String otherHadoopJob;

    private String javaHome;

    private String hadoopLogDir;

   

   

  

    /**

     * This method is called to configure the specific benchmark run

     * Tasks done in this method include reading user parameters,

     * logging them and initializing various local variables.

     *

     * @throws Exception If configuration was not successful

     **/

     @Configure public void configure() throws Exception{

        logger.info("Benchmark configure starting");

        BufferedWriter bf = null;

        runDir = getOutDir();

        runID = getRunId();

        params = getParamRepository();

        //Obtaining configuration parameters

        gridmixHome = params.getParameter("hadoopConfig/GridMixPath");

        hadoopHome = params.getParameter("hadoopConfig/hadoopHomeDir");

        hadoopVersion = params.getParameter("hadoopConfig/hadoopVersion");

        hadoopConfPath = hadoopHome + File.separator + "conf" + File.separator;

        benchmarkName = params.getParameter("hadoopConfig/benchmarkComboName");

        otherHadoopJob = params.getParameter("hadoopConfig/hadoopJob");

        //collect details for hadoop-env.sh

        javaHome = params.getParameter("fh:jvmConfig/fh:javaHome");

        hadoopLogDir = params.getParameter("hadoopConfig/hadoopLogDir");

        //collect the master(s) and slaves from the interface and place them in

        slaveList = new ArrayList<String>(Arrays.<String>asList(params.getParameter("slaveConfig/fa:hostConfig/fa:host").split(" ")));

        masterList = new ArrayList<String>(Arrays.<String>asList(params.getParameter("masterConfig/fa:hostConfig/fa:host").split(" ")));       

               

        // Create file "masters" and "slaves" in ConfPath

        try

        {           

            bf = new BufferedWriter(new FileWriter(hadoopHome + File.separator + "conf" + File.separator + "masters"));

            for(Iterator<String> it = masterList.iterator();it.hasNext();)

            {

                String master = it.next();

                bf.write(master, 0, master.length());

                bf.newLine();

            }           

            bf.close();

        } catch (Exception fe)

        {

            throw new Exception("Could not create masters file in " + hadoopConfPath, fe);

        }

       

        // Repeat for slaves       

        try

        {           

            bf = new BufferedWriter(new FileWriter(hadoopHome + File.separator + "conf" + File.separator + "slaves"));

            for(Iterator<String> it = slaveList.iterator();it.hasNext();)

            {

                String slave = it.next();

                bf.write(slave, 0, slave.length());

                bf.newLine();

            }           

            bf.close();

        } catch (Exception fe)

        {

            throw new Exception("Could not create slaves file in " + hadoopConfPath, fe);

        }

        //configure the logrecords

        if(this.generateMetricsFile(runDir) == true)

        {

            logger.info("Metrics generation config done.");

        }else

        {

            logger.warning("Metrics generation config not done.");

        }

        //alterHadoopConfigFile

        //configure the hadoop-env.xml file

        if(this.alterHadoopConfigFile(runDir) == true)

        {

            logger.info("Hadoop environment generation config done.");

        }else

        {

            logger.warning("Hadoop environment generation config not done.");

        }

                    

        //collect all advanced configs                       

       

        String key;

        String value;

       

        @SuppressWarnings("unchecked")

        ConcurrentHashMap<String,String> changesForCoreConfig = compareConfigs(getAdvancedCoreDefaultConfig(), gatherAdvancedCoreConfig());

        logger.info("changesForCoreConfig contains: "+changesForCoreConfig.size());

        for (Enumeration changesForCoreConfigEnumerator = changesForCoreConfig.keys(); changesForCoreConfigEnumerator.hasMoreElements();)

        {

            key = (String)changesForCoreConfigEnumerator.nextElement();

            value = (String)changesForCoreConfig.get(key);           

        }

        @SuppressWarnings("unchecked")

        ConcurrentHashMap<String,String> changesForHDFSConfig = compareConfigs(getAdvancedHDFSDefaultConfig(), gatherAdvancedHDFSConfig());

        logger.info("changesForHDFSConfig contains: "+changesForHDFSConfig.size());

        for (Enumeration changesForHDFSConfigEnumerator = changesForHDFSConfig.keys(); changesForHDFSConfigEnumerator.hasMoreElements();)

        {

            key = (String)changesForHDFSConfigEnumerator.nextElement();

            value = (String)changesForHDFSConfig.get(key);           

        }

        @SuppressWarnings("unchecked")

        ConcurrentHashMap <String,String>changesForMapredConfig = compareConfigs(getAdvancedMapredDefaultConfig(), gatherAdvancedMapredConfig());

        logger.info("changesForMapredConfig contains: "+changesForMapredConfig.size());

        for (Enumeration changesForMapredConfigEnumerator = changesForMapredConfig.keys(); changesForMapredConfigEnumerator.hasMoreElements();)

        {

            key = (String)changesForMapredConfigEnumerator.nextElement();

            value = (String)changesForMapredConfig.get(key);           

        }

        if(!writeConfigFiles(changesForCoreConfig,changesForHDFSConfig,changesForMapredConfig))

        {           

            logger.warning("writeConfigFiles has failed, writeConfigFiles failed");

        }

       

        logger.info("Benchmark configure ending");

    }

    /*

     * This method uses collected data from the FABAN form and alters the the hadoop-env.sh file to ensure the java_home and log diectories are correct.

     * @param void

     * @return void

     */

    public boolean alterHadoopConfigFile(final String resultsLocation)

    {    

        String originalConfigFile = hadoopHome + "/conf/hadoop-env.sh";

        String configFile = resultsLocation + "com.sun.pae.hadoop.harness/hadoop-env.sh";

    ArrayList<String> config_values = new ArrayList<String>();

    try

    {

      BufferedReader hadoop_env = new BufferedReader(new FileReader(originalConfigFile));

      String line;

      while ( (line = hadoop_env.readLine()) != null)

      {

        config_values.add(line);

      }

           

            hadoop_env.close();

    }catch (IOException ioe)

    {

            logger.warning("IOException while reading the configuration file in alterHadoopConfigFile.  The message was "+ioe.getMessage());

            return false;

    }

    try

    {

      if(config_values.size() > 0)

      {

        BufferedWriter bf = new BufferedWriter(new FileWriter(configFile));

        String value;

        for(Iterator<String> itConfig = config_values.iterator();itConfig.hasNext();)

        {

          value = itConfig.next();

          if((value.startsWith("export JAVA_HOME=") == true) || (value.startsWith("# export JAVA_HOME=") == true))

          {

            value = "export JAVA_HOME=" + javaHome;

          }else if ((value.startsWith("export HADOOP_LOG_DIR=") == true) || (value.startsWith("# export HADOOP_LOG_DIR=") == true))

          {

            value = "export HADOOP_LOG_DIR=" + hadoopLogDir;

          }

          bf.write(value, 0, value.length());

          bf.newLine();

        }

        bf.close();

                //now copy them to all of the slaves

                ArrayList<String> hostList = new ArrayList<String>();

                //first copy the master and slaves hosts into the list

                hostList = getHostList();

                String host = null;

                //ensure the directory exists if not the logrecords have not been executed.  If not perhaps this run will be invalid.

                File pushDir = new File(resultsLocation + "com.sun.pae.hadoop.harness/");

                if(pushDir.isDirectory() == true)

                {

                    logger.info(pushDir.toString() + " already exists");

                    for(Iterator<String> it = hostList.iterator(); it.hasNext();)

                    {

                        host = it.next();

                        String configurationFile = pushDir.getPath() +"/hadoop-env.sh";

                        String destiniationFile = hadoopConfPath + "hadoop-env.sh";

                        if(RunContext.pushFile(configurationFile, host, destiniationFile) != true)

                        {

                            logger.warning("hadoop config file was not written to "+host);

                        }else

                        {

                            logger.info("Configuration file written to "+host);

                        }

                    }

                }else

                {

                    logger.warning("Directory com.sun.pae.hadoop.harness does not exist");

                    return false;

                }

      }

    }catch (IOException ioe)

    {

      logger.warning("IOException while writing the configuration file in alterHadoopConfigFile.  The message was "+ioe.getMessage());

            return false;

    }

        return true;

    }

    /*

     * This method collects all of the advanced HDF configurations from the configuration form.

     * @param void

     * @return hashMap of values and keys to configure hadoop with

     */

    public ConcurrentHashMap<String, String> getAdvancedHDFSDefaultConfig()

    {

        ConcurrentHashMap <String,String>advancedHDFSConfig = new ConcurrentHashMap<String,String>(46);

        advancedHDFSConfig.put("dfs.namenode.logging.level", "info");

        advancedHDFSConfig.put("dfs.secondary.http.address", "0.0.0.0:50090");

        advancedHDFSConfig.put("dfs.datanode.address", "0.0.0.0:50010");

        advancedHDFSConfig.put("dfs.datanode.http.address", "0.0.0.0:50075");

        advancedHDFSConfig.put("dfs.datanode.ipc.address", "0.0.0.0:50020");

        advancedHDFSConfig.put("dfs.datanode.handler.count", "3");

        advancedHDFSConfig.put("dfs.http.address", "0.0.0.0:50070");

        advancedHDFSConfig.put("dfs.https.enable", "false");

        advancedHDFSConfig.put("dfs.https.need.client.auth", "false");

        advancedHDFSConfig.put("dfs.https.server.keystore.resource", "ssl-server.xml");

        advancedHDFSConfig.put("dfs.https.client.keystore.resource", "ssl-client.xml");

        advancedHDFSConfig.put("dfs.datanode.https.address", "0.0.0.0:50475");

        advancedHDFSConfig.put("dfs.https.address", "0.0.0.0:50470");

        advancedHDFSConfig.put("dfs.datanode.dns.interface", "default");

        advancedHDFSConfig.put("dfs.datanode.dns.nameserver", "default");

        advancedHDFSConfig.put("dfs.replication.considerLoad", "true");

        advancedHDFSConfig.put("dfs.default.chunk.view.size", "32768");

        advancedHDFSConfig.put("dfs.datanode.du.reserved", "0");

        advancedHDFSConfig.put("dfs.name.dir", "${hadoop.tmp.dir}/dfs/name");

        advancedHDFSConfig.put("dfs.name.edits.dir", "${dfs.name.dir}");

        advancedHDFSConfig.put("dfs.web.ugi", "webuser,webgroup");

        advancedHDFSConfig.put("dfs.permissions", "true");

        advancedHDFSConfig.put("dfs.permissions.supergroup", "supergroup");

        advancedHDFSConfig.put("dfs.data.dir", "${hadoop.tmp.dir}/dfs/data");

        advancedHDFSConfig.put("dfs.replication", "3");

        advancedHDFSConfig.put("dfs.replication.max", "512");

        advancedHDFSConfig.put("dfs.replication.min", "1");

        advancedHDFSConfig.put("dfs.block.size", "67108864");

        advancedHDFSConfig.put("dfs.df.interval", "60000");

        advancedHDFSConfig.put("dfs.client.block.write.retries", "3");

        advancedHDFSConfig.put("dfs.blockreport.intervalMsec", "3600000");

        advancedHDFSConfig.put("dfs.blockreport.initialDelay", "0");

        advancedHDFSConfig.put("dfs.heartbeat.interval", "3");

        advancedHDFSConfig.put("dfs.namenode.handler.count", "10");

        advancedHDFSConfig.put("dfs.safemode.threshold.pct", "0.999f");

        advancedHDFSConfig.put("dfs.safemode.extension", "30000");

        advancedHDFSConfig.put("dfs.balance.bandwidthPerSec", "1048576");

        advancedHDFSConfig.put("dfs.hosts", "");

        advancedHDFSConfig.put("dfs.hosts.exclude", "");

        advancedHDFSConfig.put("dfs.max.objects", "0");

        advancedHDFSConfig.put("dfs.namenode.decommission.interval", "30");

        advancedHDFSConfig.put("dfs.namenode.decommission.nodes.per.interval", "5");

        advancedHDFSConfig.put("dfs.replication.interval", "3");

        advancedHDFSConfig.put("dfs.access.time.precision", "3600000");

        advancedHDFSConfig.put("dfs.support.append", "false");

        return advancedHDFSConfig;

       

    }

    /*

     * using the info from the config form and generates a list of configuration items.  These are tested and compared

     * to the default values.  If the values are different from default they are written to the resultant ConcurrentHashMap.

     * @return advancedCoreConfig, ConcurrentHashMap of config entries that can be empty

     */

    public ConcurrentHashMap<String, String> getAdvancedCoreDefaultConfig()

    {

        ConcurrentHashMap <String,String>advancedCoreDefaultConfig = new ConcurrentHashMap<String,String>(52);

        //collect the values from the forms       

        advancedCoreDefaultConfig.put("hadoop.tmp.dir", "/tmp/hadoop-${user.name}");

        advancedCoreDefaultConfig.put("hadoop.native.lib", "true");

        advancedCoreDefaultConfig.put("hadoop.http.filter.initializers", "");

        advancedCoreDefaultConfig.put("hadoop.security.authorization", "false");

        advancedCoreDefaultConfig.put("hadoop.logfile.size", "10000000");

        advancedCoreDefaultConfig.put("hadoop.logfile.count", "10");

        advancedCoreDefaultConfig.put("io.file.buffer.size", "4096");

        advancedCoreDefaultConfig.put("io.bytes.per.checksum", "512");

        advancedCoreDefaultConfig.put("io.skip.checksum.errors", "false");

        advancedCoreDefaultConfig.put("io.compression.codecs", "org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec");

        advancedCoreDefaultConfig.put("io.serializations", "org.apache.hadoop.io.serializer.WritableSerialization");

        advancedCoreDefaultConfig.put("fs.default.name", "file:///");

        advancedCoreDefaultConfig.put("fs.trash.interval", "0");

        advancedCoreDefaultConfig.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem");

        advancedCoreDefaultConfig.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");

        advancedCoreDefaultConfig.put("fs.s3.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

        advancedCoreDefaultConfig.put("fs.s3n.impl", "org.apache.hadoop.fs.s3native.NativeS3FileSystem");

        advancedCoreDefaultConfig.put("fs.kfs.impl", "org.apache.hadoop.fs.kfs.KosmosFileSystem");

        advancedCoreDefaultConfig.put("fs.hftp.impl", "org.apache.hadoop.hdfs.HftpFileSystem");

        advancedCoreDefaultConfig.put("fs.hsftp.impl", "org.apache.hadoop.hdfs.HsftpFileSystem");

        advancedCoreDefaultConfig.put("fs.ftp.impl", "org.apache.hadoop.fs.ftp.FTPFileSystem");

        advancedCoreDefaultConfig.put("fs.ramfs.impl", "org.apache.hadoop.fs.InMemoryFileSystem");

        advancedCoreDefaultConfig.put("fs.har.impl", "org.apache.hadoop.fs.HarFileSystem");

        advancedCoreDefaultConfig.put("fs.checkpoint.dir", "${hadoop.tmp.dir}/dfs/namesecondary");

        advancedCoreDefaultConfig.put("fs.checkpoint.edits.dir", "${fs.checkpoint.dir}");

        advancedCoreDefaultConfig.put("fs.checkpoint.period", "3600");

        advancedCoreDefaultConfig.put("fs.checkpoint.size", "67108864");

        advancedCoreDefaultConfig.put("fs.s3.block.size", "67108864");

        advancedCoreDefaultConfig.put("fs.s3.buffer.dir", "${hadoop.tmp.dir}/s3");

        advancedCoreDefaultConfig.put("fs.s3.maxRetries", "4");

        advancedCoreDefaultConfig.put("fs.s3.sleepTimeSeconds", "10");

        advancedCoreDefaultConfig.put("local.cache.size", "10737418240");

        advancedCoreDefaultConfig.put("io.seqfile.compress.blocksize", "1000000");

        advancedCoreDefaultConfig.put("io.seqfile.lazydecompress", "true");

        advancedCoreDefaultConfig.put("io.seqfile.sorter.recordlimit", "1000000");

        advancedCoreDefaultConfig.put("io.mapfile.bloom.size", "1048576");

        advancedCoreDefaultConfig.put("io.mapfile.bloom.error.rate", "0.005");

        advancedCoreDefaultConfig.put("hadoop.util.hash.type", "murmur");

        advancedCoreDefaultConfig.put("ipc.client.idlethreshold", "4000");

        advancedCoreDefaultConfig.put("ipc.client.kill.max", "10");

        advancedCoreDefaultConfig.put("ipc.client.connection.maxidletime", "10000");

        advancedCoreDefaultConfig.put("ipc.client.connect.max.retries", "10");

        advancedCoreDefaultConfig.put("ipc.server.listen.queue.size", "128");

        advancedCoreDefaultConfig.put("ipc.server.tcpnodelay", "false");

        advancedCoreDefaultConfig.put("ipc.client.tcpnodelay", "false");

        advancedCoreDefaultConfig.put("webinterface.private.actions", "false");

        advancedCoreDefaultConfig.put("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.net.StandardSocketFactory");

        advancedCoreDefaultConfig.put("hadoop.rpc.socket.factory.class.ClientProtocol", "");

        advancedCoreDefaultConfig.put("hadoop.socks.server", "");

        advancedCoreDefaultConfig.put("topology.node.switch.mapping.impl", "org.apache.hadoop.net.ScriptBasedMapping");

        advancedCoreDefaultConfig.put("topology.script.file.name", "");

        advancedCoreDefaultConfig.put("topology.script.number.args", "100");

        return advancedCoreDefaultConfig;

    }

    /*