* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.

*

* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.

*

* The contents of this file are subject to the terms of either the GNU

* General Public License Version 2 only ("GPL") or the Common Development

* and Distribution License("CDDL") (collectively, the "License").  You

* may not use this file except in compliance with the License. You can obtain

* a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html

* or glassfish/bootstrap/legal/LICENSE.txt.  See the License for the specific

* language governing permissions and limitations under the License.

*

* When distributing the software, include this License Header Notice in each

* file and include the License file at glassfish/bootstrap/legal/LICENSE.txt.

* Sun designates this particular file as subject to the "Classpath" exception

* as provided by Sun in the GPL Version 2 section of the License file that

* accompanied this code.  If applicable, add the following below the License

* Header, with the fields enclosed by brackets [] replaced by your own

* identifying information: "Portions Copyrighted [year]

* [name of copyright owner]"

*

* Contributor(s):

*

* If you wish your version of this file to be governed by only the CDDL or

* only the GPL Version 2, indicate your decision by adding "[Contributor]

* elects to include this software in this distribution under the [CDDL or GPL

* Version 2] license."  If you don't indicate a single choice of license, a

* recipient has the option to distribute your version of this file under

* either the CDDL, the GPL Version 2 or to extend the choice of license to

* its licensees as provided above.  However, if you add GPL Version 2 code

* and therefore, elected the GPL Version 2 license, then the option applies

* only if the new code is made subject to such option by the copyright

* holder.

*/

* This class is run as part of a standalone application that is instantiated by

* NodeAgentMain. Its primary purpose is to keep the instances that belong to

* the NodeAgent, and the NodeAgent itself, in-sync with the Domain Admin Server

* (DAS). The NodeAgent also facilites creating, deleting and the starting of

* instances that belongs to it. The NodeAgent uses JMX Connectors for

* communication to and from the DAS and uses the internal RMI Admin Channel to

* check the state of the instances' that currently exist.

*/

    NodeAgentMBean, IAdminConstants {

    private static final Object synchronizationLock = new Object();

   

    private static final String

        DOMAIN_XML_DOMAIN_NAME_PROPERTY_NAME = "administrative.domain.name";

               

    protected static final String

        MBEAN_SERVER_DOMAIN = "com.sun.appserver.nodeagent";      

       

    private AgentConfig _agentConfig;     

  

    public static Properties env;

       

    public NodeAgent() throws Exception {       

        this(new DASPropertyReader(new AgentConfig()),

             new NodeAgentPropertyReader(new AgentConfig()), new AgentConfig());

    }

    public NodeAgent(AgentConfig agentConfig) throws Exception {

        this(new DASPropertyReader(agentConfig),

             new NodeAgentPropertyReader(agentConfig), agentConfig);

        // now reset InstanceDirs properly.

        // Note this is a create-node-agent call.  We are running in DAS, not NA

        InstanceDirs.initialize(agentConfig);

    }         

    public NodeAgent(

        DASPropertyReader dasReader, NodeAgentPropertyReader naReader)

    throws Exception {

        this(dasReader, naReader, new AgentConfig());

    }

   

    /**

     * @param dasReader Reads from das.properties

     * @param naReader  Reads from nodeagent.properties

     * @throws Exception 

     */

    public NodeAgent(DASPropertyReader dasReader,

        NodeAgentPropertyReader naReader, AgentConfig agentConfig)

    throws Exception {

        // we are running inside NA, not DAS, so use the System Properties

        InstanceDirs.initialize( System.getProperty(INSTANCE_ROOT_PROPERTY));

        setDASPropertyReader(dasReader);

        setNodeAgentPropertyReader(naReader);

        _agentConfig = agentConfig;

       

        // read in domain.xml from the nodeagent config directory

        String domainXMLLocation = System.getProperty(INSTANCE_ROOT_PROPERTY)

                                 + RELATIVE_LOCATION_DOMAIN_XML;

        setConfigContext(ConfigFactory.createConfigContext(domainXMLLocation));

    }

    // ********************************************************************

    // ********************* NodeAgentMBean interface *********************

    // ********************************************************************      

       

    /**

     * This method is used to remotely start an instance

     *

     * @param instanceName      instance to start

     * @throws AgentException   Exception thrown if anything goes wrong

     *                          with starting the instance

     */   

    public void startInstance(String instanceName) throws AgentException {

  

        // call startInstance with sync option

        startInstance(instanceName, true);

    }

   

    /**

     * This method is used to remotely start an instance

     *

     * @param instanceName      instance to start

     * @param env               environment keys to be used while starting instance

     * @throws AgentException   Exception thrown if anything goes wrong

     *                          with starting the instance

     */   

    public void startInstance(String instanceName, Properties env) throws AgentException {

  

        // call startInstance with sync option

        ASLauncher.setEnvironment(env);

        startInstance(instanceName, true);

    }

    /**

     * This method is used to remotly stop an instance

     *

     * @param instanceName    instance to stop

     * @throws AgentException Exception thrown if anything goes wrong with

     *                        stopping the instance

     */   

    public void stopInstance(String instanceName) throws AgentException {

        getLogger().log(Level.INFO, "nodeagent.stoppingInstance", instanceName);                       

            stopInstance(instanceName, false, -1);   

    }

    /**

     * This method is used to remotly stop an instance forcible

     *

     * @param instanceName    instance to stop

     * @param timeout         timeout before process killed forcibly

     * @throws AgentException Exception thrown if anything goes wrong with

     *                        stopping the instance

     */   

    public void stopInstance(String instanceName, int timeout)

    throws AgentException {

        stopInstance(instanceName, true, timeout);

    }

    /**

     * This method is used to remotly stop an instance forcibly

     *

     * @param instanceName    instance to stop

     * @param force           flag for force kill or normal kill

     * @param timeout         timeout before process killed forcibly

     * @throws AgentException Exception thrown if anything goes wrong with

     *                        stopping the instance

     */   

    private void stopInstance(String instanceName, boolean force, int timeout)

    throws AgentException {

        getLogger().log(Level.INFO, "nodeagent.stoppingInstance", instanceName);                       

       

        // make sure instance exists to start it

        if (!instanceExists(instanceName)) {

            // instance doesn't exist, this could be caused by a sync error

            // with the nodeagent so it doesn't know to create the instance

            // and then someone tries to start or stop it

            throw new AgentException(_strMgr.getString(

                "nodeAgent.instanceDoesNotExist", instanceName));

        }

        checkOnInstanceIsStarted(instanceName);

       

        try {          

            //beginning instance stopping

            InstanceStatus.getInstance().updateStatus(

                instanceName, Status.kInstanceStoppingCode);

            // stop instance and remove it from the ProcessManager

            if (!ProcessManager.getInstance().processExists(instanceName)){

                // create the process so it can be stopped this solves the

                // case of an instance being started outside the NA, like call

                // startserv script directly or NA reconvery from failure.

                // watchdogging can not be done, because associated process

                // can not be recaptured

                ProcessManager.getInstance().addProcessInstance(instanceName,

                                    new ProcessInstanceInternal(instanceName));

            }

            if (!force)  {

                ProcessManager.getInstance().stopProcess(instanceName);       

            } else {

                ProcessManager.getInstance().stopProcess(instanceName, timeout);

            }

            ProcessManager.getInstance().removeProcessInstance(instanceName);

            //instance successfully stopped

            InstanceStatus.getInstance().updateStatusFromAdminChannel(

                instanceName, Status.kInstanceNotRunningCode);

        } catch (Exception e) {

            //stop failed. We mark the instance running so that the user can

            //attempt to issue the stop a second time..

            InstanceStatus.getInstance().updateStatusFromAdminChannel(

                instanceName, Status.kInstanceRunningCode);

            ProcessInstance pInstance =

                ProcessManager.getInstance().getProcess(instanceName);

            if (pInstance !=null) pInstance.setStopping(false);

            StringManagerBase sm = StringManagerBase.getStringManager(

                                   getLogger().getResourceBundleName()); 

            getLogger().log(Level.WARNING, sm.getString(

                "nodeagent.stop.instance.exception", instanceName), e);            

            throw new AgentException(e);

        }   

    }

  

    /**

     * Notifies this NodeAgent that a sync of domain.xml is required. 

     * It also initiates a reconcilation with the configuration for the node

     * described in domain.xml and the current node's configuration. This method

     * is usually call via a notification from the DAS, but it also can be

     * called by a third party.

     *

     * @throws AgentException Exception thrown if anything goes wrong with sync

     *                        and reconciliation

     */   

    public void synchronizeWithDAS() throws AgentException {

        // this methos returns the created instances which is

        // not needed as part of the exposed api

        synchronizeWithDASInternal(

            SynchronizationDriverFactory.NODE_AGENT_CONFIG_URL);

    }

   

    public void clearRuntimeStatus() {

        RuntimeStatus.clearRuntimeStatus();

    }

           

    public RuntimeStatus getRuntimeStatus() throws AgentException {       

        try {           

            final AgentManager manager = new AgentManager(getNodeAgentConfig());

            return RuntimeStatus.getRuntimeStatus(getNodeAgentName(),

                manager);

        } catch (Exception ex) {

            StringManagerBase sm = StringManagerBase.getStringManager(

                getLogger().getResourceBundleName()); 

            getLogger().log(Level.INFO, sm.getString(

                "nodeagent.getStatus.exception", getNodeAgentName()), ex);            

            throw new AgentException(ex);

        }

    }           

    public void rotateLog() {

        LogMBean.getInstance().rotateNow();

        getLogger().log(Level.INFO, "nodeagent.rotateLog");                       

    }

   

    public String getLogFilesDirectory(String componentName)

    throws AgentException {

        String logDir="";

        try {

            logDir=getLogFileForConponent(componentName, null);

            // remove default log file from path

            logDir=logDir.substring(0, logDir.lastIndexOf("/"));

            getLogger().log(Level.FINE, "Log directory of component " +

                                        componentName + " - " + logDir);

        } catch (Exception e) {

            throw new AgentException(e);

        }

        return logDir;

    }

   

    public AttributeList getLogRecordsUsingQuery(String componentName,

        String logFile, Long fromRecord, Boolean next, Boolean forward,

        Integer requestedCount, Date fromDate, Date toDate, String logLevel,

        Boolean onlyLevel, List listOfModules, Properties nameValueMap)

    throws AgentException {

        try {

            logFile=getLogFileForConponent(componentName, logFile);

            getLogger().log(Level.FINE, "Querying logfile  - " + logFile);

            return LogMBean.getInstance().getLogRecordsUsingQuery(logFile,

                    fromRecord, next, forward, requestedCount, fromDate, toDate,

                    logLevel, onlyLevel, listOfModules, nameValueMap);

        } catch (Exception e) {

            throw new AgentException(e);

        }

    }

   

    public void dasHasMoved(String newHost, String newPort)

    throws AgentException, MBeanException {

        try {

            if (haveDASCoordinatesChanged(newHost, newPort) == false)

                return;

            StringManagerBase sm = StringManagerBase.getStringManager(

                                        getLogger().getResourceBundleName());             

            _logger.log(Level.INFO, "nodeagent.das_has_moved.info",

                        new Object[] {newHost, newPort});

            writeToDASPropertyReader(newHost, newPort);

            setJMXConnectorInfo(newHost, newPort);

            cleanCachedConnectionToDAS();

        } catch (IOException ex) {

            throw new AgentException(ex);

        } catch (ConfigException ex) {

            throw new AgentException(ex);

        } catch (InstanceException ex) {

            throw new AgentException(ex);

        }

    }

   

    private void writeToDASPropertyReader(String newHost, String newPort)

    throws IOException {

        // get appropriate das properties

        DASPropertyReader dasProperty = getDASPropertyReader();

        // something changed for the das connection, update das properties

        dasProperty.setPort(newPort);

        dasProperty.setHost(newHost);

        // logmessage so the user will know to change say /etc/hosts file

        // so it becomes pingable.

        dasProperty.write();

        getLogger().log(Level.CONFIG, "nodeagent.changeOfDASValues",

                        new Object[] {newHost + ":" + newPort, newHost} );   

    }

    private void setJMXConnectorInfo(String newHost, String newPort) 

    throws ConfigException {    

        AdminService as = ServerHelper.getAdminServiceForServer(

                            getConfigContext(), "server");

        JmxConnector jmxConn = as.getJmxConnectorByName(

                                as.getSystemJmxConnectorName());

        jmxConn.setPort(newPort);

        ElementProperty clientHostname =

            jmxConn.getElementPropertyByName(HOST_PROPERTY_NAME);

        clientHostname.setValue(HOST_PROPERTY_NAME, newHost);

        getConfigContext().flush(true);

      

    }

   

    private void cleanCachedConnectionToDAS()

    throws ConfigException, InstanceException {

   

        Server das = ServerHelper.getDAS(getConfigContext());

        InstanceRegistry.removeInstanceConnection(das.getName());

        try {

            InstanceRegistry.getDASConnection(getConfigContext());

        } catch (InstanceException ex) {

            // nothing more can be done, next attempt to connect to DAS

            // hopefully works     

        }

    }

   

    private AgentConfig getNodeAgentConfig() {

        return _agentConfig;

    }

    private void checkOnInstanceIsStarted(String instanceName)

    throws AgentException {

        // The instance must be stoppped before we proceed. This check is here

        // to prevent the same server being started multiple times.  We must

        // synchronize on status to guard agains the case when the same instance

        // is stopped at exactly the same time (and they both attempt to update

        // the status).

       

        // WBN June 12, 2007

        // this method also ensures that we can never stop an instance

        // that is stuck in the starting state (Zombie).

        // Instead we just go ahead and stop it anyways.

        // Change -- do not throw an Exception if it is not "running"

        boolean wasRunning = InstanceStatus.getInstance().updateStatusIfRunning(

                                instanceName, Status.kInstanceStoppingCode);

        if (!wasRunning) {

            StringManagerBase sm = StringManagerBase.getStringManager(

                getLogger().getResourceBundleName());

            String msg = sm.getString(

                "nodeagent.couldNotStopInstance", instanceName,

                InstanceStatus.getInstance().getStatus(

                    instanceName).getStatus().getStatusString());

            getLogger().log(Level.WARNING, msg);                                      

            //throw new AgentException(msg);

        }     

    }

    public String getNodeAgentName() {

        return getNodeAgentConfig().getRepositoryName();

    }

   

    // ********************* NodeAgent supporting methods******************

    protected String getLogFileForConponent(String componentName, String logFile)

    throws Exception {

        // see if nodeagent or server that nodeagent manages.

        if (componentName == null ||

            componentName.equals(getNodeAgentConfig().getRepositoryName())) {

            // nodeagent log, should be able to fall through and the

            // LogMBean work correctly

        } else {

            // make sure instance exists

            if (!instanceExists(componentName)) {

                // instance doesn't exist, this could be caused by a sync error

                // with the nodeagent so it doesn't know to create the instance

                // and then someone tries to start or stop it

                throw new AgentException(_strMgr.getString(

                    "nodeAgent.instanceDoesNotExist", componentName));

            }

            // create log prefix

            String logFileX = "";

            String logPrefix = getNodeAgentConfig().getRepositoryRoot() + "/" +

                               getNodeAgentConfig().getRepositoryName() + "/" +

                               componentName + "/" ;

            boolean bFound = false;

            Config config = ServerHelper.getConfigForServer(

                    getConfigContext(), componentName);

            if (config != null) {

                LogService logservice=config.getLogService();

                if (logservice != null) {

                    logFileX = logservice.getFile();

                    if (logFileX != null && !logFileX.equals("")) {

                        // set that is was found

                        bFound=true;

                        // see if log file set on the way in

                        if (logFile == null) {

                            // no log file set, use log-service default

                            logFile=logFileX;

                        } else {

                            // log file specified, remove last file, so can

                            // concatinate logFile that is passed in

                            logFile = logFileX.substring(0,

                                    logFileX.lastIndexOf("/") + 1) + logFile;

                        }

                        // see if instanceroot used in logfile location

                        // (currently it is)

                        String instanceRoot =

                            System.getProperty(INSTANCE_ROOT_PROPERTY);

                        int iPos = logFile.indexOf(instanceRoot);

                        if (iPos == 0) {

                            // remove instance root and build location of log

                            logFile = logPrefix + logFile.substring(

                                                     instanceRoot.length() + 1);

                        }

                    }

                }

            }

            // see if have found a logfile

            if (!bFound) {

                // no log set in domain.xml, but must set a fully qualified

                // path to instance use defaults

                if (logFile == null) {

                    // use all defaults

                    logFile= logPrefix + "logs/server.log";

                } else {

                    // specific log file set

                    logFile= logPrefix + "logs/" + logFile;

                }

            }

        }

        return logFile;

    }

    /**

     * This method is called to notify NodeAgent that a sync of domain.xml is

     * required. It also initiates a reconcilation with the configuration for

     * the node described in domain.xml and the current node's configuration.

     * This method is usually call via a notification from the DAS, but it

     * also can be called by a third party.

     *

     * @throws AgentException Exception thrown if anything goes wrong with

     *                        sync and reconciliation

     */   

    protected Vector synchronizeWithDASInternal() throws AgentException {       

        return synchronizeWithDASInternal(

                SynchronizationDriverFactory.NODE_AGENT_STARTUP_CONFIG_URL);

    }

   

    private Vector synchronizeWithDASInternal(String url) throws AgentException {

        getLogger().log(Level.INFO, "nodeagent.ExecutingSync");

        final DASPropertyReader dasEnv = getDASPropertyReader();

       

        // We want to control initialization ourselves. The issue is that we do

        // not want initialization (i.e.loading of NSS) to happen implicitly

        // here or we will not able to synchronize the NSS database on Windows

        // platforms since it will be in use. For this reason, we do not want

        // orb initialization in our InitialContextFactory to occur.

        SerialInitContextFactory.setInitializeOrbManager(false);

       

        // Setup as an SSL client

        if (Boolean.valueOf(dasEnv.isDASSecure()).booleanValue()) {

            String certNickname = "s1as";

            /*try {

                if (getConfigContext() != null)

                    certNickname = ServerHelper.getCertNickname(

                        getConfigContext(), DAS_SERVER_NAME);

            } catch(Exception ex) {

                getLogger().log(

                    Level.WARNING, "nodeagent.synchronization.Exception",ex);

                throw new AgentException(ex);

            }*/

            new NodeAgentSyncWithDAS_TlsClientEnvSetter(certNickname).setup();

        }

        Vector createdInstances=null;

        // check to see if bound, check here because this method is in

        // mbean interface

        if (getNodeAgentPropertyReader().isBound()) {

            // Synchronise with DAS, which will put domain.xml in

            // config directory

            long startSync=System.currentTimeMillis();

            try {

                try {

                    SynchronizationDriver sd = SynchronizationDriverFactory.

                        getSynchronizationDriver(

                            System.getProperty(INSTANCE_ROOT_PROPERTY),

                        url, null);

                    sd.synchronize();

    } catch(final DASCommunicationException de) {

                    new CommunicationExceptionLogUtils(getLogger(),

                      Level.SEVERE, dasEnv).handleDASCommunicationException(de);

                } catch (NonMatchingDASContactedException ex) {

                    stopAgentWithPause();

                } catch (Exception se) {

                    // An exception is now thrown when the das can't be reached

                    // this is an acceptable case for the nodeagent, so just

                    // log at fine level

                    getLogger().log(Level.FINE,"Synchronization Exception", se);                   

                }

                // check to make sure there is a domain.xml to use, this could

                // happen as a result of a rendezvous with the das during create

                // time and a sync never being successfully completed. Like if

                // the nodeagent is initially started when the das is down

                File domain_xml_file = new File(

                                    System.getProperty(INSTANCE_ROOT_PROPERTY) +

                                    RELATIVE_LOCATION_DOMAIN_XML);

                if (!domain_xml_file.exists()) {

                    // domain.xml doesn't exists, can't continue

                    getLogger().log(Level.SEVERE, "nodeagent.initialSyncFailed");

                    stopAgentWithPause();

                }

                // Now validate the admin-password against admin-keyfile which

                // might be in synch.

                if (domain_xml_file.exists()) {

                    try {

                        validateAdminUserAndPassword();

                    } catch (final Exception ae) {

                        this.stopAgentWithPause();

                    }

                }

                // Resync our config context in case domain.xml

                // has changed on disk.

                ConfigContext configContext = getConfigContext();

                if (configContext.isFileChangedExternally()) {

                    configContext.refresh();

                }

                String instanceRoot =

                    (String) System.getProperty(INSTANCE_ROOT_PROPERTY);

                System.setProperty( KEYSTORE_PROPERTY,

                                    instanceRoot + JKS_KEYSTORE);

                System.setProperty( TRUSTSTORE_PROPERTY,

                                    instanceRoot + JKS_TRUSTSTORE);

                // check to see if the nodeagent still exists

                if(getNodeAgentConfigBean(getConfigContext()) == null) {

                    // nodeagent has been removed

                    getLogger().log(Level.WARNING, "nodeagent.hasBeenDeleted");   

                    // finished synchonizing now reconcile configuration

                    // and remove instances

                    reconcileNodeConfiguration();

                    // clear das.properties

                    getDASPropertyReader().clearProperties();

                    getDASPropertyReader().write();

                    // set nodeAgent to deleted, as per spec it will never be

                    // able to start again

                    setNodeAgentBindStatus(AgentConfig.NODEAGENT_DELETED_STATUS);

                    // remove local domain.xml and .domain.xml.timestamp

                    // files that are used in sync

                    File file = null;

                    for (int ii = 0; ii < LOCAL_DAS_SYNC_FILES.length; ii++) {

                        file = new File(System.getProperty(INSTANCE_ROOT_PROPERTY)

                                      + LOCAL_DAS_SYNC_FILES[ii]);

                        file.delete();

                    }

                    // stop nodeagent after delete

                    System.exit(0);                   

                }  else {

                    // check to see if config for communication with DAS

                    // has changes

                    updateDASConnectionInfo();                   

                    // reinitialize NodeAgent with updated domain.xml

                    init();

                    // finished synchonizing now reconcile configuration

                    createdInstances=reconcileNodeConfiguration();                   

                }

               

            } catch (Exception e) {

                getLogger().log(Level.WARNING,

                                "nodeagent.synchronization.Exception",e);

                throw new AgentException(e);

            }       

        } else {

            // not bound

            getLogger().log(Level.FINE,

                            "Error on Synchronization, NodeAgent is not bound");

            throw new AgentException(

                    _strMgr.getString("nodeAgent.unboundSyncError"));

        }          

        return createdInstances;

    }           

   

    private void ensureInstanceIsStopped(

        String instanceName, boolean synchronizeInstance)

    throws AgentException {

        // The instance must be stoppped before we proceed. This check is here

        // to prevent the same server being started multiple times. We must

        // synchronize on status to guard agains the case when the same instance

        // is started at exactly the same time (and they both attempt to update

        // the status).

        int newStatus = Status.kInstanceStartingCode;

        if (synchronizeInstance)  {

            newStatus = Status.kInstanceSynchronizingCode;

        }

        boolean wasStopped = InstanceStatus.getInstance().updateStatusIfStopped(

                                instanceName, newStatus);

        if (!wasStopped) {

            StringManagerBase sm = StringManagerBase.getStringManager(

                getLogger().getResourceBundleName()); 

            String msg =

                sm.getString("nodeagent.couldNotStartInstance", instanceName,

                             InstanceStatus.getInstance().getStatus(

                                instanceName).getStatus().getStatusString());

            getLogger().log(Level.WARNING, msg);               

            throw new AgentException(msg);

        }

    }

 

   

   

   

   

    private void debug(String s) {

        getLogger().log(Level.INFO, "NA-DEBUG: " + s);

    }

   

    /**

     * This method is used to remotely start an instance

     *

     * @param instanceName      instance to start

     * @throws AgentException   Exception thrown if anything goes wrong with

     *                          starting the instance

     */   

    protected void startInstance(String instanceName,

        boolean synchronizeInstance) throws AgentException {

        debug("startInstance(" + instanceName + ", " + synchronizeInstance + ")");

        //debug("InstanceDirs: " + InstanceDirs.toStringStatic());

        //validate that the server exists and return its node agent

        synchronized(synchronizationLock) {

            debug("Got Lock: " + instanceName);

            getLogger().log(Level.INFO, "nodeagent.startingInstance", instanceName);

            if (bDebug) System.out.println("Starting instance " +

                instanceName + " with synch flag set to " + synchronizeInstance);

            // make sure instance exists to start it

            if (!instanceExists(instanceName)) {

                // instance doesn't exist, this could be caused by a sync error

                // with the nodeagent so it doesn't know to create the instance

                // and then someone tries to start or stop it

                throw new AgentException(_strMgr.getString(

                        "nodeAgent.instanceDoesNotExist", instanceName));

            }       

            //Ensure that the instance is in a stopped state before starting it.

            ensureInstanceIsStopped(instanceName, synchronizeInstance);       

            if (synchronizeInstance)  {           

                try {

                    //beginning instance synchronization

                    InstanceStatus.getInstance().updateStatus(

                        instanceName, Status.kInstanceSynchronizingCode);

                    // synchronise of behalf of the instance, should only be done

                    // on direct start invocation

                    getLogger().log(Level.INFO,

                        "NodeAgent synchronizing for instance", instanceName);

                    synchronizeInstanceWithDAS(instanceName);

                } catch (AgentException ae) {

                    //synchronization failed

                    InstanceStatus.getInstance().updateStatusFromAdminChannel(

                        instanceName, Status.kInstanceNotRunningCode);

                    // need to throw exception so start-instance command show

                    // proper error and fails

                   

                    debug("ERROR IN SYNCHRONIZATION!!!! " + ae);

                   

                    throw ae;

                }

            }

            debug("Release Lock: " + instanceName);

        }

        // there was no sync or it succeeded

        try {               

            //beginning instance startup

            debug("Getting ready to have ProcessManger start the instance: " + instanceName);

            InstanceStatus.getInstance().updateStatus(

                instanceName, Status.kInstanceStartingCode);                          

            // always start when asked (not checking for now)

            // add instance to ProcessManager & start it

            ProcessManager.getInstance().addProcessInstance(instanceName,

                new ProcessInstanceInternal(instanceName));

            ProcessManager.getInstance().startProcess(instanceName);

            //startup succeeded

            InstanceStatus.getInstance().updateStatusFromAdminChannel(

                instanceName, Status.kInstanceRunningCode);  

        } catch (Exception e) {

            //startup failed. We put the instance in a not running state so that

            // the user can attempt to start it again.

            InstanceStatus.getInstance().updateStatusFromAdminChannel(

                instanceName, Status.kInstanceNotRunningCode);                 

            StringManagerBase sm = StringManagerBase.getStringManager(

                getLogger().getResourceBundleName()); 

            getLogger().log(Level.WARNING,

              sm.getString("nodeagent.start.instance.exception", instanceName), e);                 

            throw new AgentException(e);

        }

    }

   

    private com.sun.enterprise.config.serverbeans.NodeAgent

        getNodeAgentConfigBean(ConfigContext configCtxt)

        throws ConfigException {                       

        // get the servers that currently exist in domain xml

        Domain domain = ConfigAPIHelper.getDomainConfigBean(configCtxt); 

        // get node controller config bean       

        return domain.getNodeAgents().getNodeAgentByName(getNodeAgentName());

    }

    Vector reconcileNodeConfiguration() throws AgentException {       

        getLogger().log(Level.FINE,

                        "Reconciling existing Node Configuration with " +

                        "configuration that is in domain.xml");

        Vector createdInstances=new Vector();

        try {           

            // get the servers that currently exist in domain xml

            Domain domain = ConfigAPIHelper.getDomainConfigBean(

                                getConfigContext()); 

            Server[] servers=domain.getServers().getServer();

           

            // get EEInstancesManager and get instances that currently

            // physically exist in repository

            InstanceConfig instanceConfig = new InstanceConfig(

                                            InstanceDirs.getInstanceRoot());

            EEInstancesManager eeInstancesManager = new EEInstancesManager(

                                                    instanceConfig);

            HashMap hmInstances = getInstancesAndStatus(

                                  instanceConfig, eeInstancesManager);

            // loop through for creatation of instances boolean match = false;

            Iterator serverIt = null;

            String instanceName = null;

            int status = 0;

           

            // get node controller config bean from domain

            com.sun.enterprise.config.serverbeans.NodeAgent nodeController

        = getNodeAgentConfigBean(getConfigContext());

            if (nodeController != null) {

                Properties props = null;

                ElementProperty elementProperty = null;

                Cluster cluster = null;

                // need a default, can't have %%%DOMAIN_NAME%%% go into

                // running server

                String domainName="DomainPropertyNotFound";

                // set default to "" because this string will actual be placed

                // in the startserv's java command for starting the instance

                // via the ProcessLauncher. This is the easiest way to have the

                // com.sun.aas.clusterName set to null for a standalone Instance

                String clusterArg = "";

               

                for (int ii = 0; ii < servers.length; ii++) {

                    // check to see if the server belongs to this

                    // node agent (repository)

                    if (instanceConfig.getRepositoryName().equals(

                            servers[ii].getNodeAgentRef())) {

                        // check if instance already exists or

                        // needs to be created.

                        if (instanceExists(servers[ii].getName())) {

                            getLogger().log(Level.INFO,

                            "nodeagent.instance.exists", servers[ii].getName());

                        } else {

                            // instance does not exist so create it

                            getLogger().log(Level.INFO,

                            "nodeagent.create.instance", servers[ii].getName());

                            instanceConfig.setInstanceName(servers[ii].getName());

                            // set properties for token replacing startserv scripts

                            props = new Properties();

                            // add proper domain

                            elementProperty = domain.getElementPropertyByName(

                                           DOMAIN_XML_DOMAIN_NAME_PROPERTY_NAME);

                            if (elementProperty != null)

                               domainName=elementProperty.getValue();

                           

                            getLogger().log(Level.INFO,

                                        "domain for instance - " + domainName);

                            props.setProperty(EEScriptsTokens.DOMAIN_NAME,

                                              domainName);

                            if (ServerHelper.isServerClustered(

                                  getConfigContext(), servers[ii].getName())) {

                                try {

                                    // see if part of cluster

                                    cluster = ClusterHelper.getClusterForInstance(

                                      getConfigContext(), servers[ii].getName());

                                    if (cluster != null)

                                        clusterArg = "-D" + CLUSTER_NAME + "=\""

                                                   + cluster.getName() + "\"";                                   

                                } catch (ConfigException ce) {

                                    // show at finest, because the server may

                                    // not be part of cluster

                                    getLogger().log(Level.FINEST,

                                        "Server isn't part of cluster", ce);

                                }

                            }

                            props.setProperty(EEScriptsTokens.CLUSTER_NAME, clusterArg);                           

                            // set the overriding properties used in token replacement

                            eeInstancesManager.setOverridingProperties(props);

                            // create instance

                            eeInstancesManager.createInstance();                           

                            // make list of created server instances to know which ones to

                            // sync on nodeagent startup.

                            createdInstances.add(servers[ii].getName());

                        }

                        // remove it from delete instances array

                        hmInstances.remove(servers[ii].getName());

                    }

                }

            }

           

            // loop through and delete instances if nessesary

            serverIt = hmInstances.keySet().iterator();

            while (serverIt.hasNext()) {

                instanceName = (String)serverIt.next();

                status = ((Status)hmInstances.get(instanceName)).getStatusCode();

               

                // delete instances

                if (bDebug) System.out.println("Delete Server =" + instanceName

                                             + " with status:" + status);

                getLogger().log(Level.INFO, "nodeagent.delete.instance",

                                instanceName);

                instanceConfig.setInstanceName(instanceName);

               

                // check to see if instances is

                if(status != Status.kInstanceNotRunningCode) {

                    try {

                        stopInstance(instanceName);

                    } catch (AgentException e) {

                        // only log exception, and continue deleting

                        StringManagerBase sm = StringManagerBase.getStringManager(

                            getLogger().getResourceBundleName()); 

                        getLogger().log(

                          Level.INFO,

                          sm.getString(

                             "nodeagent.stop.instance.exception", instanceName),

                          e);                         

                    }

                }

                // delete instance, without checking for JMS

                // instances (RepositoryManager)

                eeInstancesManager.deleteInstance(false);

            }

           

            // after all has been completed successfully

            // synchonize newly created instances to make sure that the new

            // synchroniztion strategy doesn't leave an instance without a

            // config, which could be potentially started by a nodeagent restart

            // loop through and delete instances if nessesary

            serverIt = createdInstances.iterator();

            while (serverIt.hasNext()) {

                instanceName = (String)serverIt.next();

                // sync instance, if failed it should have been logged

                try {

                    synchronizeInstanceWithDAS(instanceName);

                } catch (AgentException ae) {

                    // exception should already be logged

                    // don't stop creation loop, so just keep looping

                    // this was only a fail safe on restart

                }

            }

        } catch (Exception e) {

            throw new AgentException(e);

        }

        return createdInstances;

    }

   

    protected boolean instanceExists(String iName) throws AgentException {

        boolean match=false;

        Iterator serverIt=null;

        String instanceName=null;

        // get instance and status' in repository

        try {

            HashMap hmInstances=getInstancesAndStatus();

            serverIt=hmInstances.keySet().iterator();

            while(serverIt.hasNext()) {

                instanceName=(String)serverIt.next();

                if(instanceName.equals(iName)) {

                    match=true;

                    break;

                }

            }

        } catch (RepositoryException e) {

            throw new AgentException(e);

        }

        return match;

    }

   

    /**

     * This method gets the reposority instances and their status

     *

     * @return HashMap of instance names as the key and the status as the value

     */

    protected HashMap getInstancesAndStatus() throws RepositoryException {

        InstanceConfig instanceConfig =

            new InstanceConfig(InstanceDirs.getInstanceRoot());

        EEInstancesManager eeInstancesManager =

            new EEInstancesManager(instanceConfig);

        return getInstancesAndStatus(instanceConfig, eeInstancesManager);

    }

   

   

    /**

     * This method gets the reposority instances and their status

     *

     * @param instanceConfig

     * @param eeInstancesManager

     * @return HashMap of instance names as the key and the status as value

     */

    protected HashMap getInstancesAndStatus(InstanceConfig instanceConfig,

        EEInstancesManager eeInstancesManager) throws RepositoryException {

       

        // get EEInstancesManager and get instances that currently physically

        // exist in repository

        HashMap hmInstances = new HashMap();

        RuntimeStatusList rtstatusList =

            eeInstancesManager.getRuntimeStatus(instanceConfig);

        RuntimeStatus rtstatus = null;

        for (int i = 0; i < rtstatusList.size(); i++) {

            rtstatus = rtstatusList.getStatus(i);

            hmInstances.put(rtstatus.getName(), rtstatus.getStatus());

        }

        return hmInstances;

    }

         

    /**

     * This method attempts a "local" rendezvous. We look for all the domains

     * at the same level (i.e. same parent directory) as this node agent, and we

     * register ourselves with the first domain without checking host and port.

     * This code is meant to be called by the installer only and is potentially

     * dangerous since it is common to create multiple domains with the same

     * port, etc in testing scenerios. Furthermore, it is important that the DAS

     * not be running when this is invoked. This writes domain.xml directly, and

     * could result in issues if the DAS is running since the DAS config context

     * will not be refreshed with the new domain.xml.

     * @throws Exception

     */   

    public void localRendezvousWithDAS() throws Exception

    {

        getLogger().log(Level.FINE, "Attempting to Rendezvous with local DAS...");

        AgentConfig agentConfig = getNodeAgentConfig();

        File root = new File(

                      new File(agentConfig.getRepositoryRoot()).getParentFile(),         

                      "domains");       

        NodeAgentPropertyReader naReader = getNodeAgentPropertyReader(); 

        DASPropertyReader dasReader = getDASPropertyReader();       

        ConfigContext goodConfigContext = null;       

        boolean tooMany = false;

        String dasHost2 = dasReader.getHost();

        String dasPort2 = dasReader.getPort();

        if (root.exists()) {

            //Find all the domains with the same parent as this node agent

            File[] domains = root.listFiles();           

            if (domains != null) {                               

                for (int i = 0; i < domains.length; i++) {                                       

                    File domainXml = new File(new File(domains[i], "config"), 

                        PEFileLayout.DOMAIN_XML_FILE);                   

                    getLogger().log(Level.FINE, "Looking for " +

                                                domainXml.getAbsolutePath());

                    if (domainXml.exists()) {                       

                        ConfigContext configContext =

                            ConfigFactory.createConfigContext(

                                domainXml.getAbsolutePath());                       

                        String dasName =

                            ServerHelper.getDAS(configContext).getName();           

                        Config dasConfig = ServerHelper.getConfigForServer(

                                                configContext, dasName);

                        JMXConnectorConfig dasInfo =

                            ServerHelper.getJMXConnectorInfo(configContext, 

                                                             dasName);

                        String dasPort =

                            dasConfig.getHttpService().getHttpListenerById(

                            "admin-listener").getPort();

                        String dasHost = dasInfo.getHost();                       

                       

                        getLogger().log(Level.FINE, "dasHost: " + dasHost +

                              " dasPort: " + dasPort + " dasHost2: " + dasHost2

                            + " dasPort2: " + dasPort2);

                       

                        // Look for the first domain matching the specified host

                        // and port. There must be exactly one or we error out.

                        if (dasPort.equals(dasPort2) && (dasHost.equals(dasHost2) ||

                            PortConflictCheckerConfigBean.isLocalHost(dasHost2))) {

                            if (goodConfigContext != null) {

                                tooMany = true;

                                break; //we have found more than 1 matching domain