// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//

//   http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.

// Possible console proxy state transition cases

//    Stopped --> Starting -> Running

//    HA -> Stopped -> Starting -> Running

//    Migrating -> Running  (if previous state is Running before it enters into Migrating state

//    Migrating -> Stopped  (if previous state is not Running before it enters into Migrating state)

//    Running -> HA      (if agent lost connection)

//    Stopped -> Destroyed

//

// Starting, HA, Migrating, Running state are all counted as "Open" for available capacity calculation

// because sooner or later, it will be driven into Running state

//

VirtualMachineGuru, SystemVmLoadScanHandler<Long>, ResourceStateAdapter {

    private static final Logger s_logger = Logger.getLogger(ConsoleProxyManagerImpl.class);

    private static final int DEFAULT_CAPACITY_SCAN_INTERVAL = 30000; // 30 seconds

    private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC = 180; // 3 minutes

    private static final int STARTUP_DELAY = 60000; // 60 seconds

    private int _consoleProxyPort = ConsoleProxyManager.DEFAULT_PROXY_VNC_PORT;

    private int _mgmt_port = 8250;

    private List<ConsoleProxyAllocator> _consoleProxyAllocators;

    @Inject

    private ConsoleProxyDao _consoleProxyDao;

    @Inject

    private DataCenterDao _dcDao;

    @Inject

    private VMTemplateDao _templateDao;

    @Inject

    private HostPodDao _podDao;

    @Inject

    private HostDao _hostDao;

    @Inject

    private ConfigurationDao _configDao;

    @Inject

    private CertificateDao _certDao;

    @Inject

    private VMInstanceDao _instanceDao;

    @Inject

    private TemplateDataStoreDao _vmTemplateStoreDao;

    @Inject

    private AgentManager _agentMgr;

    @Inject

    private StorageManager _storageMgr;

    @Inject

    NetworkOrchestrationService _networkMgr;

    @Inject

    NetworkModel _networkModel;

    @Inject

    AccountManager _accountMgr;

    @Inject

    ServiceOfferingDao _offeringDao;

    @Inject

    DiskOfferingDao _diskOfferingDao;

    @Inject

    NetworkOfferingDao _networkOfferingDao;

    @Inject

    PrimaryDataStoreDao _storagePoolDao;

    @Inject

    UserVmDetailsDao _vmDetailsDao;

    @Inject

    ResourceManager _resourceMgr;

    @Inject

    NetworkDao _networkDao;

    @Inject

    RulesManager _rulesMgr;

    @Inject

    TemplateManager templateMgr;

    @Inject

    IPAddressDao _ipAddressDao;

    @Inject

    ManagementServer _ms;

    @Inject

    ClusterManager _clusterMgr;

    private ConsoleProxyListener _listener;

    private ServiceOfferingVO _serviceOffering;

    NetworkOffering _publicNetworkOffering;

    NetworkOffering _managementNetworkOffering;

    NetworkOffering _linkLocalNetworkOffering;

    @Inject

    private VirtualMachineManager _itMgr;

    /*

     * private final ExecutorService _requestHandlerScheduler = Executors.newCachedThreadPool(new

     * NamedThreadFactory("Request-handler"));

     */

    private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL;

    private int _capacityPerProxy = ConsoleProxyManager.DEFAULT_PROXY_CAPACITY;

    private int _standbyCapacity = ConsoleProxyManager.DEFAULT_STANDBY_CAPACITY;

    private boolean _use_lvm;

    private boolean _use_storage_vm;

    private boolean _disable_rp_filter = false;

    private String _instance;

    private int _proxySessionTimeoutValue = DEFAULT_PROXY_SESSION_TIMEOUT;

    private boolean _sslEnabled = true;

    private String _consoleProxyUrlDomain;

    // global load picture at zone basis

    private SystemVmLoadScanner<Long> _loadScanner;

    private Map<Long, ZoneHostInfo> _zoneHostInfoMap; // map <zone id, info about running host in zone>

    private Map<Long, ConsoleProxyLoadInfo> _zoneProxyCountMap; // map <zone id, info about proxy VMs count in zone>

    private Map<Long, ConsoleProxyLoadInfo> _zoneVmCountMap; // map <zone id, info about running VMs count in zone>

    private String _staticPublicIp;

    private int _staticPort;

    private final GlobalLock _allocProxyLock = GlobalLock.getInternLock(getAllocProxyLockName());

    @Inject

    private KeystoreDao _ksDao;

    @Inject

    private KeystoreManager _ksMgr;

    public class VmBasedAgentHook extends AgentHookBase {

        public VmBasedAgentHook(VMInstanceDao instanceDao, HostDao hostDao, ConfigurationDao cfgDao,

                KeystoreManager ksMgr, AgentManager agentMgr, ManagementServer ms) {

            super(instanceDao, hostDao, cfgDao, ksMgr, agentMgr, ms);

        }

        @Override

        public void onLoadReport(ConsoleProxyLoadReportCommand cmd) {

            if (cmd.getLoadInfo() == null) {

                return;

            }

            ConsoleProxyStatus status = null;

            try {

                GsonBuilder gb = new GsonBuilder();

                gb.setVersion(1.3);

                Gson gson = gb.create();

                status = gson.fromJson(cmd.getLoadInfo(), ConsoleProxyStatus.class);

            } catch (Throwable e) {

                s_logger.warn("Unable to parse load info from proxy, proxy vm id : " + cmd.getProxyVmId() + ", info : " + cmd.getLoadInfo());

            }

            if (status != null) {

                int count = 0;

                if (status.getConnections() != null) {

                    count = status.getConnections().length;

                }

                byte[] details = null;

                if (cmd.getLoadInfo() != null) {

                    details = cmd.getLoadInfo().getBytes(Charset.forName("US-ASCII"));

                }

                _consoleProxyDao.update(cmd.getProxyVmId(), count, DateUtil.currentGMTTime(), details);

            } else {

                if (s_logger.isTraceEnabled()) {

                    s_logger.trace("Unable to get console proxy load info, id : " + cmd.getProxyVmId());

                }

                _consoleProxyDao.update(cmd.getProxyVmId(), 0, DateUtil.currentGMTTime(), null);

            }

        }

        @Override

        public void onAgentDisconnect(long agentId, com.cloud.host.Status state) {

            if (state == com.cloud.host.Status.Alert || state == com.cloud.host.Status.Disconnected) {

                // be it either in alert or in disconnected state, the agent

                // process

                // may be gone in the VM,

                // we will be reacting to stop the corresponding VM and let the

                // scan

                // process to

                HostVO host = _hostDao.findById(agentId);

                if (host.getType() == Type.ConsoleProxy) {

                    String name = host.getName();

                    if (s_logger.isInfoEnabled()) {

                        s_logger.info("Console proxy agent disconnected, proxy: " + name);

                    }

                    if (name != null && name.startsWith("v-")) {

                        String[] tokens = name.split("-");

                        long proxyVmId = 0;

                        try {

                            proxyVmId = Long.parseLong(tokens[1]);

                        } catch (NumberFormatException e) {

                            s_logger.error("Unexpected exception " + e.getMessage(), e);

                            return;

                        }

                        final ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

                        if (proxy != null) {

                            // Disable this feature for now, as it conflicts

                            // with

                            // the case of allowing user to reboot console proxy

                            // when rebooting happens, we will receive

                            // disconnect

                            // here and we can't enter into stopping process,

                            // as when the rebooted one comes up, it will kick

                            // off a

                            // newly started one and trigger the process

                            // continue on forever

                            /*

                             * _capacityScanScheduler.execute(new Runnable() {

                             * public void run() { if(s_logger.isInfoEnabled())

                             * s_logger.info("Stop console proxy " +

                             * proxy.getName() +

                             * " VM because of that the agent running inside it has disconnected"

                             * ); stopProxy(proxy.getId()); } });

                             */

                        } else {

                            if (s_logger.isInfoEnabled()) {

                                s_logger.info("Console proxy agent disconnected but corresponding console proxy VM no longer exists in DB, proxy: "

                                        + name);

                            }

                        }

                    } else {

                        assert (false) : "Invalid console proxy name: " + name;

                    }

                }

            }

        }

        @Override

        protected HostVO findConsoleProxyHost(StartupProxyCommand startupCmd) {

            long proxyVmId = startupCmd.getProxyVmId();

            ConsoleProxyVO consoleProxy = _consoleProxyDao.findById(proxyVmId);

            if (consoleProxy == null) {

                s_logger.info("Proxy " + proxyVmId + " is no longer in DB, skip sending startup command");

                return null;

            }

            assert (consoleProxy != null);

            return findConsoleProxyHostByName(consoleProxy.getHostName());

        }

    }

    @Override

    public ConsoleProxyInfo assignProxy(final long dataCenterId, final long vmId) {

        ConsoleProxyVO proxy = doAssignProxy(dataCenterId, vmId);

        if (proxy == null) {

            return null;

        }

        if (proxy.getPublicIpAddress() == null) {

            s_logger.warn("Assigned console proxy does not have a valid public IP address");

            return null;

        }

        KeystoreVO ksVo = _ksDao.findByName(ConsoleProxyManager.CERTIFICATE_NAME);

        assert (ksVo != null);

        if (_staticPublicIp == null) {

            return new ConsoleProxyInfo(proxy.isSslEnabled(), proxy.getPublicIpAddress(), _consoleProxyPort, proxy.getPort(), _consoleProxyUrlDomain);

        } else {

            return new ConsoleProxyInfo(proxy.isSslEnabled(), _staticPublicIp, _consoleProxyPort, _staticPort, _consoleProxyUrlDomain);

        }

    }

    public ConsoleProxyVO doAssignProxy(long dataCenterId, long vmId) {

        ConsoleProxyVO proxy = null;

        VMInstanceVO vm = _instanceDao.findById(vmId);

        if (vm == null) {

            s_logger.warn("VM " + vmId + " no longer exists, return a null proxy for vm:" + vmId);

            return null;

        }

        if (vm != null && vm.getState() != State.Running) {

            if (s_logger.isInfoEnabled()) {

                s_logger.info("Detected that vm : " + vmId + " is not currently at running state, we will fail the proxy assignment for it");

            }

            return null;

        }

        if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {

            try {

                if (vm.getProxyId() != null) {

                    proxy = _consoleProxyDao.findById(vm.getProxyId());

                    if (proxy != null) {

                        if (!isInAssignableState(proxy)) {

                            if (s_logger.isInfoEnabled()) {

                                s_logger.info("A previous assigned proxy is not assignable now, reassign console proxy for user vm : " + vmId);

                            }

                            proxy = null;

                        } else {

                            if (_consoleProxyDao.getProxyActiveLoad(proxy.getId()) < _capacityPerProxy || hasPreviousSession(proxy, vm)) {

                                if (s_logger.isTraceEnabled()) {

                                    s_logger.trace("Assign previous allocated console proxy for user vm : " + vmId);

                                }

                                if (proxy.getActiveSession() >= _capacityPerProxy) {

                                    s_logger.warn("Assign overloaded proxy to user VM as previous session exists, user vm : " + vmId);

                                }

                            } else {

                                proxy = null;

                            }

                        }

                    }

                }

                if (proxy == null) {

                    proxy = assignProxyFromRunningPool(dataCenterId);

                }

            } finally {

                _allocProxyLock.unlock();

            }

        } else {

            s_logger.error("Unable to acquire synchronization lock to get/allocate proxy resource for vm :" + vmId + ". Previous console proxy allocation is taking too long");

        }

        if (proxy == null) {

            s_logger.warn("Unable to find or allocate console proxy resource");

            return null;

        }

        // if it is a new assignment or a changed assignment, update the record

        if (vm.getProxyId() == null || vm.getProxyId().longValue() != proxy.getId()) {

            _instanceDao.updateProxyId(vmId, proxy.getId(), DateUtil.currentGMTTime());

        }

        proxy.setSslEnabled(_sslEnabled);

        if (_sslEnabled) {

            proxy.setPort(443);

        } else {

            proxy.setPort(80);

        }

        return proxy;

    }

    private static boolean isInAssignableState(ConsoleProxyVO proxy) {

        // console proxies that are in states of being able to serve user VM

        State state = proxy.getState();

        if (state == State.Running) {

            return true;

        }

        return false;

    }

    private boolean hasPreviousSession(ConsoleProxyVO proxy, VMInstanceVO vm) {

        ConsoleProxyStatus status = null;

        try {

            GsonBuilder gb = new GsonBuilder();

            gb.setVersion(1.3);

            Gson gson = gb.create();

            byte[] details = proxy.getSessionDetails();

            status = gson.fromJson(details != null ? new String(details, Charset.forName("US-ASCII")) : null, ConsoleProxyStatus.class);

        } catch (Throwable e) {

            s_logger.warn("Unable to parse proxy session details : " + Arrays.toString(proxy.getSessionDetails()));

        }

        if (status != null && status.getConnections() != null) {

            ConsoleProxyConnectionInfo[] connections = status.getConnections();

            for (int i = 0; i < connections.length; i++) {

                long taggedVmId = 0;

                if (connections[i].tag != null) {

                    try {

                        taggedVmId = Long.parseLong(connections[i].tag);

                    } catch (NumberFormatException e) {

                        s_logger.warn("Unable to parse console proxy connection info passed through tag: " + connections[i].tag, e);

                    }

                }

                if (taggedVmId == vm.getId()) {

                    return true;

                }

            }

            //

            // even if we are not in the list, it may because we haven't

            // received load-update yet

            // wait until session time

            //

            if (DateUtil.currentGMTTime().getTime() - vm.getProxyAssignTime().getTime() < _proxySessionTimeoutValue) {

                return true;

            }

            return false;

        } else {

            s_logger.error("No proxy load info on an overloaded proxy ?");

            return false;

        }

    }

    @Override

    public ConsoleProxyVO startProxy(long proxyVmId) {

        try {

            ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

            if (proxy.getState() == VirtualMachine.State.Running) {

                return proxy;

            }

            String restart = _configDao.getValue(Config.ConsoleProxyRestart.key());

            if (restart != null && restart.equalsIgnoreCase("false")) {

                return null;

            }

            if (proxy.getState() == VirtualMachine.State.Stopped) {

                _itMgr.advanceStart(proxy.getUuid(), null, null);

                proxy = _consoleProxyDao.findById(proxy.getId());

            }

            // For VMs that are in Stopping, Starting, Migrating state, let client to wait by returning null

            // as sooner or later, Starting/Migrating state will be transited to Running and Stopping will be transited

            // to

            // Stopped to allow

            // Starting of it

            s_logger.warn("Console proxy is not in correct state to be started: " + proxy.getState());

            return null;

        } catch (StorageUnavailableException e) {

            s_logger.warn("Exception while trying to start console proxy", e);

            return null;

        } catch (InsufficientCapacityException e) {

            s_logger.warn("Exception while trying to start console proxy", e);

            return null;

        } catch (ResourceUnavailableException e) {

            s_logger.warn("Exception while trying to start console proxy", e);

            return null;

        } catch (ConcurrentOperationException e) {

            s_logger.warn("Runtime Exception while trying to start console proxy", e);

            return null;

        } catch (CloudRuntimeException e) {

            s_logger.warn("Runtime Exception while trying to start console proxy", e);

            return null;

        } catch (OperationTimedoutException e) {

            s_logger.warn("Runtime Exception while trying to start console proxy", e);

            return null;

        }

    }

    public ConsoleProxyVO assignProxyFromRunningPool(long dataCenterId) {

        if (s_logger.isTraceEnabled()) {

            s_logger.trace("Assign console proxy from running pool for request from data center : " + dataCenterId);

        }

        ConsoleProxyAllocator allocator = getCurrentAllocator();

        assert (allocator != null);

        List<ConsoleProxyVO> runningList = _consoleProxyDao.getProxyListInStates(dataCenterId, State.Running);

        if (runningList != null && runningList.size() > 0) {

            Iterator<ConsoleProxyVO> it = runningList.iterator();

            while (it.hasNext()) {

                ConsoleProxyVO proxy = it.next();

                if(proxy.getActiveSession() >= _capacityPerProxy){

                    it.remove();

                }

            }

            if (s_logger.isTraceEnabled()) {

                s_logger.trace("Running proxy pool size : " + runningList.size());

                for (ConsoleProxyVO proxy : runningList) {

                    s_logger.trace("Running proxy instance : " + proxy.getHostName());

                }

            }

            List<Pair<Long, Integer>> l = _consoleProxyDao.getProxyLoadMatrix();

            Map<Long, Integer> loadInfo = new HashMap<Long, Integer>();

            if (l != null) {

                for (Pair<Long, Integer> p : l) {

                    loadInfo.put(p.first(), p.second());

                    if (s_logger.isTraceEnabled()) {

                        s_logger.trace("Running proxy instance allocation load { proxy id : " + p.first() + ", load : " + p.second() + "}");

                    }

                }

            }

            Long allocated = allocator.allocProxy(runningList, loadInfo, dataCenterId);

            if (allocated == null) {

                s_logger.debug("Unable to find a console proxy ");

                return null;

            }

            return _consoleProxyDao.findById(allocated);

        } else {

            if (s_logger.isTraceEnabled()) {

                s_logger.trace("Empty running proxy pool for now in data center : " + dataCenterId);

            }

        }

        return null;

    }

    public ConsoleProxyVO assignProxyFromStoppedPool(long dataCenterId) {

        // practically treat all console proxy VM that is not in Running state but can be entering into Running state as

        // candidates

        // this is to prevent launching unneccessary console proxy VMs because of temporarily unavailable state

        List<ConsoleProxyVO> l = _consoleProxyDao.getProxyListInStates(dataCenterId, State.Starting, State.Stopped, State.Migrating, State.Stopping);

        if (l != null && l.size() > 0) {

            return l.get(0);

        }

        return null;

    }

    public ConsoleProxyVO startNew(long dataCenterId) throws ConcurrentOperationException {

        if (s_logger.isDebugEnabled()) {

            s_logger.debug("Assign console proxy from a newly started instance for request from data center : " + dataCenterId);

        }

        if (!allowToLaunchNew(dataCenterId)) {

            s_logger.warn("The number of launched console proxy on zone " + dataCenterId + " has reached to limit");

            return null;

        }

        VMTemplateVO template = null;

        HypervisorType availableHypervisor = _resourceMgr.getAvailableHypervisor(dataCenterId);

        template = _templateDao.findSystemVMReadyTemplate(dataCenterId, availableHypervisor);

        if (template == null) {

            throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId);

        }

        Map<String, Object> context = createProxyInstance(dataCenterId, template);

        long proxyVmId = (Long) context.get("proxyVmId");

        if (proxyVmId == 0) {

            if (s_logger.isTraceEnabled()) {

                s_logger.trace("Creating proxy instance failed, data center id : " + dataCenterId);

            }

            return null;

        }

        ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

        if (proxy != null) {

            SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,

                    new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATED, dataCenterId, proxy.getId(), proxy, null));

            return proxy;

        } else {

            if (s_logger.isDebugEnabled()) {

                s_logger.debug("Unable to allocate console proxy storage, remove the console proxy record from DB, proxy id: " + proxyVmId);

            }

            SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,

                    new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, proxyVmId, null, "Unable to allocate storage"));

        }

        return null;

    }

    protected Map<String, Object> createProxyInstance(long dataCenterId, VMTemplateVO template) throws ConcurrentOperationException {

        long id = _consoleProxyDao.getNextInSequence(Long.class, "id");

        String name = VirtualMachineName.getConsoleProxyName(id, _instance);

        DataCenterVO dc = _dcDao.findById(dataCenterId);

        Account systemAcct = _accountMgr.getSystemAccount();

        DataCenterDeployment plan = new DataCenterDeployment(dataCenterId);

        NetworkVO defaultNetwork = null;

        if (dc.getNetworkType() == NetworkType.Advanced && dc.isSecurityGroupEnabled()) {

            List<NetworkVO> networks = _networkDao.listByZoneSecurityGroup(dataCenterId);

            if (networks == null || networks.size() == 0) {

                throw new CloudRuntimeException("Can not found security enabled network in SG Zone " + dc);

            }

            defaultNetwork = networks.get(0);

        } else {

            TrafficType defaultTrafficType = TrafficType.Public;

            if (dc.getNetworkType() == NetworkType.Basic || dc.isSecurityGroupEnabled()) {

                defaultTrafficType = TrafficType.Guest;

            }

            List<NetworkVO> defaultNetworks = _networkDao.listByZoneAndTrafficType(dataCenterId, defaultTrafficType);

            // api should never allow this situation to happen

            if (defaultNetworks.size() != 1) {

                throw new CloudRuntimeException("Found " + defaultNetworks.size() + " networks of type "

                        + defaultTrafficType + " when expect to find 1");

            }

            defaultNetwork = defaultNetworks.get(0);

        }

        List<? extends NetworkOffering> offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork, NetworkOffering.SystemManagementNetwork);

        LinkedHashMap<Network, List<? extends NicProfile>> networks = new LinkedHashMap<Network, List<? extends NicProfile>>(offerings.size() + 1);

        NicProfile defaultNic = new NicProfile();

        defaultNic.setDefaultNic(true);

        defaultNic.setDeviceId(2);

        networks.put(_networkMgr.setupNetwork(systemAcct, _networkOfferingDao.findById(defaultNetwork.getNetworkOfferingId()), plan, null, null, false).get(0),

                new ArrayList<NicProfile>(Arrays.asList(defaultNic)));

        for (NetworkOffering offering : offerings) {

            networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), new ArrayList<NicProfile>());

        }

        ConsoleProxyVO proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(),

                systemAcct.getId(), 0, _serviceOffering.getOfferHA());

        proxy.setDynamicallyScalable(template.isDynamicallyScalable());

        proxy = _consoleProxyDao.persist(proxy);

        try {

            _itMgr.allocate(name, template, _serviceOffering, networks, plan, null);

        } catch (InsufficientCapacityException e) {

            s_logger.warn("InsufficientCapacity", e);

            throw new CloudRuntimeException("Insufficient capacity exception", e);

        }

        Map<String, Object> context = new HashMap<String, Object>();

        context.put("dc", dc);

        HostPodVO pod = _podDao.findById(proxy.getPodIdToDeployIn());

        context.put("pod", pod);

        context.put("proxyVmId", proxy.getId());

        return context;

    }

    private ConsoleProxyAllocator getCurrentAllocator() {

        // for now, only one adapter is supported

        for(ConsoleProxyAllocator allocator : _consoleProxyAllocators) {

            return allocator;

        }

        return null;

    }

    public void onLoadAnswer(ConsoleProxyLoadAnswer answer) {

        if (answer.getDetails() == null) {

            return;

        }

        ConsoleProxyStatus status = null;

        try {

            GsonBuilder gb = new GsonBuilder();

            gb.setVersion(1.3);

            Gson gson = gb.create();

            status = gson.fromJson(answer.getDetails(), ConsoleProxyStatus.class);

        } catch (Throwable e) {

            s_logger.warn("Unable to parse load info from proxy, proxy vm id : " + answer.getProxyVmId() + ", info : " + answer.getDetails());

        }

        if (status != null) {

            int count = 0;

            if (status.getConnections() != null) {

                count = status.getConnections().length;

            }

            byte[] details = null;

            if (answer.getDetails() != null) {

                details = answer.getDetails().getBytes(Charset.forName("US-ASCII"));

            }

            _consoleProxyDao.update(answer.getProxyVmId(), count, DateUtil.currentGMTTime(), details);

        } else {

            if (s_logger.isTraceEnabled()) {

                s_logger.trace("Unable to get console proxy load info, id : " + answer.getProxyVmId());

            }

            _consoleProxyDao.update(answer.getProxyVmId(), 0, DateUtil.currentGMTTime(), null);

            // TODO : something is wrong with the VM, restart it?

        }

    }

    public void handleAgentDisconnect(long agentId, com.cloud.host.Status state) {

        if (state == com.cloud.host.Status.Alert || state == com.cloud.host.Status.Disconnected) {

            // be it either in alert or in disconnected state, the agent process

            // may be gone in the VM,

            // we will be reacting to stop the corresponding VM and let the scan

            // process to

            HostVO host = _hostDao.findById(agentId);

            if (host.getType() == Type.ConsoleProxy) {

                String name = host.getName();

                if (s_logger.isInfoEnabled()) {

                    s_logger.info("Console proxy agent disconnected, proxy: " + name);

                }

                if (name != null && name.startsWith("v-")) {

                    String[] tokens = name.split("-");

                    long proxyVmId = 0;

                    try {

                        proxyVmId = Long.parseLong(tokens[1]);

                    } catch (NumberFormatException e) {

                        s_logger.error("Unexpected exception " + e.getMessage(), e);

                        return;

                    }

                    final ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

                    if (proxy != null) {

                        // Disable this feature for now, as it conflicts with

                        // the case of allowing user to reboot console proxy

                        // when rebooting happens, we will receive disconnect

                        // here and we can't enter into stopping process,

                        // as when the rebooted one comes up, it will kick off a

                        // newly started one and trigger the process

                        // continue on forever

                        /*

                         * _capacityScanScheduler.execute(new Runnable() { public void run() {

                         * if(s_logger.isInfoEnabled())

                         * s_logger.info("Stop console proxy " + proxy.getName() +

                         * " VM because of that the agent running inside it has disconnected" );

                         * stopProxy(proxy.getId()); } });

                         */

                    } else {

                        if (s_logger.isInfoEnabled()) {

                            s_logger.info("Console proxy agent disconnected but corresponding console proxy VM no longer exists in DB, proxy: " + name);

                        }

                    }

                } else {

                    assert (false) : "Invalid console proxy name: " + name;

                }

            }

        }

    }

    private boolean reserveStandbyCapacity() {

        ConsoleProxyManagementState state = getManagementState();

        if (state == null || state != ConsoleProxyManagementState.Auto) {

            return false;

        }

        return true;

    }

    private boolean isConsoleProxyVmRequired(long dcId) {

        DataCenterVO dc = _dcDao.findById(dcId);

        _dcDao.loadDetails(dc);

        String cpvmReq = dc.getDetail(ZoneConfig.EnableConsoleProxyVm.key());

        if (cpvmReq != null) {

            return Boolean.parseBoolean(cpvmReq);

        }

        return true;

    }

    private boolean allowToLaunchNew(long dcId) {

        if (!isConsoleProxyVmRequired(dcId)) {

            if (s_logger.isDebugEnabled()) {

                s_logger.debug("Console proxy vm not required in zone " + dcId + " not launching");

            }

            return false;

        }

        List<ConsoleProxyVO> l = _consoleProxyDao.getProxyListInStates(dcId, VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Stopped,

                VirtualMachine.State.Migrating, VirtualMachine.State.Shutdowned, VirtualMachine.State.Unknown);

        String value = _configDao.getValue(Config.ConsoleProxyLaunchMax.key());

        int launchLimit = NumbersUtil.parseInt(value, 10);

        return l.size() < launchLimit;

    }

    private HypervisorType currentHypervisorType(long dcId) {

        List<ConsoleProxyVO> l = _consoleProxyDao.getProxyListInStates(dcId, VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Stopped,

                VirtualMachine.State.Migrating, VirtualMachine.State.Shutdowned, VirtualMachine.State.Unknown);

        return l.size() > 0 ? l.get(0).getHypervisorType() : HypervisorType.Any;

    }

    private boolean checkCapacity(ConsoleProxyLoadInfo proxyCountInfo, ConsoleProxyLoadInfo vmCountInfo) {

        if (proxyCountInfo.getCount() * _capacityPerProxy - vmCountInfo.getCount() <= _standbyCapacity) {

            return false;

        }

        return true;

    }

    private void allocCapacity(long dataCenterId) {

        if (s_logger.isTraceEnabled()) {

            s_logger.trace("Allocate console proxy standby capacity for data center : " + dataCenterId);

        }

        ConsoleProxyVO proxy = null;

        if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {

            try {

                proxy = assignProxyFromStoppedPool(dataCenterId);

                if (proxy == null) {

                    if (s_logger.isInfoEnabled()) {

                        s_logger.info("No stopped console proxy is available, need to allocate a new console proxy");

                    }

                    try {

                        proxy = startNew(dataCenterId);

                    } catch (ConcurrentOperationException e) {

                        s_logger.info("Concurrent Operation caught " + e);

                    }

                } else {

                    if (s_logger.isInfoEnabled()) {

                        s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId());

                    }

                }

            } finally {

                _allocProxyLock.unlock();

            }

        } else {

            if (s_logger.isInfoEnabled()) {

                s_logger.info("Unable to acquire proxy allocation lock, skip for next time");

            }

        }

        if (proxy != null) {

            long proxyVmId = proxy.getId();

            proxy = startProxy(proxyVmId);

            if (proxy != null) {

                if (s_logger.isInfoEnabled()) {

                    s_logger.info("Console proxy " + proxy.getHostName() + " is started");

                }

            }

        }

    }

    public boolean isZoneReady(Map<Long, ZoneHostInfo> zoneHostInfoMap, long dataCenterId) {

        ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId);

        if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) {

            VMTemplateVO template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any);

            if (template == null) {

                if (s_logger.isDebugEnabled()) {

                    s_logger.debug("System vm template is not ready at data center " + dataCenterId + ", wait until it is ready to launch console proxy vm");

                }

                return false;

            }

            TemplateDataStoreVO templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId,

                    Status.DOWNLOADED);

            if (templateHostRef != null) {

                List<Pair<Long, Integer>> l = _consoleProxyDao.getDatacenterStoragePoolHostInfo(dataCenterId, _use_lvm);

                if (l != null && l.size() > 0 && l.get(0).second().intValue() > 0) {

                    return true;

                } else {

                    if (s_logger.isDebugEnabled()) {

                        s_logger.debug("Primary storage is not ready, wait until it is ready to launch console proxy");

                    }

                }

            } else {

                if (s_logger.isDebugEnabled()) {

                    if (template == null) {

                        s_logger.debug("Zone host is ready, but console proxy template is null");

                    } else {

                        s_logger.debug("Zone host is ready, but console proxy template: " + template.getId() + " is not ready on secondary storage.");

                    }

                }

            }

        }

        return false;

    }

    private boolean isZoneHostReady(ZoneHostInfo zoneHostInfo) {

        int expectedFlags = 0;

        if (_use_storage_vm) {

            expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK;

        } else {

            expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ALL_HOST_MASK;

        }

        return (zoneHostInfo.getFlags() & expectedFlags) == expectedFlags;

    }

    private synchronized Map<Long, ZoneHostInfo> getZoneHostInfo() {

        Date cutTime = DateUtil.currentGMTTime();

        List<RunningHostCountInfo> l = _hostDao.getRunningHostCounts(new Date(cutTime.getTime() - ClusterManager.HeartbeatThreshold.value()));

        RunningHostInfoAgregator aggregator = new RunningHostInfoAgregator();

        if (l.size() > 0) {

            for (RunningHostCountInfo countInfo : l) {

                aggregator.aggregate(countInfo);

            }

        }

        return aggregator.getZoneHostInfoMap();

    }

    @Override

    public boolean start() {

        if (s_logger.isInfoEnabled()) {

            s_logger.info("Start console proxy manager");

        }

        return true;

    }

    @Override

    public boolean stop() {

        if (s_logger.isInfoEnabled()) {

            s_logger.info("Stop console proxy manager");

        }

        _loadScanner.stop();

        _allocProxyLock.releaseRef();

        _resourceMgr.unregisterResourceStateAdapter(this.getClass().getSimpleName());

        return true;

    }

    @Override

    public boolean stopProxy(long proxyVmId) {

        ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

        if (proxy == null) {

            if (s_logger.isDebugEnabled()) {

                s_logger.debug("Stopping console proxy failed: console proxy " + proxyVmId + " no longer exists");

            }

            return false;

        }

        try {

            _itMgr.stop(proxy.getUuid());

            return true;

        } catch (ResourceUnavailableException e) {

            s_logger.warn("Stopping console proxy " + proxy.getHostName() + " failed : exception ", e);

            return false;

        } catch (CloudRuntimeException e) {

            s_logger.warn("Unable to stop proxy ", e);

            return false;

        }

    }

    @Override

    @DB

    public void setManagementState(final ConsoleProxyManagementState state) {

        try {

            final ConsoleProxyManagementState lastState = getManagementState();

            if (lastState == null) {

                return;

            }

            if (lastState != state) {

                Transaction.execute(new TransactionCallbackNoReturn() {

                    @Override

                    public void doInTransactionWithoutResult(TransactionStatus status) {

                        _configDao.update(Config.ConsoleProxyManagementLastState.key(), Config.ConsoleProxyManagementLastState.getCategory(), lastState.toString());

                        _configDao.update(Config.ConsoleProxyManagementState.key(), Config.ConsoleProxyManagementState.getCategory(), state.toString());

                    }

                });

            }

        } catch (Throwable e) {

            s_logger.error("Failed to set managment state", e);

        }

    }

    @Override

    public ConsoleProxyManagementState getManagementState() {

        String value = _configDao.getValue(Config.ConsoleProxyManagementState.key());

        if (value != null) {

            ConsoleProxyManagementState state = ConsoleProxyManagementState.valueOf(value);

            if (state == null) {

                s_logger.error("Invalid console proxy management state: " + value);

            }

            return state;

        }

        s_logger.error("Invalid console proxy management state: " + value);

        return null;

    }

    @Override

    @DB

    public void resumeLastManagementState() {

        try {

            ConsoleProxyManagementState state = getManagementState();

            ConsoleProxyManagementState lastState = getLastManagementState();

            if (lastState == null) {

                return;

            }

            if (lastState != state) {

                _configDao.update(Config.ConsoleProxyManagementState.key(), Config.ConsoleProxyManagementState.getCategory(), lastState.toString());

            }

        } catch (Throwable e) {

            s_logger.error("Failed to resume last management state", e);

        }

    }

    private ConsoleProxyManagementState getLastManagementState() {

        String value = _configDao.getValue(Config.ConsoleProxyManagementLastState.key());

        if (value != null) {

            ConsoleProxyManagementState state = ConsoleProxyManagementState.valueOf(value);

            if (state == null) {

                s_logger.error("Invalid console proxy management state: " + value);

            }

            return state;

        }

        s_logger.error("Invalid console proxy management state: " + value);

        return null;

    }

    @Override

    public boolean rebootProxy(long proxyVmId) {

        final ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);

        if (proxy == null || proxy.getState() == State.Destroyed) {

            return false;

        }

        if (proxy.getState() == State.Running && proxy.getHostId() != null) {

            final RebootCommand cmd = new RebootCommand(proxy.getInstanceName());

            final Answer answer = _agentMgr.easySend(proxy.getHostId(), cmd);

            if (answer != null && answer.getResult()) {

                if (s_logger.isDebugEnabled()) {

                    s_logger.debug("Successfully reboot console proxy " + proxy.getHostName());

                }

                SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,

                        new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_REBOOTED, proxy.getDataCenterId(), proxy.getId(), proxy, null));

                return true;

            } else {

                if (s_logger.isDebugEnabled()) {

                    s_logger.debug("failed to reboot console proxy : " + proxy.getHostName());

                }

                return false;

            }

        } else {

            return startProxy(proxyVmId) != null;

        }

    }

    @Override

    public boolean destroyProxy(long vmId) {

        ConsoleProxyVO proxy = _consoleProxyDao.findById(vmId);

        try {

            //expunge the vm

            _itMgr.expunge(proxy.getUuid());

            proxy.setPublicIpAddress(null);

            proxy.setPublicMacAddress(null);

            proxy.setPublicNetmask(null);

            proxy.setPrivateMacAddress(null);

            proxy.setPrivateIpAddress(null);

            _consoleProxyDao.update(proxy.getId(), proxy);

            _consoleProxyDao.remove(vmId);