* Copyright (C) 2008-2014 VoltDB Inc.

*

* This program is free software: you can redistribute it and/or modify

* it under the terms of the GNU Affero General Public License as

* published by the Free Software Foundation, either version 3 of the

* License, or (at your option) any later version.

*

* This program is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

* GNU Affero General Public License for more details.

*

* You should have received a copy of the GNU Affero General Public License

* along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.

*/

* Represents VoltDB's connection to client libraries outside the cluster.

* This class accepts new connections and manages existing connections through

* <code>ClientConnection</code> instances.

*

*/

    static long TOPOLOGY_CHANGE_CHECK_MS = Long.getLong("TOPOLOGY_CHANGE_CHECK_MS", 5000);

    static long AUTH_TIMEOUT_MS = Long.getLong("AUTH_TIMEOUT_MS", 30000);

    //Same as in Distributer.java

    public static final long ASYNC_TOPO_HANDLE = Long.MAX_VALUE - 1;

    // reasons a connection can fail

    public static final byte AUTHENTICATION_FAILURE = Constants.AUTHENTICATION_FAILURE;

    public static final byte MAX_CONNECTIONS_LIMIT_ERROR = Constants.MAX_CONNECTIONS_LIMIT_ERROR;

    public static final byte WIRE_PROTOCOL_TIMEOUT_ERROR = Constants.WIRE_PROTOCOL_TIMEOUT_ERROR;

    public static final byte WIRE_PROTOCOL_FORMAT_ERROR = Constants.WIRE_PROTOCOL_FORMAT_ERROR;

    public static final byte AUTHENTICATION_FAILURE_DUE_TO_REJOIN = Constants.AUTHENTICATION_FAILURE_DUE_TO_REJOIN;

    public static final byte EXPORT_DISABLED_REJECTION = Constants.EXPORT_DISABLED_REJECTION;

    // authentication handshake codes

    public static final byte AUTH_HANDSHAKE_VERSION = Constants.AUTH_HANDSHAKE_VERSION;

    public static final byte AUTH_SERVICE_NAME = Constants.AUTH_SERVICE_NAME;

    public static final byte AUTH_HANDSHAKE = Constants.AUTH_HANDSHAKE;

    // connection IDs used by internal adapters

    public static final long RESTORE_AGENT_CID          = Long.MIN_VALUE + 1;

    public static final long SNAPSHOT_UTIL_CID          = Long.MIN_VALUE + 2;

    public static final long ELASTIC_JOIN_CID           = Long.MIN_VALUE + 3;

    // Leave CL_REPLAY_BASE_CID at the end, it uses this as a base and generates more cids

    public static final long CL_REPLAY_BASE_CID         = Long.MIN_VALUE + 100;

    private static final VoltLogger log = new VoltLogger(ClientInterface.class.getName());

    private static final VoltLogger authLog = new VoltLogger("AUTH");

    private static final VoltLogger hostLog = new VoltLogger("HOST");

    private static final VoltLogger networkLog = new VoltLogger("NETWORK");

    private final ClientAcceptor m_acceptor;

    private ClientAcceptor m_adminAcceptor;

    private final SnapshotDaemon m_snapshotDaemon = new SnapshotDaemon();

    private final SnapshotDaemonAdapter m_snapshotDaemonAdapter = new SnapshotDaemonAdapter();

    // Atomically allows the catalog reference to change between access

    private final AtomicReference<CatalogContext> m_catalogContext = new AtomicReference<CatalogContext>(null);

    /**

     * Counter of the number of client connections. Used to enforce a limit on the maximum number of connections

     */

    private final AtomicInteger m_numConnections = new AtomicInteger(0);

    /**

     * ZooKeeper is used for @Promote to trigger a truncation snapshot.

     */

    ZooKeeper m_zk;

    /**

     * The CIHM is unique to the connection and the ACG is shared by all connections

     * serviced by the associated network thread. They are paired so as to only do a single

     * lookup.

     */

    private final ConcurrentHashMap<Long, ClientInterfaceHandleManager> m_cihm =

            new ConcurrentHashMap<Long, ClientInterfaceHandleManager>(2048, .75f, 128);

    private final RateLimitedClientNotifier m_notifier = new RateLimitedClientNotifier();

    private final Cartographer m_cartographer;

    //This validator will check permissions in AUTH system.

    private final PermissionValidator m_permissionValidator = new PermissionValidator();

    //This validator will verify params or per procedure invocation vaidation.

    private final InvocationValidator m_invocationValidator;

    /*

     * Allow the async compiler thread to immediately process completed planning tasks

     * without waiting for the periodic work thread to poll the mailbox.

     */

    private final  AsyncCompilerWorkCompletionHandler m_adhocCompletionHandler = new AsyncCompilerWorkCompletionHandler() {

        @Override

        public void onCompletion(AsyncCompilerResult result) {

            processFinishedCompilerWork(result);

        }

    };

    /*

     * This list of ACGs is iterated to retrieve initiator statistics in IV2.

     * They are thread local, and the ACG happens to be thread local, and if you squint

     * right admission control seems like a reasonable place to store stats about

     * what has been admitted.

     */

    private final CopyOnWriteArrayList<AdmissionControlGroup> m_allACGs =

            new CopyOnWriteArrayList<AdmissionControlGroup>();

    /*

     * A thread local is a convenient way to keep the ACG out of volt core. The lookup is paired

     * with the CIHM in m_connectionSpecificStuff in fast path code.

     *

     * With these initial values if you have 16 hardware threads you will end up with 4  ACGs

     * and 32 megs/4k transactions and double that with 32 threads.

     */

    private final ThreadLocal<AdmissionControlGroup> m_acg = new ThreadLocal<AdmissionControlGroup>() {

        @Override

        public AdmissionControlGroup initialValue() {

            AdmissionControlGroup acg = new AdmissionControlGroup( 1024 * 1024 * 8, 1000);

            m_allACGs.add(acg);

            return acg;

        }

    };

    // IV2 doesn't use this at all. Leave it here for now for legacy.

    private final int m_allPartitions[];

    private ImmutableMap<Integer, Long> m_localReplicas = ImmutableMap.<Integer, Long>builder().build();

    final long m_siteId;

    final long m_plannerSiteId;

    final Mailbox m_mailbox;

    /**

     * This boolean allows the DTXN to communicate to the

     * ClientInputHandler the presence of DTXN backpressure.

     * The m_connections ArrayList is used as the synchronization

     * point to ensure that modifications to read interest ops

     * that are based on the status of this information are atomic.

     * Additionally each connection must be synchronized on before modification

     * because the disabling of read selection for an individual connection

     * due to backpressure (not DTXN backpressure, client backpressure due to a client

     * that refuses to read responses) occurs inside the SimpleDTXNInitiator which

     * doesn't have access to m_connections

     */

    private final boolean m_hasDTXNBackPressure = false;

    // MAX_CONNECTIONS is updated to be (FD LIMIT - 300) after startup

    private final AtomicInteger MAX_CONNECTIONS = new AtomicInteger(800);

    private ScheduledFuture<?> m_maxConnectionUpdater;

    private final boolean m_isConfiguredForHSQL;

    public class ClientAcceptor implements Runnable {

        private final int m_port;

        private final ServerSocketChannel m_serverSocket;

        private final VoltNetworkPool m_network;

        private volatile boolean m_running = true;

        private Thread m_thread = null;

        private final boolean m_isAdmin;

        private final InetAddress m_interface;

        /**

         * Used a cached thread pool to accept new connections.

         */

        private final ExecutorService m_executor = CoreUtils.getBoundedThreadPoolExecutor(128, 10L, TimeUnit.SECONDS,

                        CoreUtils.getThreadFactory("Client authentication threads", "Client authenticator"));

        ClientAcceptor(InetAddress intf, int port, VoltNetworkPool network, boolean isAdmin)

        {

            m_interface = intf;

            m_network = network;

            m_port = port;

            m_isAdmin = isAdmin;

            ServerSocketChannel socket;

            try {

                socket = ServerSocketChannel.open();

            } catch (IOException e) {

                if (m_isAdmin) {

                    hostLog.fatal("Failed to open admin wire protocol listener on port "

                            + m_port + "(" + e.getMessage() + ")");

                }

                else {

                    hostLog.fatal("Failed to open native wire protocol listener on port "

                            + m_port + "(" + e.getMessage() + ")");

                }

                throw new RuntimeException(e);

            }

            m_serverSocket = socket;

        }

        public void start() throws IOException {

            if (m_thread != null) {

                throw new IllegalStateException("A thread for this ClientAcceptor is already running");

            }

            if (!m_serverSocket.socket().isBound()) {

                try {

                    if (m_interface != null) {

                        m_serverSocket.socket().bind(new InetSocketAddress(m_interface, m_port));

                    } else {

                        m_serverSocket.socket().bind(new InetSocketAddress(m_port));

                    }

                }

                catch (IOException e) {

                    String msg = "Client interface failed to bind to"

                            + (m_isAdmin ? " Admin " : " ") + "port: " + m_port;

                    MiscUtils.printPortsInUse(hostLog);

                    VoltDB.crashLocalVoltDB(msg, false, e);

                }

            }

            m_running = true;

            String threadName = m_isAdmin ? "AdminPort connection acceptor" : "ClientPort connection acceptor";

            m_thread = new Thread( null, this, threadName, 262144);

            m_thread.setDaemon(true);

            m_thread.start();

        }

        public void shutdown() throws InterruptedException {

            //sync prevents interruption while shuttown down executor

            if (m_thread != null) {

                synchronized (this) {

                    m_running = false;

                    m_thread.interrupt();

                }

                m_thread.join();

            }

        }

        //Thread for Running authentication of client.

        class AuthRunnable implements Runnable {

            final SocketChannel m_socket;

            AuthRunnable(SocketChannel socket) {

                this.m_socket = socket;

            }

            @Override

            public void run() {

                if (m_socket != null) {

                    boolean success = false;

                    //Populated on timeout

                    AtomicReference<String> timeoutRef = new AtomicReference<String>();

                    try {

                        final InputHandler handler = authenticate(m_socket, timeoutRef);

                        if (handler != null) {

                            m_socket.configureBlocking(false);

                            if (handler instanceof ClientInputHandler) {

                                m_socket.socket().setTcpNoDelay(true);

                            }

                            m_socket.socket().setKeepAlive(true);

                            if (handler instanceof ClientInputHandler) {

                                m_network.registerChannel(

                                                m_socket,

                                                handler,

                                                0,

                                                ReverseDNSPolicy.ASYNCHRONOUS);

                                /*

                                 * If IV2 is enabled the logic initially enabling read is

                                 * in the started method of the InputHandler

                                 */

                            } else {

                                m_network.registerChannel(

                                        m_socket,

                                        handler,

                                        SelectionKey.OP_READ,

                                        ReverseDNSPolicy.ASYNCHRONOUS);

                            }

                            success = true;

                        }

                    } catch (Exception e) {

                        try {

                            m_socket.close();

                        } catch (IOException e1) {

                            //Don't care connection is already lost anyways

                        }

                        if (m_running) {

                            if (timeoutRef.get() != null) {

                                hostLog.warn(timeoutRef.get());

                            } else {

                                hostLog.warn("Exception authenticating and "

                                        + "registering user in ClientAcceptor", e);

                            }

                        }

                    } finally {

                        if (!success) {

                            m_numConnections.decrementAndGet();

                        }

                    }

                }

            }

        }

        @Override

        public void run() {

            try {

                do {

                    final SocketChannel socket;

                    try

                    {

                        socket = m_serverSocket.accept();

                    }

                    catch (IOException ioe)

                    {

                        if (ioe.getMessage() != null &&

                            ioe.getMessage().contains("Too many open files"))

                        {

                            networkLog.warn("Rejected accepting new connection due to too many open files");

                            continue;

                        }

                        else

                        {

                            throw ioe;

                        }

                    }

                    /*

                     * Enforce a limit on the maximum number of connections

                     */

                    if (m_numConnections.get() >= MAX_CONNECTIONS.get()) {

                        networkLog.warn("Rejected connection from " +

                                socket.socket().getRemoteSocketAddress() +

                                " because the connection limit of " + MAX_CONNECTIONS + " has been reached");

                        try {

                            /*

                             * Send rejection message with reason code

                             */

                            final ByteBuffer b = ByteBuffer.allocate(1);

                            b.put(MAX_CONNECTIONS_LIMIT_ERROR);

                            b.flip();

                            socket.configureBlocking(true);

                            for (int ii = 0; ii < 4 && b.hasRemaining(); ii++) {

                                socket.write(b);

                            }

                            socket.close();

                        } catch (IOException e) {}//don't care keep running

                        continue;

                    }

                    /*

                     * Increment the number of connections even though this one hasn't been authenticated

                     * so that a flood of connection attempts (with many doomed) will not result in

                     * successful authentication of connections that would put us over the limit.

                     */

                    m_numConnections.incrementAndGet();

                    final AuthRunnable authRunnable = new AuthRunnable(socket);

                    while (true) {

                        try {

                            m_executor.execute(authRunnable);

                            break;

                        } catch (RejectedExecutionException e) {

                            Thread.sleep(1);

                        }

                    }

                } while (m_running);

            } catch (Exception e) {

                if (m_running) {

                    hostLog.error("Exception in ClientAcceptor. The acceptor has died", e);

                }

            } finally {

                try {

                    m_serverSocket.close();

                } catch (IOException e) {

                    hostLog.fatal(null, e);

                }

                //Prevent interruption

                synchronized (this) {

                    Thread.interrupted();

                    m_executor.shutdownNow();

                    try {

                        m_executor.awaitTermination(5, TimeUnit.MINUTES);

                    } catch (InterruptedException e) {

                        String msg = "Client Listener Interrupted while shutting down "

                                + (m_isAdmin ? " Admin " : " ") + "port: " + m_port;

                        VoltDB.crashLocalVoltDB(msg, false, e);

                    }

                }

            }

        }

        /**

         * Attempt to authenticate the user associated with this socket connection

         * @param socket

         * @param timeoutRef Populated with error on timeout

         * @return AuthUser a set of user permissions or null if authentication fails

         * @throws IOException

         */

        private InputHandler

        authenticate(final SocketChannel socket, final AtomicReference<String> timeoutRef) throws IOException

        {

            ByteBuffer responseBuffer = ByteBuffer.allocate(6);

            byte version = (byte)0;

            responseBuffer.putInt(2);//message length

            responseBuffer.put(version);//version

            /*

             * The login message is a length preceded name string followed by a length preceded

             * SHA-1 single hash of the password.

             */

            socket.configureBlocking(true);

            socket.socket().setTcpNoDelay(true);//Greatly speeds up requests hitting the wire

            final ByteBuffer lengthBuffer = ByteBuffer.allocate(4);

            /*

             * Schedule a timeout to close the socket in case there is no response for the timeout

             * period. This will wake up the current thread that is blocked on reading the login message

             */

            final long start = System.currentTimeMillis();

            ScheduledFuture<?> timeoutFuture =

                    VoltDB.instance().schedulePriorityWork(new Runnable() {

                        @Override

                        public void run() {

                            long delta = System.currentTimeMillis() - start;

                            double seconds = delta / 1000.0;

                            StringBuilder sb = new StringBuilder();

                            sb.append("Timed out authenticating client from ");

                            sb.append(socket.socket().getRemoteSocketAddress().toString());

                            sb.append(String.format(" after %.2f seconds (timeout target is %.2f seconds)", seconds, AUTH_TIMEOUT_MS / 1000.0));

                            timeoutRef.set(sb.toString());

                            try {

                                socket.close();

                            } catch (IOException e) {

                                //Don't care

                            }

                        }

                    }, AUTH_TIMEOUT_MS, 0, TimeUnit.MILLISECONDS);

            try {

                while (lengthBuffer.hasRemaining()) {

                    int read = socket.read(lengthBuffer);

                    if (read == -1) {

                        socket.close();

                        timeoutFuture.cancel(false);

                        return null;

                    }

                }

            } catch (AsynchronousCloseException e) {}//This is the timeout firing and closing the channel

            //Didn't get the value. Client isn't going to get anymore time.

            if (lengthBuffer.hasRemaining()) {

                timeoutFuture.cancel(false);

                authLog.debug("Failure to authenticate connection(" + socket.socket().getRemoteSocketAddress() +

                              "): wire protocol violation (timeout reading message length).");

                //Send negative response

                responseBuffer.put(WIRE_PROTOCOL_TIMEOUT_ERROR).flip();

                socket.write(responseBuffer);

                socket.close();

                return null;

            }

            lengthBuffer.flip();

            final int messageLength = lengthBuffer.getInt();

            if (messageLength < 0) {

                timeoutFuture.cancel(false);

                authLog.warn("Failure to authenticate connection(" + socket.socket().getRemoteSocketAddress() +

                             "): wire protocol violation (message length " + messageLength + " is negative).");

                //Send negative response

                responseBuffer.put(WIRE_PROTOCOL_FORMAT_ERROR).flip();

                socket.write(responseBuffer);

                socket.close();

                return null;

            }

            if (messageLength > ((1024 * 1024) * 2)) {

                timeoutFuture.cancel(false);

                authLog.warn("Failure to authenticate connection(" + socket.socket().getRemoteSocketAddress() +

                             "): wire protocol violation (message length " + messageLength + " is too large).");

                //Send negative response

                responseBuffer.put(WIRE_PROTOCOL_FORMAT_ERROR).flip();

                socket.write(responseBuffer);

                socket.close();

                return null;

              }

            final ByteBuffer message = ByteBuffer.allocate(messageLength);

            try {

                while (message.hasRemaining()) {

                    int read = socket.read(message);

                    if (read == -1) {

                        socket.close();

                        timeoutFuture.cancel(false);

                        return null;

                    }

                }

            } catch (AsynchronousCloseException e) {}//This is the timeout firing and closing the channel

            //Didn't get the whole message. Client isn't going to get anymore time.

            if (message.hasRemaining()) {

                timeoutFuture.cancel(false);

                authLog.warn("Failure to authenticate connection(" + socket.socket().getRemoteSocketAddress() +

                             "): wire protocol violation (timeout reading authentication strings).");

                //Send negative response

                responseBuffer.put(WIRE_PROTOCOL_TIMEOUT_ERROR).flip();

                socket.write(responseBuffer);

                socket.close();

                return null;

            }

            /*

             * Since we got the login message, cancel the timeout.

             * If cancellation fails then the socket is dead and the connection lost

             */

            if (!timeoutFuture.cancel(false)) {

                return null;

            }

            message.flip().position(1);//skip version

            FastDeserializer fds = new FastDeserializer(message);

            final String service = fds.readString();

            final String username = fds.readString();

            final byte password[] = new byte[20];

            //We should be left with SHA-1 bytes only.

            if (message.remaining() != 20) {

                authLog.warn("Failure to authenticate connection(" + socket.socket().getRemoteSocketAddress()

                        + "): user " + username + " failed authentication.");

                //Send negative response

                responseBuffer.put(AUTHENTICATION_FAILURE).flip();

                socket.write(responseBuffer);

                socket.close();

                return null;

            }

            message.get(password);

            CatalogContext context = m_catalogContext.get();

            AuthProvider ap = null;

            try {

                ap = AuthProvider.fromService(service);

            } catch (IllegalArgumentException unkownProvider) {

                // handle it bellow

            }

            if (ap == null) {

                //Send negative response

                responseBuffer.put(EXPORT_DISABLED_REJECTION).flip();

                socket.write(responseBuffer);

                socket.close();

                authLog.warn("Rejected user " + username +

                        " attempting to use disabled or unconfigured service " +

                        service + ".");

                authLog.warn("VoltDB Export services are no longer available through clients.");

                return null;

            }

            /*

             * Don't use the auth system during recovery. Not safe to use

             * the node to initiate multi-partition txns during recovery

             */

            if (!VoltDB.instance().rejoining()) {