*   Copyright (C) 2012 by H-Store Project                                 *

*   Brown University                                                      *

*   Massachusetts Institute of Technology                                 *

*   Yale University                                                       *

*                                                                         *

*   Permission is hereby granted, free of charge, to any person obtaining *

*   a copy of this software and associated documentation files (the       *

*   "Software"), to deal in the Software without restriction, including   *

*   without limitation the rights to use, copy, modify, merge, publish,   *

*   distribute, sublicense, and/or sell copies of the Software, and to    *

*   permit persons to whom the Software is furnished to do so, subject to *

*   the following conditions:                                             *

*                                                                         *

*   The above copyright notice and this permission notice shall be        *

*   included in all copies or substantial portions of the Software.       *

*                                                                         *

*   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       *

*   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    *

*   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*

*   IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR     *

*   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, *

*   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR *

*   OTHER DEALINGS IN THE SOFTWARE.                                       *

***************************************************************************/

* THE ALL POWERFUL H-STORE SITE!

* This is the central hub for a site and all of its partitions

* All incoming transactions come into this and all transactions leave through this

* @author pavlo

*/

    public static final Logger LOG = Logger.getLogger(HStoreSite.class);

    private static final LoggerBoolean debug = new LoggerBoolean();

    private static final LoggerBoolean trace = new LoggerBoolean();

    static {

        LoggerUtil.setupLogging();

        LoggerUtil.attachObserver(LOG, debug, trace);

    }

   

    // ----------------------------------------------------------------------------

    // INSTANCE MEMBERS

    // ----------------------------------------------------------------------------

    /**

     * The H-Store Configuration Object

     */

    private final HStoreConf hstore_conf;

    /** Catalog Stuff **/

    private long instanceId;

    private final CatalogContext catalogContext;

    private final Host catalog_host;

    private final Site catalog_site;

    private final int site_id;

    private final String site_name;

   

    /**

     * This buffer pool is used to serialize ClientResponses to send back

     * to clients.

     */

    private final DBBPool buffer_pool = new DBBPool(false, false);

   

    /**

     * Incoming request deserializer

     */

    private final ThreadLocal<FastDeserializer> incomingDeserializers = new ThreadLocal<FastDeserializer>() {

        @Override

        protected FastDeserializer initialValue() {

            return (new FastDeserializer(new byte[0]));

        }

    };

   

    /**

     * Outgoing response serializers

     */

    private final ThreadLocal<FastSerializer> outgoingSerializers = new ThreadLocal<FastSerializer>() {

        @Override

        protected FastSerializer initialValue() {

            return (new FastSerializer(HStoreSite.this.buffer_pool));

        }

    };

   

    /**

     * This is the object that we use to generate unqiue txn ids used by our

     * H-Store specific code. There can either be a single manager for the entire site,

     * or we can use one per partition.

     * @see HStoreConf.site.txn_partition_id_managers

     */

    private final TransactionIdManager txnIdManagers[];

    /**

     * The TransactionInitializer is used to figure out what txns will do

     *  before we start executing them

     */

    private final TransactionInitializer txnInitializer;

   

    /**

     * This class determines what partitions transactions/queries will

     * need to execute on based on their input parameters.

     */

    private final PartitionEstimator p_estimator;

    private final AbstractHasher hasher;

   

    /**

     * Keep track of which txns that we have in-flight right now

     */

    private final Map<Long, AbstractTransaction> inflight_txns =

                        new ConcurrentHashMap<Long, AbstractTransaction>();

   

    /**

     * Queues for transactions that are ready to be cleaned up and deleted

     * There is one queue for each Status type

     */

    private final Map<Status, Queue<Long>> deletable_txns = new HashMap<Status, Queue<Long>>();

   

    /**

     * The list of the last txn ids that were successfully deleted

     * This is primarily used for debugging

     */

    private final CircularFifoBuffer<String> deletable_last = new CircularFifoBuffer<String>(10);

   

    /**

     * This TransactionEstimator is a stand-in for transactions that need to access

     * this partition but who are running at some other node in the cluster.

     */

    private final RemoteEstimator remoteTxnEstimator;

   

    // ----------------------------------------------------------------------------

    // STATS STUFF

    // ----------------------------------------------------------------------------

   

    private final StatsAgent statsAgent = new StatsAgent();

    private TransactionProfilerStats txnProfilerStats;

    private MemoryStats memoryStats;

   

    // ----------------------------------------------------------------------------

    // NETWORKING STUFF

    // ----------------------------------------------------------------------------

   

    /**

     * This thread is responsible for listening for incoming txn requests from

     * clients. It will then forward the request to HStoreSite.procedureInvocation()

     */

//    private final NIOEventLoop procEventLoops[];

   

    private final VoltNetwork voltNetwork;

    private ClientInterface clientInterface;

   

    // ----------------------------------------------------------------------------

    // TRANSACTION COORDINATOR/PROCESSING THREADS

    // ----------------------------------------------------------------------------

   

    /**

     * This manager is used to pin threads to specific CPU cores

     */

    private final HStoreThreadManager threadManager;

   

    /**

     * PartitionExecutors

     * These are the single-threaded execution engines that have exclusive

     * access to a partition. Any transaction that needs to access data at a partition

     * will have to first get queued up by one of these executors.

     */

    private final PartitionExecutor executors[];

    private final Thread executor_threads[];

   

    /**

     * DependencyTrackers

     * One per partition.

     */

    private final DependencyTracker depTrackers[];

   

    /**

     * The queue manager is responsible for deciding what distributed transaction

     * is allowed to acquire the locks for each partition. It can also requeue

     * restart transactions.

     */

    private final TransactionQueueManager txnQueueManager;

   

    /**

     * The HStoreCoordinator is responsible for communicating with other HStoreSites

     * in the cluster to execute distributed transactions.

     * NOTE: We will bind this variable after construction so that we can inject some

     * testing code as needed.

     */

    private HStoreCoordinator hstore_coordinator;

    /**

     * TransactionPreProcessor Threads

     */

    private List<TransactionPreProcessor> preProcessors = null;

    private BlockingQueue<Pair<ByteBuffer, RpcCallback<ClientResponseImpl>>> preProcessorQueue = null;

   

    /**

     * TransactionPostProcessor Thread

     * These threads allow a PartitionExecutor to send back ClientResponses back to

     * the clients without blocking

     */

    private List<TransactionPostProcessor> postProcessors = null;

    private BlockingQueue<Object[]> postProcessorQueue = null;

   

    /**

     * Transaction Handle Cleaner

     */

    private final List<TransactionCleaner> txnCleaners = new ArrayList<TransactionCleaner>();

   

    /**

     * MapReduceHelperThread

     */

    private boolean mr_helper_started = false;

    private final MapReduceHelperThread mr_helper;

   

    /**

     * Transaction Command Logger (WAL)

     */

    private final CommandLogWriter commandLogger;

    /**

     * AdHoc: This thread waits for AdHoc queries.

     */

    private boolean adhoc_helper_started = false;

    private final AsyncCompilerWorkThread asyncCompilerWorkThread;

   

    /**

     * Anti-Cache Abstraction Layer

     */

    private final AntiCacheManager anticacheManager;

   

    /**

     * This catches any exceptions that are thrown in the various

     * threads spawned by this HStoreSite

     */

    private final EventObservableExceptionHandler exceptionHandler = new EventObservableExceptionHandler();

   

    // ----------------------------------------------------------------------------

    // INTERNAL STATE OBSERVABLES

    // ----------------------------------------------------------------------------

   

    /**

     * EventObservable for when the HStoreSite is finished initializing

     * and is now ready to execute transactions.

     */

    private boolean ready = false;

    private final EventObservable<HStoreSite> ready_observable = new EventObservable<HStoreSite>();

   

    /**

     * EventObservable for when we receive the first non-sysproc stored procedure

     * Other components of the system can attach to the EventObservable to be told when this occurs

     */

    private boolean startWorkload = false;

    private final EventObservable<HStoreSite> startWorkload_observable =

                        new EventObservable<HStoreSite>();

   

    /**

     * EventObservable for when the HStoreSite has been told that it needs to shutdown.

     */

    private Shutdownable.ShutdownState shutdown_state = ShutdownState.INITIALIZED;

    private final EventObservable<Object> prepare_observable = new EventObservable<Object>();

    private final EventObservable<Object> shutdown_observable = new EventObservable<Object>();

   

    // ----------------------------------------------------------------------------

    // PARTITION SPECIFIC MEMBERS

    // ----------------------------------------------------------------------------

   

    /**

     * Collection of local partitions managed at this HStoreSite

     */

    private final PartitionSet local_partitions = new PartitionSet();

   

    /**

     * PartitionId -> Internal Offset

     * This is so that we don't have to keep long arrays of local partition information

     */

    private final int local_partition_offsets[];

   

    // ----------------------------------------------------------------------------

    // TRANSACTION ESTIMATION

    // ----------------------------------------------------------------------------

    /**

     * Estimation Thresholds

     */

    private EstimationThresholds thresholds = new EstimationThresholds(); // default values

   

    // ----------------------------------------------------------------------------

    // STATUS + PROFILING MEMBERS

    // ----------------------------------------------------------------------------

    /**

     * Status Monitor

     */

    private final HStoreSiteStatus status_monitor;

   

    /**

     * Profiler

     */

    private HStoreSiteProfiler profiler = new HStoreSiteProfiler();

   

    // ----------------------------------------------------------------------------

    // CACHED STRINGS

    // ----------------------------------------------------------------------------

   

    private final String REJECTION_MESSAGE;   

   

    // ----------------------------------------------------------------------------   

    // ARIES

    // ----------------------------------------------------------------------------

    private AriesLog m_ariesLog = null;

       

    private String m_ariesLogFileName = null;   

    //XXX Must match with AriesLogProxy

    private final String m_ariesDefaultLogFileName = "aries.log";

   

    @SuppressWarnings("unused")

    private VoltLogger m_recoveryLog = null;   

   

    public AriesLog getAriesLogger() {

        return m_ariesLog;

    }

    public String getAriesLogFileName() {

        return m_ariesLogFileName;

    }

    // ----------------------------------------------------------------------------

    // CONSTRUCTOR

    // ----------------------------------------------------------------------------

   

    /**

     * Constructor

     * @param coordinators

     * @param p_estimator

     */

    protected HStoreSite(int site_id, CatalogContext catalogContext, HStoreConf hstore_conf) {

        assert(hstore_conf != null);

        assert(catalogContext != null);

        this.hstore_conf = hstore_conf;

        this.catalogContext = catalogContext;

       

        this.catalog_site = this.catalogContext.getSiteById(site_id);

        if (this.catalog_site == null) throw new RuntimeException("Invalid site #" + site_id);

       

        this.catalog_host = this.catalog_site.getHost();

        this.site_id = this.catalog_site.getId();

        this.site_name = HStoreThreadManager.getThreadName(this.site_id, null);

       

        final int num_partitions = this.catalogContext.numberOfPartitions;

        this.local_partitions.addAll(CatalogUtil.getLocalPartitionIds(catalog_site));

        int num_local_partitions = this.local_partitions.size();

       

         for (Status s : Status.values()) {

             this.deletable_txns.put(s, new ConcurrentLinkedQueue<Long>());

         } // FOR

       

        this.executors = new PartitionExecutor[num_partitions];

        this.executor_threads = new Thread[num_partitions];

        this.depTrackers = new DependencyTracker[num_partitions];

       

        // Get the hasher we will use for this HStoreSite

        this.hasher = ClassUtil.newInstance(hstore_conf.global.hasher_class,

                                             new Object[]{ this.catalogContext, num_partitions },

                                             new Class<?>[]{ CatalogContext.class, int.class });

        this.p_estimator = new PartitionEstimator(this.catalogContext, this.hasher);

        this.remoteTxnEstimator = new RemoteEstimator(this.p_estimator);

       

        // ARIES

        if(hstore_conf.site.aries){

            // Don't use both recovery modes

            assert(hstore_conf.site.snapshot == false);

            LOG.warn("Starting ARIES recovery at site");          

            String siteName = HStoreThreadManager.formatSiteName(this.getSiteId());

            String ariesSiteDirPath = hstore_conf.site.aries_dir + File.separatorChar + siteName + File.separatorChar;

          

            this.m_ariesLogFileName =  ariesSiteDirPath + m_ariesDefaultLogFileName ;

            int numPartitionsPerSite =   this.catalog_site.getPartitions().size();

            int numSites = this.catalogContext.numberOfSites;

            LOG.warn("ARIES : Log Native creation :: numSites : "+numSites+" numPartitionsPerSite : "+numPartitionsPerSite);          

            this.m_ariesLog = new AriesLogNative(numSites, numPartitionsPerSite, this.m_ariesLogFileName);

            this.m_recoveryLog = new VoltLogger("RECOVERY");

        }

                       

        // **IMPORTANT**

        // Always clear out the CatalogUtil and BatchPlanner before we start our new HStoreSite

        // TODO: Move this cache information into CatalogContext

        CatalogUtil.clearCache(this.catalogContext.database);

        BatchPlanner.clear(this.catalogContext.numberOfPartitions);

        TransactionCounter.resetAll(this.catalogContext);