Package org.voltdb.regressionsuites

Source Code of org.voltdb.regressionsuites.TestStatisticsSuite$RebalanceStatsChecker

/* This file is part of VoltDB.
* Copyright (C) 2008-2014 VoltDB Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/

package org.voltdb.regressionsuites;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import junit.framework.Test;

import org.HdrHistogram_voltpatches.AbstractHistogram;
import org.HdrHistogram_voltpatches.Histogram;
import org.hsqldb_voltpatches.HSQLInterface;
import org.voltcore.utils.CompressionStrategySnappy;
import org.voltdb.BackendTarget;
import org.voltdb.VoltDB;
import org.voltdb.VoltTable;
import org.voltdb.VoltTable.ColumnInfo;
import org.voltdb.VoltType;
import org.voltdb.client.Client;
import org.voltdb.client.ProcCallException;
import org.voltdb.compiler.VoltProjectBuilder;
import org.voltdb.iv2.MpInitiator;
import org.voltdb.join.BalancePartitionsStatistics;
import org.voltdb.utils.MiscUtils;
import org.voltdb_testprocs.regressionsuites.malicious.GoSleep;

public class TestStatisticsSuite extends SaveRestoreBase {

    private static int SITES = 2;
    private static int HOSTS = 3;
    private static int KFACTOR = MiscUtils.isPro() ? 1 : 0;
    private static int PARTITIONS = (SITES * HOSTS) / (KFACTOR + 1);
    private static boolean hasLocalServer = false;

    static final Class<?>[] PROCEDURES =
    {
        GoSleep.class
    };

    public TestStatisticsSuite(String name) {
        super(name);
    }

    // For the provided table, verify that there is a row for each host in the cluster where
    // the column designated by 'columnName' has the value 'rowId'.  For example, for
    // Initiator stats, if columnName is 'PROCEDURE_NAME' and rowId is 'foo', this
    // will verify that the initiator at each node has seen a procedure invocation for 'foo'
    public void validateRowSeenAtAllHosts(VoltTable result, String columnName, String rowId,
            boolean enforceUnique)
    {
        result.resetRowPosition();
        Set<Long> hostsSeen = new HashSet<Long>();
        while (result.advanceRow()) {
            String procName = result.getString(columnName);
            if (procName.equalsIgnoreCase(rowId)) {
                Long thisHostId = result.getLong("HOST_ID");
                if (enforceUnique) {
                    assertFalse("HOST_ID: " + thisHostId + " seen twice in table looking for " + rowId +
                            " in column " + columnName, hostsSeen.contains(thisHostId));
                }
                hostsSeen.add(thisHostId);
            }
        }
        // Before failing the assert, report details of the non-conforming result.
        if (HOSTS != hostsSeen.size()) {
            System.out.println("Something in the following results will fail the assert.");
            result.resetRowPosition();
            while (result.advanceRow()) {
                String procName = result.getString(columnName);
                Long thisHostId = result.getLong("HOST_ID");
                if (procName.equalsIgnoreCase(rowId)) {
                    System.out.println("Found the match at host " + thisHostId + " for proc " + procName +
                                       (hostsSeen.add(thisHostId) ? " added" : " duplicated"));
                } else {
                    System.out.println("Found non-match at host " + thisHostId + " for proc " + procName);
                }
            }
        }
        assertEquals(HOSTS, hostsSeen.size());
    }

    // For the provided table, verify that there is a row for each site in the cluster where
    // the column designated by 'columnName' has the value 'rowId'.  For example, for
    // Table stats, if columnName is 'TABLE_NAME' and rowId is 'foo', this
    // will verify that each site has returned results for table 'foo'
    public boolean validateRowSeenAtAllSites(VoltTable result, String columnName, String rowId,
            boolean enforceUnique)
    {
        result.resetRowPosition();
        Set<Long> sitesSeen = new HashSet<Long>();
        while (result.advanceRow()) {
            String procName = result.getString(columnName);
            if (procName.equalsIgnoreCase(rowId)) {
                long hostId = result.getLong("HOST_ID");
                long thisSiteId = result.getLong("SITE_ID");
                thisSiteId |= hostId << 32;
                if (enforceUnique) {
                    assertFalse("SITE_ID: " + thisSiteId + " seen twice in table looking for " + rowId +
                            " in column " + columnName, sitesSeen.contains(thisSiteId));
                }
                sitesSeen.add(thisSiteId);
            }
        }
        return (HOSTS * SITES) == sitesSeen.size();
    }

    // For the provided table, verify that there is a row for each partition in the cluster where
    // the column designated by 'columnName' has the value 'rowId'.
    public void validateRowSeenAtAllPartitions(VoltTable result, String columnName, String rowId,
            boolean enforceUnique)
    {
        result.resetRowPosition();
        Set<Long> partsSeen = new HashSet<Long>();
        while (result.advanceRow()) {
            String procName = result.getString(columnName);
            if (procName.equalsIgnoreCase(rowId)) {
                long thisPartId = result.getLong("PARTITION_ID");
                if (enforceUnique) {
                    assertFalse("PARTITION_ID: " + thisPartId + " seen twice in table looking for " + rowId +
                            " in column " + columnName, partsSeen.contains(thisPartId));
                }
                partsSeen.add(thisPartId);
            }
        }
        assertEquals(PARTITIONS, partsSeen.size());
    }

    public void testInvalidCalls() throws Exception {
        System.out.println("\n\nTESTING INVALID CALLS\n\n\n");
        Client client = getFullyConnectedClient();
        //
        // invalid selector
        //
        try {
            // No selector at all.
            client.callProcedure("@Statistics");
            fail();
        }
        catch (ProcCallException ex) {
            // All badness gets turned into ProcCallExceptions, so we need
            // to check specifically for this error, otherwise things that
            // crash the cluster also turn into ProcCallExceptions and don't
            // trigger failure (ENG-2347)
            assertEquals("Incorrect number of arguments to @Statistics (expects 2, received 0)",
                         ex.getMessage());
        }
        try {
            // extra stuff
            client.callProcedure("@Statistics", "table", 0, "OHHAI");
            fail();
        }
        catch (ProcCallException ex) {
            assertEquals("Incorrect number of arguments to @Statistics (expects 2, received 3)",
                         ex.getMessage());
        }
        try {
            // Invalid selector
            client.callProcedure("@Statistics", "garbage", 0);
            fail();
        }
        catch (ProcCallException ex) {}

    }

    public void testLatencyStatistics() throws Exception {
        System.out.println("\n\nTESTING LATENCY STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[5];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("HISTOGRAM", VoltType.VARBINARY);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        // Do some stuff to generate some latency stats
        for (int i = 0; i < SITES * HOSTS; i++) {
            results = client.callProcedure("NEW_ORDER.insert", i).getResults();
        }
        results = client.callProcedure("@Statistics", "LATENCY", 0).getResults();
        // one aggregate table returned
        assertEquals(1, results.length);
        System.out.println("Test latency table: " + results[0].toString());

        validateSchema(results[0], expectedTable);
        // should have at least one row from each host
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
        // actually, there are 26 rows per host so:
        assertEquals(HOSTS, results[0].getRowCount());
        // Check for non-zero invocations (ENG-4668)
        long invocations = 0;
        results[0].resetRowPosition();
        while (results[0].advanceRow()) {
            byte histogramBytes[] = results[0].getVarbinary("HISTOGRAM");
            Histogram h = AbstractHistogram.fromCompressedBytes(histogramBytes, CompressionStrategySnappy.INSTANCE);
            invocations += h.getHistogramData().getTotalCount();
        }
        assertTrue(invocations > 0);
    }

    public void testInitiatorStatistics() throws Exception {
        System.out.println("\n\nTESTING INITIATOR STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[13];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("CONNECTION_ID", VoltType.BIGINT);
        expectedSchema[5] = new ColumnInfo("CONNECTION_HOSTNAME", VoltType.STRING);
        expectedSchema[6] = new ColumnInfo("PROCEDURE_NAME", VoltType.STRING);
        expectedSchema[7] = new ColumnInfo("INVOCATIONS", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("AVG_EXECUTION_TIME", VoltType.INTEGER);
        expectedSchema[9] = new ColumnInfo("MIN_EXECUTION_TIME", VoltType.INTEGER);
        expectedSchema[10] = new ColumnInfo("MAX_EXECUTION_TIME", VoltType.INTEGER);
        expectedSchema[11] = new ColumnInfo("ABORTS", VoltType.BIGINT);
        expectedSchema[12] = new ColumnInfo("FAILURES", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        //
        // initiator selector
        //
        VoltTable results[] = null;
        // This should get us an invocation at each host
        for (int i = 0; i < 1000; i++) {
            results = client.callProcedure("NEW_ORDER.insert", i).getResults();
        }
        results = client.callProcedure("@Statistics", "INITIATOR", 0).getResults();
        // one aggregate table returned
        assertEquals(1, results.length);
        System.out.println("Test initiators table: " + results[0].toString());
        // Check the schema
        validateSchema(results[0], expectedTable);
        // One WAREHOUSE.select row per host
        assertEquals(HOSTS, results[0].getRowCount());

        // Verify the invocation counts
        int counts = 0;
        while (results[0].advanceRow()) {
            String procName = results[0].getString("PROCEDURE_NAME");
            if (procName.equals("@SystemCatalog")) {
                // One for each connection from the client
                assertEquals(HOSTS, results[0].getLong("INVOCATIONS"));
            } else if (procName.equals("NEW_ORDER.insert")) {
                counts += results[0].getLong("INVOCATIONS");
            }
        }
        assertEquals(1000, counts);
        // verify that each node saw a NEW_ORDER.insert initiation
        validateRowSeenAtAllHosts(results[0], "PROCEDURE_NAME", "NEW_ORDER.insert", true);
    }

    public void testPartitionCount() throws Exception {
        System.out.println("\n\nTESTING PARTITION COUNT\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[4];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("PARTITION_COUNT", VoltType.INTEGER);
        VoltTable expectedTable = new VoltTable(expectedSchema);
        VoltTable[] results = null;

        results = client.callProcedure("@Statistics", "PARTITIONCOUNT", 0).getResults();
        // Only one table returned
        assertEquals(1, results.length);
        validateSchema(results[0], expectedTable);
        // Should only get one row, total
        assertEquals(1, results[0].getRowCount());
        results[0].advanceRow();
        int partCount = (int)results[0].getLong("PARTITION_COUNT");
        assertEquals(PARTITIONS, partCount);
    }

    public void testTableStatistics() throws Exception {
        System.out.println("\n\nTESTING TABLE STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[13];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("PARTITION_ID", VoltType.BIGINT);
        expectedSchema[5] = new ColumnInfo("TABLE_NAME", VoltType.STRING);
        expectedSchema[6] = new ColumnInfo("TABLE_TYPE", VoltType.STRING);
        expectedSchema[7] = new ColumnInfo("TUPLE_COUNT", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("TUPLE_ALLOCATED_MEMORY", VoltType.INTEGER);
        expectedSchema[9] = new ColumnInfo("TUPLE_DATA_MEMORY", VoltType.INTEGER);
        expectedSchema[10] = new ColumnInfo("STRING_DATA_MEMORY", VoltType.INTEGER);
        expectedSchema[11] = new ColumnInfo("TUPLE_LIMIT", VoltType.INTEGER);
        expectedSchema[12] = new ColumnInfo("PERCENT_FULL", VoltType.INTEGER);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;
        boolean success = false;
        long start = System.currentTimeMillis();
        while (!success) {
            if (System.currentTimeMillis() - start > 60000) fail("Took too long");
            success = true;
            // table
            //
            results = client.callProcedure("@Statistics", "table", 0).getResults();
            System.out.println("Test statistics table: " + results[0].toString());
            // one aggregate table returned
            assertEquals(1, results.length);
            validateSchema(results[0], expectedTable);
            // with 10 rows per site. Can be two values depending on the test scenario of cluster vs. local.
            if (HOSTS * SITES * 3 != results[0].getRowCount()) {
                success = false;
            }
            // Validate that each site returns a result for each table
            if (success) {
                success = validateRowSeenAtAllSites(results[0], "TABLE_NAME", "WAREHOUSE", true);
            }
            if (success) {
                success = validateRowSeenAtAllSites(results[0], "TABLE_NAME", "NEW_ORDER", true);
            }
            if (success) {
                validateRowSeenAtAllSites(results[0], "TABLE_NAME", "ITEM", true);
            }
            if (success) break;
        }
    }

    public void testIndexStatistics() throws Exception {
        System.out.println("\n\nTESTING INDEX STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[12];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("PARTITION_ID", VoltType.BIGINT);
        expectedSchema[5] = new ColumnInfo("INDEX_NAME", VoltType.STRING);
        expectedSchema[6] = new ColumnInfo("TABLE_NAME", VoltType.STRING);
        expectedSchema[7] = new ColumnInfo("INDEX_TYPE", VoltType.STRING);
        expectedSchema[8] = new ColumnInfo("IS_UNIQUE", VoltType.TINYINT);
        expectedSchema[9] = new ColumnInfo("IS_COUNTABLE", VoltType.TINYINT);
        expectedSchema[10] = new ColumnInfo("ENTRY_COUNT", VoltType.BIGINT);
        expectedSchema[11] = new ColumnInfo("MEMORY_ESTIMATE", VoltType.INTEGER);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        boolean success = false;
        long start = System.currentTimeMillis();
        while (!success) {
            if (System.currentTimeMillis() - start > 60000) fail("Took too long");
            success = true;
            results = client.callProcedure("@Statistics", "index", 0).getResults();
            System.out.println("Index results: " + results[0].toString());
            assertEquals(1, results.length);
            validateSchema(results[0], expectedTable);
            if (success) {
                success = validateRowSeenAtAllSites(results[0], "INDEX_NAME",
                        HSQLInterface.AUTO_GEN_CONSTRAINT_WRAPPER_PREFIX + "W_PK_TREE", true);
            }
            if (success) {
                success = validateRowSeenAtAllSites(results[0], "INDEX_NAME",
                        HSQLInterface.AUTO_GEN_CONSTRAINT_WRAPPER_PREFIX + "I_PK_TREE", true);
            }
            if (success) break;
        }
    }

    public void testMemoryStatistics() throws Exception {
        System.out.println("\n\nTESTING MEMORY STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[13];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("RSS", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("JAVAUSED", VoltType.INTEGER);
        expectedSchema[5] = new ColumnInfo("JAVAUNUSED", VoltType.INTEGER);
        expectedSchema[6] = new ColumnInfo("TUPLEDATA", VoltType.INTEGER);
        expectedSchema[7] = new ColumnInfo("TUPLEALLOCATED", VoltType.INTEGER);
        expectedSchema[8] = new ColumnInfo("INDEXMEMORY", VoltType.INTEGER);
        expectedSchema[9] = new ColumnInfo("STRINGMEMORY", VoltType.INTEGER);
        expectedSchema[10] = new ColumnInfo("TUPLECOUNT", VoltType.BIGINT);
        expectedSchema[11] = new ColumnInfo("POOLEDMEMORY", VoltType.BIGINT);
        expectedSchema[12] = new ColumnInfo("PHYSICALMEMORY", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        //
        // memory
        //
        // give time to seed the stats cache?
        Thread.sleep(1000);
        results = client.callProcedure("@Statistics", "memory", 0).getResults();
        System.out.println("Node memory statistics table: " + results[0].toString());
        // one aggregate table returned
        assertEquals(1, results.length);
        validateSchema(results[0], expectedTable);
        results[0].advanceRow();
        // Hacky, on a single local cluster make sure that all 'nodes' are present.
        // MEMORY stats lacks a common string across nodes, but we can hijack the hostname in this case.
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), true);
    }

    public void testCpuStatistics() throws Exception {
        System.out.println("\n\nTESTING CPU STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[4];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("PERCENT_USED", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        //
        // cpu
        //
        // give time to seed the stats cache?
        Thread.sleep(1000);
        results = client.callProcedure("@Statistics", "cpu", 0).getResults();
        System.out.println("Node cpu statistics table: " + results[0].toString());
        // one aggregate table returned
        assertEquals(1, results.length);
        validateSchema(results[0], expectedTable);
        results[0].advanceRow();
        // Hacky, on a single local cluster make sure that all 'nodes' are present.
        // CPU stats lacks a common string across nodes, but we can hijack the hostname in this case.
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), true);
    }

    public void testProcedureStatistics() throws Exception {
        System.out.println("\n\nTESTING PROCEDURE STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[19];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("PARTITION_ID", VoltType.INTEGER);
        expectedSchema[5] = new ColumnInfo("PROCEDURE", VoltType.STRING);
        expectedSchema[6] = new ColumnInfo("INVOCATIONS", VoltType.BIGINT);
        expectedSchema[7] = new ColumnInfo("TIMED_INVOCATIONS", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("MIN_EXECUTION_TIME", VoltType.BIGINT);
        expectedSchema[9] = new ColumnInfo("MAX_EXECUTION_TIME", VoltType.BIGINT);
        expectedSchema[10] = new ColumnInfo("AVG_EXECUTION_TIME", VoltType.BIGINT);
        expectedSchema[11] = new ColumnInfo("MIN_RESULT_SIZE", VoltType.INTEGER);
        expectedSchema[12] = new ColumnInfo("MAX_RESULT_SIZE", VoltType.INTEGER);
        expectedSchema[13] = new ColumnInfo("AVG_RESULT_SIZE", VoltType.INTEGER);
        expectedSchema[14] = new ColumnInfo("MIN_PARAMETER_SET_SIZE", VoltType.INTEGER);
        expectedSchema[15] = new ColumnInfo("MAX_PARAMETER_SET_SIZE", VoltType.INTEGER);
        expectedSchema[16] = new ColumnInfo("AVG_PARAMETER_SET_SIZE", VoltType.INTEGER);
        expectedSchema[17] = new ColumnInfo("ABORTS", VoltType.BIGINT);
        expectedSchema[18] = new ColumnInfo("FAILURES", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        //
        // procedure
        //
        // Induce procedure invocations on all partitions.  May fail in non-legacy hashing case
        // this plus R/W replication should ensure that every site on every node runs this transaction
        // at least once

        results = client.callProcedure("@GetPartitionKeys", "INTEGER").getResults();
        VoltTable keys = results[0];
        for (int k = 0;k < keys.getRowCount(); k++) {
            long key = keys.fetchRow(k).getLong(1);
            client.callProcedure("NEW_ORDER.insert", key);
        }

        for (int i = 0; i < HOSTS * SITES; i++) {
            client.callProcedure("NEW_ORDER.insert", i);
        }
        // 3 seconds translates to 3 billion nanos, which overflows internal
        // values (ENG-1039)
        //It's possible that the nanosecond count goes backwards... so run this a couple
        //of times to make sure the min value gets set
        for (int ii = 0; ii < 3; ii++) {
            results = client.callProcedure("GoSleep", 3000, 0, null).getResults();
        }
        results = client.callProcedure("@Statistics", "procedure", 0).getResults();
        System.out.println("Test procedures table: " + results[0].toString());
        // one aggregate table returned
        assertEquals(1, results.length);
        validateSchema(results[0], expectedTable);
        // For this table, where unique HSID isn't written to SITE_ID, these
        // two checks should ensure we get all the rows we expect?
        validateRowSeenAtAllHosts(results[0], "PROCEDURE", "NEW_ORDER.insert", false);
        validateRowSeenAtAllPartitions(results[0], "PROCEDURE", "NEW_ORDER.insert", false);
        results[0].resetRowPosition();

        VoltTable stats = results[0];
        String procname = "blerg";
        while (!procname.equals("org.voltdb_testprocs.regressionsuites.malicious.GoSleep")) {
            stats.advanceRow();
            procname = (String)stats.get("PROCEDURE", VoltType.STRING);
        }

        // Retrieve all statistics
        long min_time = (Long)stats.get("MIN_EXECUTION_TIME", VoltType.BIGINT);
        long max_time = (Long)stats.get("MAX_EXECUTION_TIME", VoltType.BIGINT);
        long avg_time = (Long)stats.get("AVG_EXECUTION_TIME", VoltType.BIGINT);
        long min_result_size = (Long)stats.get("MIN_RESULT_SIZE", VoltType.BIGINT);
        long max_result_size = (Long)stats.get("MAX_RESULT_SIZE", VoltType.BIGINT);
        long avg_result_size = (Long)stats.get("AVG_RESULT_SIZE", VoltType.BIGINT);
        long min_parameter_set_size = (Long)stats.get("MIN_PARAMETER_SET_SIZE", VoltType.BIGINT);
        long max_parameter_set_size = (Long)stats.get("MAX_PARAMETER_SET_SIZE", VoltType.BIGINT);
        long avg_parameter_set_size = (Long)stats.get("AVG_PARAMETER_SET_SIZE", VoltType.BIGINT);

        // Check for overflow
        assertTrue("Failed MIN_EXECUTION_TIME > 0, value was: " + min_time,
                   min_time > 0);
        assertTrue("Failed MAX_EXECUTION_TIME > 0, value was: " + max_time,
                   max_time > 0);
        assertTrue("Failed AVG_EXECUTION_TIME > 0, value was: " + avg_time,
                   avg_time > 0);
        assertTrue("Failed MIN_RESULT_SIZE > 0, value was: " + min_result_size,
                   min_result_size >= 0);
        assertTrue("Failed MAX_RESULT_SIZE > 0, value was: " + max_result_size,
                   max_result_size >= 0);
        assertTrue("Failed AVG_RESULT_SIZE > 0, value was: " + avg_result_size,
                   avg_result_size >= 0);
        assertTrue("Failed MIN_PARAMETER_SET_SIZE > 0, value was: " + min_parameter_set_size,
                   min_parameter_set_size >= 0);
        assertTrue("Failed MAX_PARAMETER_SET_SIZE > 0, value was: " + max_parameter_set_size,
                   max_parameter_set_size >= 0);
        assertTrue("Failed AVG_PARAMETER_SET_SIZE > 0, value was: " + avg_parameter_set_size,
                   avg_parameter_set_size >= 0);

        // check for reasonable values
        assertTrue("Failed MIN_EXECUTION_TIME > 2,400,000,000ns, value was: " +
                   min_time,
                   min_time > 2400000000L);
        assertTrue("Failed MAX_EXECUTION_TIME > 2,400,000,000ns, value was: " +
                   max_time,
                   max_time > 2400000000L);
        assertTrue("Failed AVG_EXECUTION_TIME > 2,400,000,000ns, value was: " +
                   avg_time,
                   avg_time > 2400000000L);
        assertTrue("Failed MIN_RESULT_SIZE < 1,000,000, value was: " +
                   min_result_size,
                   min_result_size < 1000000L);
        assertTrue("Failed MAX_RESULT_SIZE < 1,000,000, value was: " +
                   max_result_size,
                   max_result_size < 1000000L);
        assertTrue("Failed AVG_RESULT_SIZE < 1,000,000, value was: " +
                   avg_result_size,
                   avg_result_size < 1000000L);
        assertTrue("Failed MIN_PARAMETER_SET_SIZE < 1,000,000, value was: " +
                   min_parameter_set_size,
                   min_parameter_set_size < 1000000L);
        assertTrue("Failed MAX_PARAMETER_SET_SIZE < 1,000,000, value was: " +
                   max_parameter_set_size,
                   max_parameter_set_size < 1000000L);
        assertTrue("Failed AVG_PARAMETER_SET_SIZE < 1,000,000, value was: " +
                   avg_parameter_set_size,
                   avg_parameter_set_size < 1000000L);

        // Validate the PROCEDUREPROFILE aggregation.
        results = client.callProcedure("@Statistics", "procedureprofile", 0).getResults();
        System.out.println("\n\n\n" + results[0].toString() + "\n\n\n");

        // expect NEW_ORDER.insert, GoSleep
        // see TestStatsProcProfile.java for tests of the aggregation itself.
        assertEquals("Validate site filtering for PROCEDUREPROFILE",
                2, results[0].getRowCount());
    }

    public void testIOStatistics() throws Exception {
        System.out.println("\n\nTESTING IO STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        // Based on doc, not code
        // HOST_ID, SITE_ID, and PARTITION_ID all differ.  Fixed to match
        // reality so tests would pass, but, ugh.
        ColumnInfo[] expectedSchema = new ColumnInfo[9];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("CONNECTION_ID", VoltType.BIGINT);
        expectedSchema[4] = new ColumnInfo("CONNECTION_HOSTNAME", VoltType.STRING);
        expectedSchema[5] = new ColumnInfo("BYTES_READ", VoltType.BIGINT);
        expectedSchema[6] = new ColumnInfo("MESSAGES_READ", VoltType.BIGINT);
        expectedSchema[7] = new ColumnInfo("BYTES_WRITTEN", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("MESSAGES_WRITTEN", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;
        //
        // iostats
        //
        results = client.callProcedure("@Statistics", "iostats", 0).getResults();
        System.out.println("Test iostats table: " + results[0].toString());
        // one aggregate table returned
        assertEquals(1, results.length);
        validateSchema(results[0], expectedTable);
    }

    public void testTopoStatistics() throws Exception {
        System.out.println("\n\nTESTING TOPO STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema1 = new ColumnInfo[3];
        expectedSchema1[0] = new ColumnInfo("Partition", VoltType.INTEGER);
        expectedSchema1[1] = new ColumnInfo("Sites", VoltType.STRING);
        expectedSchema1[2] = new ColumnInfo("Leader", VoltType.STRING);
        VoltTable expectedTable1 = new VoltTable(expectedSchema1);

        ColumnInfo[] expectedSchema2 = new ColumnInfo[2];
        expectedSchema2[0] = new ColumnInfo("HASHTYPE", VoltType.STRING);
        expectedSchema2[1] = new ColumnInfo("HASHCONFIG", VoltType.VARBINARY);
        VoltTable expectedTable2 = new VoltTable(expectedSchema2);

        VoltTable[] results = null;

        //
        // TOPO
        //
        results = client.callProcedure("@Statistics", "TOPO", 0).getResults();
        // two aggregate tables returned
        assertEquals(2, results.length);
        System.out.println("Test TOPO table: " + results[0].toString());
        System.out.println("Test TOPO table: " + results[1].toString());
        validateSchema(results[0], expectedTable1);
        validateSchema(results[1], expectedTable2);
        VoltTable topo = results[0];
        // Should have partitions + 1 rows in the first table
        assertEquals(PARTITIONS + 1, results[0].getRowCount());
        // Make sure we can find the MPI, at least
        boolean found = false;
        while (topo.advanceRow()) {
            if ((int)topo.getLong("Partition") == MpInitiator.MP_INIT_PID) {
                found = true;
            }
        }
        assertTrue(found);
        // and only one row in the second table
        assertEquals(1, results[1].getRowCount());
    }

    //
    // planner statistics
    //
    public void testPlannerStatistics() throws Exception {
        System.out.println("\n\nTESTING PLANNER STATS\n\n\n");
        Client client  = getClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[14];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("PARTITION_ID", VoltType.INTEGER);
        expectedSchema[5] = new ColumnInfo("CACHE1_LEVEL", VoltType.INTEGER);
        expectedSchema[6] = new ColumnInfo("CACHE2_LEVEL", VoltType.INTEGER);
        expectedSchema[7] = new ColumnInfo("CACHE1_HITS", VoltType.INTEGER);
        expectedSchema[8] = new ColumnInfo("CACHE2_HITS", VoltType.INTEGER);
        expectedSchema[9] = new ColumnInfo("CACHE_MISSES", VoltType.INTEGER);
        expectedSchema[10] = new ColumnInfo("PLAN_TIME_MIN", VoltType.BIGINT);
        expectedSchema[11] = new ColumnInfo("PLAN_TIME_MAX", VoltType.BIGINT);
        expectedSchema[12] = new ColumnInfo("PLAN_TIME_AVG", VoltType.BIGINT);
        expectedSchema[13] = new ColumnInfo("FAILURES", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;

        // Clear the interval statistics
        results = client.callProcedure("@Statistics", "planner", 1).getResults();
        assertEquals(1, results.length);

        // Invoke a few select queries a few times to get some cache hits and misses,
        // and to exceed the sampling frequency.
        // This does not use level 2 cache (parameterization) or trigger failures.
        for (String query : new String[] {
                "select * from warehouse",
                "select * from new_order",
                "select * from item",
                }) {
            for (int i = 0; i < 10; i++) {
                client.callProcedure("@AdHoc", query).getResults();
                assertEquals(1, results.length);
            }
        }

        // Get the final interval statistics
        results = client.callProcedure("@Statistics", "planner", 1).getResults();
        assertEquals(1, results.length);
        System.out.println("Test planner table: " + results[0].toString());
        validateSchema(results[0], expectedTable);
        VoltTable stats = results[0];

        // Sample the statistics
        List<Long> siteIds = new ArrayList<Long>();
        int cache1_level = 0;
        int cache2_level = 0;
        int cache1_hits  = 0;
        int cache2_hits  = 0;
        int cache_misses = 0;
        long plan_time_min_min = Long.MAX_VALUE;
        long plan_time_max_max = Long.MIN_VALUE;
        long plan_time_avg_tot = 0;
        int failures = 0;
        while (stats.advanceRow()) {
            cache1_level += (Integer)stats.get("CACHE1_LEVEL", VoltType.INTEGER);
            cache2_level += (Integer)stats.get("CACHE2_LEVEL", VoltType.INTEGER);
            cache1_hits  += (Integer)stats.get("CACHE1_HITS", VoltType.INTEGER);
            cache2_hits  += (Integer)stats.get("CACHE2_HITS", VoltType.INTEGER);
            cache_misses += (Integer)stats.get("CACHE_MISSES", VoltType.INTEGER);
            plan_time_min_min = Math.min(plan_time_min_min, (Long)stats.get("PLAN_TIME_MIN", VoltType.BIGINT));
            plan_time_max_max = Math.max(plan_time_max_max, (Long)stats.get("PLAN_TIME_MAX", VoltType.BIGINT));
            plan_time_avg_tot += (Long)stats.get("PLAN_TIME_AVG", VoltType.BIGINT);
            failures += (Integer)stats.get("FAILURES", VoltType.INTEGER);
            siteIds.add((Long)stats.get("SITE_ID", VoltType.BIGINT));
        }

        // Check for reasonable results
        int globalPlanners = 0;
        assertTrue("Failed siteIds count >= 2", siteIds.size() >= 2);
        for (long siteId : siteIds) {
            if (siteId == -1) {
                globalPlanners++;
            }
        }
        assertTrue("Global planner sites not 1, value was: " + globalPlanners, globalPlanners == 1);
        assertTrue("Failed total CACHE1_LEVEL > 0, value was: " + cache1_level, cache1_level > 0);
        assertTrue("Failed total CACHE1_LEVEL < 1,000,000, value was: " + cache1_level, cache1_level < 1000000);
        assertTrue("Failed total CACHE2_LEVEL >= 0, value was: " + cache2_level, cache2_level >= 0);
        assertTrue("Failed total CACHE2_LEVEL < 1,000,000, value was: " + cache2_level, cache2_level < 1000000);
        assertTrue("Failed total CACHE1_HITS > 0, value was: " + cache1_hits, cache1_hits > 0);
        assertTrue("Failed total CACHE1_HITS < 1,000,000, value was: " + cache1_hits, cache1_hits < 1000000);
        assertTrue("Failed total CACHE2_HITS == 0, value was: " + cache2_hits, cache2_hits == 0);
        assertTrue("Failed total CACHE2_HITS < 1,000,000, value was: " + cache2_hits, cache2_hits < 1000000);
        assertTrue("Failed total CACHE_MISSES > 0, value was: " + cache_misses, cache_misses > 0);
        assertTrue("Failed total CACHE_MISSES < 1,000,000, value was: " + cache_misses, cache_misses < 1000000);
        assertTrue("Failed min PLAN_TIME_MIN > 0, value was: " + plan_time_min_min, plan_time_min_min > 0);
        assertTrue("Failed total PLAN_TIME_MIN < 100,000,000,000, value was: " + plan_time_min_min, plan_time_min_min < 100000000000L);
        assertTrue("Failed max PLAN_TIME_MAX > 0, value was: " + plan_time_max_max, plan_time_max_max > 0);
        assertTrue("Failed total PLAN_TIME_MAX < 100,000,000,000, value was: " + plan_time_max_max, plan_time_max_max < 100000000000L);
        assertTrue("Failed total PLAN_TIME_AVG > 0, value was: " + plan_time_avg_tot, plan_time_avg_tot > 0);
        assertTrue("Failed total FAILURES == 0, value was: " + failures, failures == 0);
    }

    public void testDRNodeStatistics() throws Exception {
        if (!VoltDB.instance().getConfig().m_isEnterprise) {
            System.out.println("SKIPPING DRNODE STATS TESTS FOR COMMUNITY VERSION");
            return;
        }
        System.out.println("\n\nTESTING DRNODE STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema2 = new ColumnInfo[7];
        expectedSchema2[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema2[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema2[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema2[3] = new ColumnInfo("ENABLED", VoltType.STRING);
        expectedSchema2[4] = new ColumnInfo("SYNCSNAPSHOTSTATE", VoltType.STRING);
        expectedSchema2[5] = new ColumnInfo("ROWSINSYNCSNAPSHOT", VoltType.BIGINT);
        expectedSchema2[6] = new ColumnInfo("ROWSACKEDFORSYNCSNAPSHOT", VoltType.BIGINT);
        VoltTable expectedTable2 = new VoltTable(expectedSchema2);

        VoltTable[] results = null;
        //
        // DRNODE
        //
        results = client.callProcedure("@Statistics", "DRNODE", 0).getResults();
        // one aggregate tables returned
        assertEquals(1, results.length);
        System.out.println("Test DRNODE table: " + results[0].toString());
        validateSchema(results[0], expectedTable2);
        // One row per host for DRNODE stats
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), true);
    }

    public void testDRPartitionStatistics() throws Exception {
        if (!VoltDB.instance().getConfig().m_isEnterprise) {
            System.out.println("SKIPPING DRPARTITION STATS TESTS FOR COMMUNITY VERSION");
            return;
        }
        System.out.println("\n\nTESTING DRPARTITION STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema1 = new ColumnInfo[11];
        expectedSchema1[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema1[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema1[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema1[3] = new ColumnInfo("PARTITION_ID", VoltType.INTEGER);
        expectedSchema1[4] = new ColumnInfo("STREAMTYPE", VoltType.STRING);
        expectedSchema1[5] = new ColumnInfo("TOTALBYTES", VoltType.BIGINT);
        expectedSchema1[6] = new ColumnInfo("TOTALBYTESINMEMORY", VoltType.BIGINT);
        expectedSchema1[7] = new ColumnInfo("TOTALBUFFERS", VoltType.BIGINT);
        expectedSchema1[8] = new ColumnInfo("LASTACKTIMESTAMP", VoltType.BIGINT);
        expectedSchema1[9] = new ColumnInfo("ISSYNCED", VoltType.STRING);
        expectedSchema1[10] = new ColumnInfo("MODE", VoltType.STRING);
        VoltTable expectedTable1 = new VoltTable(expectedSchema1);

        VoltTable[] results = null;
        //
        // DRPARTITION
        //
        results = client.callProcedure("@Statistics", "DRPARTITION", 0).getResults();
        // one aggregate tables returned
        assertEquals(1, results.length);
        System.out.println("Test DR table: " + results[0].toString());
        validateSchema(results[0], expectedTable1);
        // One row per site, don't have HSID for ease of check, just check a bunch of stuff
        assertEquals(HOSTS * SITES, results[0].getRowCount());
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
        results[0].advanceRow();
        validateRowSeenAtAllPartitions(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
    }

    public void testDRStatistics() throws Exception {
        if (!VoltDB.instance().getConfig().m_isEnterprise) {
            System.out.println("SKIPPING DR STATS TESTS FOR COMMUNITY VERSION");
            return;
        }
        System.out.println("\n\nTESTING DR STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema1 = new ColumnInfo[11];
        expectedSchema1[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema1[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema1[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema1[3] = new ColumnInfo("PARTITION_ID", VoltType.INTEGER);
        expectedSchema1[4] = new ColumnInfo("STREAMTYPE", VoltType.STRING);
        expectedSchema1[5] = new ColumnInfo("TOTALBYTES", VoltType.BIGINT);
        expectedSchema1[6] = new ColumnInfo("TOTALBYTESINMEMORY", VoltType.BIGINT);
        expectedSchema1[7] = new ColumnInfo("TOTALBUFFERS", VoltType.BIGINT);
        expectedSchema1[8] = new ColumnInfo("LASTACKTIMESTAMP", VoltType.BIGINT);
        expectedSchema1[9] = new ColumnInfo("ISSYNCED", VoltType.STRING);
        expectedSchema1[10] = new ColumnInfo("MODE", VoltType.STRING);
        VoltTable expectedTable1 = new VoltTable(expectedSchema1);

        ColumnInfo[] expectedSchema2 = new ColumnInfo[7];
        expectedSchema2[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema2[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema2[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema2[3] = new ColumnInfo("ENABLED", VoltType.STRING);
        expectedSchema2[4] = new ColumnInfo("SYNCSNAPSHOTSTATE", VoltType.STRING);
        expectedSchema2[5] = new ColumnInfo("ROWSINSYNCSNAPSHOT", VoltType.BIGINT);
        expectedSchema2[6] = new ColumnInfo("ROWSACKEDFORSYNCSNAPSHOT", VoltType.BIGINT);
        VoltTable expectedTable2 = new VoltTable(expectedSchema2);

        VoltTable[] results = null;
        //
        // DR
        //
        results = client.callProcedure("@Statistics", "DR", 0).getResults();
        // two aggregate tables returned
        assertEquals(2, results.length);
        System.out.println("Test DR table: " + results[0].toString());
        System.out.println("Test DR table: " + results[1].toString());
        validateSchema(results[0], expectedTable1);
        validateSchema(results[1], expectedTable2);
        // One row per site, don't have HSID for ease of check, just check a bunch of stuff
        assertEquals(HOSTS * SITES, results[0].getRowCount());
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
        results[0].advanceRow();
        validateRowSeenAtAllPartitions(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
        // One row per host for DRNODE stats
        results[1].advanceRow();
        validateRowSeenAtAllHosts(results[1], "HOSTNAME", results[1].getString("HOSTNAME"), true);
    }

    public void testLiveClientsStatistics() throws Exception {
        System.out.println("\n\nTESTING LIVECLIENTS STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[9];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("CONNECTION_ID", VoltType.BIGINT);
        expectedSchema[4] = new ColumnInfo("CLIENT_HOSTNAME", VoltType.STRING);
        expectedSchema[5] = new ColumnInfo("ADMIN", VoltType.TINYINT);
        expectedSchema[6] = new ColumnInfo("OUTSTANDING_REQUEST_BYTES", VoltType.BIGINT);
        expectedSchema[7] = new ColumnInfo("OUTSTANDING_RESPONSE_MESSAGES", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("OUTSTANDING_TRANSACTIONS", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;
        //
        // LIVECLIENTS
        //
        results = client.callProcedure("@Statistics", "LIVECLIENTS", 0).getResults();
        // one aggregate table returned
        assertEquals(1, results.length);
        System.out.println("Test LIVECLIENTS table: " + results[0].toString());
        validateSchema(results[0], expectedTable);
        // Hacky, on a single local cluster make sure that all 'nodes' are present.
        // LiveClients stats lacks a common string across nodes, but we can hijack the hostname in this case.
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), true);
    }

    public void testStarvationStatistics() throws Exception {
        System.out.println("\n\nTESTING STARVATION STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[10];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("SITE_ID", VoltType.INTEGER);
        expectedSchema[4] = new ColumnInfo("COUNT", VoltType.BIGINT);
        expectedSchema[5] = new ColumnInfo("PERCENT", VoltType.FLOAT);
        expectedSchema[6] = new ColumnInfo("AVG", VoltType.BIGINT);
        expectedSchema[7] = new ColumnInfo("MIN", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("MAX", VoltType.BIGINT);
        expectedSchema[9] = new ColumnInfo("STDDEV", VoltType.BIGINT);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        VoltTable[] results = null;
        //
        // STARVATION
        //
        results = client.callProcedure("@Statistics", "STARVATION", 0).getResults();
        // one aggregate table returned
        assertEquals(1, results.length);
        System.out.println("Test STARVATION table: " + results[0].toString());
        validateSchema(results[0], expectedTable);
        // One row per site, we don't use HSID though, so hard to do straightforward
        // per-site unique check.  Finesse it.
        // We also get starvation stats for the MPI, so we need to add a site per host.
        assertEquals(HOSTS * (SITES + 1), results[0].getRowCount());
        results[0].advanceRow();
        validateRowSeenAtAllHosts(results[0], "HOSTNAME", results[0].getString("HOSTNAME"), false);
    }

    public void testSnapshotStatus() throws Exception {
        System.out.println("\n\nTESTING SNAPSHOTSTATUS\n\n\n");
        if (KFACTOR == 0) {
            // SnapshotSave is a PRO feature starting from 4.0
            return;
        }

        Client client  = getFullyConnectedClient();

        ColumnInfo[] expectedSchema = new ColumnInfo[14];
        expectedSchema[0] = new ColumnInfo("TIMESTAMP", VoltType.BIGINT);
        expectedSchema[1] = new ColumnInfo("HOST_ID", VoltType.INTEGER);
        expectedSchema[2] = new ColumnInfo("HOSTNAME", VoltType.STRING);
        expectedSchema[3] = new ColumnInfo("TABLE", VoltType.STRING);
        expectedSchema[4] = new ColumnInfo("PATH", VoltType.STRING);
        expectedSchema[5] = new ColumnInfo("FILENAME", VoltType.STRING);
        expectedSchema[6] = new ColumnInfo("NONCE", VoltType.STRING);
        expectedSchema[7] = new ColumnInfo("TXNID", VoltType.BIGINT);
        expectedSchema[8] = new ColumnInfo("START_TIME", VoltType.BIGINT);
        expectedSchema[9] = new ColumnInfo("END_TIME", VoltType.BIGINT);
        expectedSchema[10] = new ColumnInfo("SIZE", VoltType.BIGINT);
        expectedSchema[11] = new ColumnInfo("DURATION", VoltType.BIGINT);
        expectedSchema[12] = new ColumnInfo("THROUGHPUT", VoltType.FLOAT);
        expectedSchema[13] = new ColumnInfo("RESULT", VoltType.STRING);
        VoltTable expectedTable = new VoltTable(expectedSchema);

        // Finagle a snapshot
        client.callProcedure("@SnapshotSave", TMPDIR, TESTNONCE, 1);

        VoltTable[] results = null;
        //
        // SNAPSHOTSTATUS
        //
        results = client.callProcedure("@Statistics", "SNAPSHOTSTATUS", 0).getResults();
        // one aggregate table returned
        assertEquals(1, results.length);
        System.out.println("Test SNAPSHOTSTATUS table: " + results[0].toString());
        validateSchema(results[0], expectedTable);
        // One row per table per node
        validateRowSeenAtAllHosts(results[0], "TABLE", "WAREHOUSE", true);
        validateRowSeenAtAllHosts(results[0], "TABLE", "NEW_ORDER", true);
        validateRowSeenAtAllHosts(results[0], "TABLE", "ITEM", true);
    }

    public void testManagementStats() throws Exception {
        System.out.println("\n\nTESTING MANAGEMENT STATS\n\n\n");
        Client client  = getFullyConnectedClient();

        VoltTable[] results = null;
        //
        // LIVECLIENTS
        //
        results = client.callProcedure("@Statistics", "MANAGEMENT", 0).getResults();
        // eight aggregate tables returned.  Assume that we have selected the right
        // subset of stats internally, just check that we get stuff.
        assertEquals(8, results.length);
    }

    class RebalanceStatsChecker
    {
        final double fuzzFactor;
        final int rangesToMove;

        long tStartMS;
        long rangesMoved = 0;
        long bytesMoved = 0;
        long rowsMoved = 0;
        long invocations = 0;
        long totalInvTimeMS = 0;

        RebalanceStatsChecker(int rangesToMove, double fuzzFactor)
        {
            this.fuzzFactor = fuzzFactor;
            this.rangesToMove = rangesToMove;
            this.tStartMS = System.currentTimeMillis();
        }

        void update(int ranges, int bytes, int rows)
        {
            rangesMoved += ranges;
            bytesMoved += bytes;
            rowsMoved += rows;
            invocations++;
        }

        void checkFuzz(double expected, double actual)
        {
            double delta = Math.abs((expected - actual) / expected);
            if (delta > fuzzFactor) {
                assertFalse(Math.abs((expected - actual) / expected) > fuzzFactor);
            }
        }

        void check(BalancePartitionsStatistics.StatsPoint stats)
        {
            double totalTimeS = (System.currentTimeMillis() - tStartMS) / 1000.0;
            double statsRangesMoved1 = (stats.getPercentageMoved() / 100.0) * rangesToMove;
            checkFuzz(rangesMoved, statsRangesMoved1);
            double statsRangesMoved2 = stats.getRangesPerSecond() * totalTimeS;
            checkFuzz(rangesMoved, statsRangesMoved2);
            double statsBytesMoved = stats.getMegabytesPerSecond() * 1000000.0 * totalTimeS;
            checkFuzz(bytesMoved, statsBytesMoved);
            double statsRowsMoved = stats.getRowsPerSecond() * totalTimeS;
            checkFuzz(rowsMoved, statsRowsMoved);
            double statsInvocations = stats.getInvocationsPerSecond() * totalTimeS;
            checkFuzz(invocations, statsInvocations);
            double statsInvTimeMS = stats.getAverageInvocationTime() * invocations;
            assertTrue(Math.abs((totalInvTimeMS - statsInvTimeMS) / totalInvTimeMS) <= fuzzFactor);
            checkFuzz(totalInvTimeMS, statsInvTimeMS);
            double estTimeRemainingS = totalTimeS * (rangesToMove / (double)rangesMoved - 1.0);
            double statsEstTimeRemainingS = stats.getEstimatedRemaining() / 1000.0;
            checkFuzz(estTimeRemainingS, statsEstTimeRemainingS);
        }
    }

    public void testRebalanceStats() throws Exception {
        System.out.println("testRebalanceStats");
        // Test constants
        final int DURATION_SECONDS = 10;
        final int INVOCATION_SLEEP_MILLIS = 500;
        final int IDLE_SLEEP_MILLIS = 200;
        final int RANGES_TO_MOVE = Integer.MAX_VALUE;
        final int BYTES_TO_MOVE = 10000000;
        final int ROWS_TO_MOVE = 1000000;
        final double FUZZ_FACTOR = .1;

        RebalanceStatsChecker checker = new RebalanceStatsChecker(RANGES_TO_MOVE, FUZZ_FACTOR);
        BalancePartitionsStatistics bps = new BalancePartitionsStatistics(RANGES_TO_MOVE);
        Random r = new Random(2222);
        // Random numbers are between zero and the constant, so everything will average out
        // to half the time and quantities. Nothing will be exhausted by the test.
        final int loopCount = (DURATION_SECONDS * 1000) / (INVOCATION_SLEEP_MILLIS + IDLE_SLEEP_MILLIS);
        for (int i = 0; i < loopCount; i++) {
            bps.logBalanceStarts();
            int invocationTimeMS = r.nextInt(INVOCATION_SLEEP_MILLIS);
            Thread.sleep(invocationTimeMS);
            checker.totalInvTimeMS += invocationTimeMS;
            int ranges = r.nextInt(RANGES_TO_MOVE / loopCount);
            int bytes = r.nextInt(BYTES_TO_MOVE / loopCount);
            int rows = r.nextInt(ROWS_TO_MOVE / loopCount);
            bps.logBalanceEnds(ranges, bytes, TimeUnit.MILLISECONDS.toNanos(invocationTimeMS), TimeUnit.MILLISECONDS.toNanos(invocationTimeMS), rows);
            checker.update(ranges, bytes, rows);
            checker.check(bps.getLastStatsPoint());
            int idleTimeMS = r.nextInt(IDLE_SLEEP_MILLIS);
            Thread.sleep(idleTimeMS);
        }
        // Check the results with fuzzing to avoid rounding errors.
        checker.check(bps.getOverallStats());
    }

    //
    // Build a list of the tests to be run. Use the regression suite
    // helpers to allow multiple backends.
    // JUnit magic that uses the regression suite helper classes.
    //
    static public Test suite() throws IOException {
        VoltServerConfig config = null;

        MultiConfigSuiteBuilder builder =
            new MultiConfigSuiteBuilder(TestStatisticsSuite.class);

        // Not really using TPCC functionality but need a database.
        // The testLoadMultipartitionTable procedure assumes partitioning
        // on warehouse id.
        VoltProjectBuilder project = new VoltProjectBuilder();
        project.addLiteralSchema(
                        "CREATE TABLE WAREHOUSE (\n" +
                        "  W_ID SMALLINT DEFAULT '0' NOT NULL,\n" +
                        "  W_NAME VARCHAR(16) DEFAULT NULL,\n" +
                        "  W_STREET_1 VARCHAR(32) DEFAULT NULL,\n" +
                        "  W_STREET_2 VARCHAR(32) DEFAULT NULL,\n" +
                        "  W_CITY VARCHAR(32) DEFAULT NULL,\n" +
                        "  W_STATE VARCHAR(2) DEFAULT NULL,\n" +
                        "  W_ZIP VARCHAR(9) DEFAULT NULL,\n" +
                        "  W_TAX FLOAT DEFAULT NULL,\n" +
                        "  W_YTD FLOAT DEFAULT NULL,\n" +
                        "  CONSTRAINT W_PK_TREE PRIMARY KEY (W_ID)\n" +
                        ");\n" +
                        "CREATE TABLE ITEM (\n" +
                        "  I_ID INTEGER DEFAULT '0' NOT NULL,\n" +
                        "  I_IM_ID INTEGER DEFAULT NULL,\n" +
                        "  I_NAME VARCHAR(32) DEFAULT NULL,\n" +
                        "  I_PRICE FLOAT DEFAULT NULL,\n" +
                        "  I_DATA VARCHAR(64) DEFAULT NULL,\n" +
                        "  CONSTRAINT I_PK_TREE PRIMARY KEY (I_ID)\n" +
                        ");\n" +
                        "CREATE TABLE NEW_ORDER (\n" +
                        "  NO_W_ID SMALLINT DEFAULT '0' NOT NULL\n" +
                        ");\n");

        project.addPartitionInfo("WAREHOUSE", "W_ID");
        project.addPartitionInfo("NEW_ORDER", "NO_W_ID");
        project.addProcedures(PROCEDURES);

        /*
         * Add a cluster configuration for sysprocs too
         */
        config = new LocalCluster("statistics-cluster.jar", TestStatisticsSuite.SITES,
                TestStatisticsSuite.HOSTS, TestStatisticsSuite.KFACTOR,
                BackendTarget.NATIVE_EE_JNI);
        ((LocalCluster) config).setHasLocalServer(hasLocalServer);
        boolean success = config.compile(project);
        assertTrue(success);
        builder.addServerConfig(config);

        return builder;
    }
}
TOP

Related Classes of org.voltdb.regressionsuites.TestStatisticsSuite$RebalanceStatsChecker

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.