Examples of org.apache.cassandra.metrics.RestorableMeter

org.apache.cassandra.metrics.RestorableMeter
pedia.org/wiki/Moving_average#Exponential_moving_average">EMA

    }


    private static List<SSTableReader> resetSummaries(List<SSTableReader> sstables, long originalOffHeapSize) throws IOException
    {
        for (SSTableReader sstable : sstables)
            sstable.readMeter = new RestorableMeter(100.0, 100.0);


        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, originalOffHeapSize * sstables.size());
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());

View Full Code Here

        int numRows = 256;
        createSSTables(ksname, cfname, numSSTables, numRows);


        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
        for (SSTableReader sstable : sstables)
            sstable.readMeter = new RestorableMeter(100.0, 100.0);


        for (SSTableReader sstable : sstables)
            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001);


        // double the min_index_interval

View Full Code Here

        int numRows = 256;
        createSSTables(ksname, cfname, numSSTables, numRows);


        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
        for (SSTableReader sstable : sstables)
            sstable.readMeter = new RestorableMeter(100.0, 100.0);


        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, sstables, 1);
        sstables = new ArrayList<>(cfs.getSSTables());
        for (SSTableReader sstable : sstables)
            assertEquals(cfs.metadata.getMaxIndexInterval(), sstable.getEffectiveIndexInterval(), 0.01);

View Full Code Here


        int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata.getMinIndexInterval()) / cfs.metadata.getMaxIndexInterval();


        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
        for (SSTableReader sstable : sstables)
            sstable.readMeter = new RestorableMeter(100.0, 100.0);


        long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize();


        // there should be enough space to not downsample anything
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * numSSTables));
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
        assertEquals(singleSummaryOffHeapSpace * numSSTables, totalOffHeapSize(sstables));
        validateData(cfs, numRows);


        // everything should get cut in half
        assert sstables.size() == 4;
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * (numSSTables / 2)));
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // everything should get cut to a quarter
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * (numSSTables / 4)));
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // upsample back up to half
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables,(singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
        assert sstables.size() == 4;
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // upsample back up to the original index summary
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * numSSTables));
        for (SSTableReader sstable : sstables)
            assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // make two of the four sstables cold, only leave enough space for three full index summaries,
        // so the two cold sstables should get downsampled to be half of their original size
        sstables.get(0).readMeter = new RestorableMeter(50.0, 50.0);
        sstables.get(1).readMeter = new RestorableMeter(50.0, 50.0);
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * 3));
        Collections.sort(sstables, hotnessComparator);
        assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // small increases or decreases in the read rate don't result in downsampling or upsampling
        double lowerRate = 50.0 * (DOWNSAMPLE_THESHOLD + (DOWNSAMPLE_THESHOLD * 0.10));
        double higherRate = 50.0 * (UPSAMPLE_THRESHOLD - (UPSAMPLE_THRESHOLD * 0.10));
        sstables.get(0).readMeter = new RestorableMeter(lowerRate, lowerRate);
        sstables.get(1).readMeter = new RestorableMeter(higherRate, higherRate);
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * 3));
        Collections.sort(sstables, hotnessComparator);
        assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
        validateData(cfs, numRows);


        // reset, and then this time, leave enough space for one of the cold sstables to not get downsampled
        sstables = resetSummaries(sstables, singleSummaryOffHeapSpace);
        sstables.get(0).readMeter = new RestorableMeter(1.0, 1.0);
        sstables.get(1).readMeter = new RestorableMeter(2.0, 2.0);
        sstables.get(2).readMeter = new RestorableMeter(1000.0, 1000.0);
        sstables.get(3).readMeter = new RestorableMeter(1000.0, 1000.0);


        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * 3) + 50);
        Collections.sort(sstables, hotnessComparator);


        if (sstables.get(0).getIndexSummarySamplingLevel() == minSamplingLevel)
            assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummarySamplingLevel());
        else
            assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummarySamplingLevel());


        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
        assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
        validateData(cfs, numRows);




        // Cause a mix of upsampling and downsampling. We'll leave enough space for two full index summaries. The two
        // coldest sstables will get downsampled to 4/128 of their size, leaving us with 1 and 92/128th index
        // summaries worth of space.  The hottest sstable should get a full index summary, and the one in the middle
        // should get the remainder.
        sstables.get(0).readMeter = new RestorableMeter(0.0, 0.0);
        sstables.get(1).readMeter = new RestorableMeter(0.0, 0.0);
        sstables.get(2).readMeter = new RestorableMeter(92, 92);
        sstables.get(3).readMeter = new RestorableMeter(128.0, 128.0);
        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
        Collections.sort(sstables, hotnessComparator);
        assertEquals(1, sstables.get(0).getIndexSummarySize());  // at the min sampling level
        assertEquals(1, sstables.get(0).getIndexSummarySize());  // at the min sampling level
        assertTrue(sstables.get(2).getIndexSummarySamplingLevel() > minSamplingLevel);

View Full Code Here

    {
        String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and columnfamily_name=? and generation=?";
        UntypedResultSet results = executeInternal(String.format(cql, SSTABLE_ACTIVITY_TABLE), keyspace, table, generation);


        if (results.isEmpty())
            return new RestorableMeter();


        UntypedResultSet.Row row = results.one();
        double m15rate = row.getDouble("rate_15m");
        double m120rate = row.getDouble("rate_120m");
        return new RestorableMeter(m15rate, m120rate);
    }

View Full Code Here

        Pair<List<SSTableReader>, Double> bucket;


        List<SSTableReader> interestingBucket = mostInterestingBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32);
        assertTrue("nothing should be returned when all buckets are below the min threshold", interestingBucket.isEmpty());


        sstrs.get(0).readMeter = new RestorableMeter(100.0, 100.0);
        sstrs.get(1).readMeter = new RestorableMeter(200.0, 200.0);
        sstrs.get(2).readMeter = new RestorableMeter(300.0, 300.0);


        long estimatedKeys = sstrs.get(0).estimatedKeys();


        // if we have more than the max threshold, the coldest should be dropped
        bucket = trimToThresholdWithHotness(sstrs, 2);

View Full Code Here

            sstr.readMeter = null;
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("when there are no read meters, no sstables should be filtered", sstrs.size(), filtered.size());


        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(0.0, 0.0);
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("when all read meters are zero, no sstables should be filtered", sstrs.size(), filtered.size());


        // leave all read rates at 0 besides one
        sstrs.get(0).readMeter = new RestorableMeter(1000.0, 1000.0);
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("there should only be one hot sstable", 1, filtered.size());
        assertEquals(1000.0, filtered.get(0).readMeter.twoHourRate(), 0.5);


        // the total read rate is 100, and we'll set a threshold of 2.5%, so two of the sstables with read
        // rate 1.0 should be ignored, but not the third
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(0.0, 0.0);
        sstrs.get(0).readMeter = new RestorableMeter(97.0, 97.0);
        sstrs.get(1).readMeter = new RestorableMeter(1.0, 1.0);
        sstrs.get(2).readMeter = new RestorableMeter(1.0, 1.0);
        sstrs.get(3).readMeter = new RestorableMeter(1.0, 1.0);


        filtered = filterColdSSTables(sstrs, 0.025);
        assertEquals(2, filtered.size());
        assertEquals(98.0, filtered.get(0).readMeter.twoHourRate() + filtered.get(1).readMeter.twoHourRate(), 0.5);


        // make sure a threshold of 0.0 doesn't result in any sstables being filtered
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(1.0, 1.0);
        filtered = filterColdSSTables(sstrs, 0.0);
        assertEquals(sstrs.size(), filtered.size());


        // just for fun, set a threshold where all sstables are considered cold
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(1.0, 1.0);
        filtered = filterColdSSTables(sstrs, 1.0);
        assertTrue(filtered.isEmpty());
    }

View Full Code Here

                                                                 keyspace,
                                                                 table,
                                                                 generation));


        if (results.isEmpty())
            return new RestorableMeter();


        UntypedResultSet.Row row = results.one();
        double m15rate = row.getDouble("rate_15m");
        double m120rate = row.getDouble("rate_120m");
        return new RestorableMeter(m15rate, m120rate);
    }

View Full Code Here

        Pair<List<SSTableReader>, Double> bucket;


        List<SSTableReader> interestingBucket = mostInterestingBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32);
        assertTrue("nothing should be returned when all buckets are below the min threshold", interestingBucket.isEmpty());


        sstrs.get(0).readMeter = new RestorableMeter(100.0, 100.0);
        sstrs.get(1).readMeter = new RestorableMeter(200.0, 200.0);
        sstrs.get(2).readMeter = new RestorableMeter(300.0, 300.0);


        long estimatedKeys = sstrs.get(0).estimatedKeys();


        // if we have more than the max threshold, the coldest should be dropped
        bucket = trimToThresholdWithHotness(sstrs, 2);

View Full Code Here

            sstr.readMeter = null;
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("when there are no read meters, no sstables should be filtered", sstrs.size(), filtered.size());


        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(0.0, 0.0);
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("when all read meters are zero, no sstables should be filtered", sstrs.size(), filtered.size());


        // leave all read rates at 0 besides one
        sstrs.get(0).readMeter = new RestorableMeter(1000.0, 1000.0);
        filtered = filterColdSSTables(sstrs, 0.05);
        assertEquals("there should only be one hot sstable", 1, filtered.size());
        assertEquals(1000.0, filtered.get(0).readMeter.twoHourRate(), 0.5);


        // the total read rate is 100, and we'll set a threshold of 2.5%, so two of the sstables with read
        // rate 1.0 should be ignored, but not the third
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(0.0, 0.0);
        sstrs.get(0).readMeter = new RestorableMeter(97.0, 97.0);
        sstrs.get(1).readMeter = new RestorableMeter(1.0, 1.0);
        sstrs.get(2).readMeter = new RestorableMeter(1.0, 1.0);
        sstrs.get(3).readMeter = new RestorableMeter(1.0, 1.0);


        filtered = filterColdSSTables(sstrs, 0.025);
        assertEquals(2, filtered.size());
        assertEquals(98.0, filtered.get(0).readMeter.twoHourRate() + filtered.get(1).readMeter.twoHourRate(), 0.5);


        // make sure a threshold of 0.0 doesn't result in any sstables being filtered
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(1.0, 1.0);
        filtered = filterColdSSTables(sstrs, 0.0);
        assertEquals(sstrs.size(), filtered.size());


        // just for fun, set a threshold where all sstables are considered cold
        for (SSTableReader sstr : sstrs)
            sstr.readMeter = new RestorableMeter(1.0, 1.0);
        filtered = filterColdSSTables(sstrs, 1.0);
        assertTrue(filtered.isEmpty());
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.cassandra.metrics.RestorableMeter

org.apache.cassandra.db.compaction.SizeTieredCompactionStrategyTest

org.apache.cassandra.db.SystemKeyspace

org.apache.cassandra.io.sstable.IndexSummaryManagerTest

java.util.concurrent.atomic.AtomicLong

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.