Examples of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics

Package org.apache.commons.math3.stat.descriptive

Examples of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics

org.apache.commons.math3.stat.descriptive.DescriptiveStatistics
Maintains a dataset of values of a single variable and computes descriptive statistics based on stored data. The {@link #getWindowSize() windowSize}property sets a limit on the number of values that can be stored in the dataset. The default value, INFINITE_WINDOW, puts no limit on the size of the dataset. This value should be used with caution, as the backing store will grow without bound in this case. For very large datasets, {@link SummaryStatistics}, which does not store the dataset, should be used instead of this class. If windowSize is not INFINITE_WINDOW and more values are added than can be stored in the dataset, new values are added in a "rolling" manner, with new values replacing the "oldest" values in the dataset.
Note: this class is not threadsafe. Use {@link SynchronizedDescriptiveStatistics} if concurrent access from multiplethreads is required.

        }


        @Override
        public Block evaluateFinal()
        {
            DescriptiveStatistics statistics = new DescriptiveStatistics();
            for (int i = 0; i < accumulators.size(); i++) {
                BlockCursor cursor = accumulators.get(i).evaluateFinal().cursor();
                checkArgument(cursor.advanceNextPosition(), "accumulator returned no results");
                statistics.addValue(getNumeric(cursor));
            }


            BlockBuilder builder = new BlockBuilder(SINGLE_VARBINARY);
            builder.append(formatApproximateOutput(statistics, confidence));
            return builder.build();

View Full Code Here

        }


        @Override
        public void evaluateFinal(int groupId, BlockBuilder output)
        {
            DescriptiveStatistics statistics = new DescriptiveStatistics();
            for (int i = 0; i < accumulators.size(); i++) {
                BlockBuilder builder = new BlockBuilder(accumulators.get(i).getFinalTupleInfo());
                accumulators.get(i).evaluateFinal(groupId, builder);
                BlockCursor cursor = builder.build().cursor();
                checkArgument(cursor.advanceNextPosition(), "accumulator returned no results");
                statistics.addValue(getNumeric(cursor));
            }


            output.append(formatApproximateOutput(statistics, confidence));
        }

View Full Code Here


    @Test
    public void testMultiplePositions()
            throws Exception
    {
        DescriptiveStatistics stats = new DescriptiveStatistics();


        for (int i = 0; i < 500; ++i) {
            int uniques = ThreadLocalRandom.current().nextInt(20000) + 1;


            List<Object> values = createRandomSample(uniques, (int) (uniques * 1.5));


            long actual = estimateGroupByCount(values);
            double error = (actual - uniques) * 1.0 / uniques;


            stats.addValue(error);
        }


        assertLessThan(stats.getMean(), 1.0e-2);
        assertLessThan(Math.abs(stats.getStandardDeviation() - ApproximateCountDistinctAggregations.getStandardError()), 1.0e-2);
    }

View Full Code Here

                new int[]{10, 0, 0, 0},
                false);
        rp.reset(-3806751651286565680L);
        System.out.println("Random Seed = " + rp.getSeed());
        System.out.println("NM Seed = " + CC.getNameManager().getSeed());
        DescriptiveStatistics timeStats = new DescriptiveStatistics();
        DescriptiveStatistics trysStats = new DescriptiveStatistics();
        int count = 0;
        while (++count < 500) {
//            CC.resetTensorNames();
            Tensor t = rp.nextProduct(15);
            if (!(t instanceof Product))
                continue;


            Product from = (Product) t, target = from;


            long start = System.nanoTime();
            ProductsBijectionsPort port = new ProductsBijectionsPort(from.getContent(), target.getContent());


            int[] bijection;
            boolean good = false;
            int trys = 0;
            OUTER:
            while (trys++ < 5000 && (bijection = port.take()) != null) {
                for (int i = 0; i < bijection.length; ++i)
                    if (bijection[i] != i)
                        continue OUTER;
                good = true;
                break;
            }


            double millis = 1E-6 * (System.nanoTime() - start);
            timeStats.addValue(millis);
            trysStats.addValue(trys);


            if (!good)
                throw new RuntimeException();
        }
        System.out.println(timeStats);

View Full Code Here

                new int[]{10, 0, 0, 0},
                false);
        rp.reset(-3806751651286565680L);
        System.out.println("Random Seed = " + rp.getSeed());
        System.out.println("NM Seed = " + CC.getNameManager().getSeed());
        DescriptiveStatistics timeStats = new DescriptiveStatistics();
        DescriptiveStatistics trysStats = new DescriptiveStatistics();
        int count = 0;
        while (++count < 500) {
//            CC.resetTensorNames();
            Tensor t = rp.nextProduct(15);
            if (!(t instanceof Product))
                continue;


            Product from = (Product) t;


            long start = System.nanoTime();
            ProductsBijectionsPort port = new ProductsBijectionsPort(from.getContent(), from.getContent());


            int[] bijection;
            boolean good = false;
            int trys = 0;
            OUTER:
            while (trys++ < 5000 && (bijection = port.take()) != null) {
                for (int i = 0; i < bijection.length; ++i)
                    if (bijection[i] != i)
                        continue OUTER;
                good = true;
                break;
            }


            double millis = 1E-6 * (System.nanoTime() - start);
            timeStats.addValue(millis);
            trysStats.addValue(trys);


            if (!good)
                throw new RuntimeException();
        }
        System.out.println(timeStats);

View Full Code Here

            BufferedWriter timSigOut = new BufferedWriter(new FileWriter("/home/stas/Projects/stableSort/timSig.dat"));
            BufferedWriter insertionSigOut = new BufferedWriter(new FileWriter("/home/stas/Projects/stableSort/insertionSig.dat"));






            DescriptiveStatistics timSort;
            DescriptiveStatistics insertionSort;


            int tryies = 200;
            int arrayLength = 0;
            for (; arrayLength < 1000; ++arrayLength) {


                int[] coSort = nextArray(arrayLength, bitsStreamGenerator);


                timSort = new DescriptiveStatistics();
                insertionSort = new DescriptiveStatistics();
                for (int i = 0; i < tryies; ++i) {
                    int[] t1 = nextArray(arrayLength, bitsStreamGenerator);
                    int[] t2 = t1.clone();


                    long start = System.currentTimeMillis();
                    ArraysUtils.timSort(t1, coSort);
                    long stop = System.currentTimeMillis();
                    timSort.addValue(stop - start);


                    start = System.currentTimeMillis();
                    ArraysUtils.insertionSort(t2, coSort);
                    stop = System.currentTimeMillis();
                    insertionSort.addValue(stop - start);
                }
                timMeanOut.write(arrayLength + "\t" + timSort.getMean() + "\n");
                insertionMeanOut.write(arrayLength + "\t" + insertionSort.getMean() + "\n");


                timMaxOut.write(arrayLength + "\t" + timSort.getMax() + "\n");
                insertionMaxOut.write(arrayLength + "\t" + insertionSort.getMax() + "\n");


                timSigOut.write(arrayLength + "\t" + timSort.getStandardDeviation() + "\n");
                insertionSigOut.write(arrayLength + "\t" + insertionSort.getStandardDeviation() + "\n");
            }
            timMeanOut.close();
            insertionMeanOut.close();
            timMaxOut.close();
            insertionMaxOut.close();

View Full Code Here


    @Test
    public void testTableSampleBernoulli()
            throws Exception
    {
        DescriptiveStatistics stats = new DescriptiveStatistics();


        int total = computeExpected("SELECT orderkey FROM orders", ImmutableList.of(BIGINT)).getMaterializedRows().size();


        for (int i = 0; i < 100; i++) {
            List<MaterializedRow> values = computeActual("SELECT orderkey FROM ORDERS TABLESAMPLE BERNOULLI (50)").getMaterializedRows();


            assertEquals(values.size(), ImmutableSet.copyOf(values).size(), "TABLESAMPLE produced duplicate rows");
            stats.addValue(values.size() * 1.0 / total);
        }


        double mean = stats.getGeometricMean();
        assertTrue(mean > 0.45 && mean < 0.55, format("Expected mean sampling rate to be ~0.5, but was %s", mean));
    }

View Full Code Here


    @Test
    public void testTableSamplePoissonized()
            throws Exception
    {
        DescriptiveStatistics stats = new DescriptiveStatistics();


        int total = computeExpected("SELECT orderkey FROM orders", ImmutableList.of(BIGINT)).getMaterializedRows().size();


        for (int i = 0; i < 100; i++) {
            List<MaterializedRow> values = computeActual("SELECT orderkey FROM ORDERS TABLESAMPLE POISSONIZED (50)").getMaterializedRows();
            stats.addValue(values.size() * 1.0 / total);
        }


        double mean = stats.getGeometricMean();
        assertTrue(mean > 0.45 && mean < 0.55, format("Expected mean sampling rate to be ~0.5, but was %s", mean));
    }

View Full Code Here


    @Test
    public void testTableSamplePoissonizedRescaled()
            throws Exception
    {
        DescriptiveStatistics stats = new DescriptiveStatistics();


        long total = (long) computeExpected("SELECT COUNT(*) FROM orders", ImmutableList.of(BIGINT)).getMaterializedRows().get(0).getField(0);


        for (int i = 0; i < 100; i++) {
            String value = (String) computeActual("SELECT COUNT(*) FROM orders TABLESAMPLE POISSONIZED (50) RESCALED APPROXIMATE AT 95 CONFIDENCE").getMaterializedRows().get(0).getField(0);
            stats.addValue(Long.parseLong(value.split(" ")[0]) * 1.0 / total);
        }


        double mean = stats.getGeometricMean();
        assertTrue(mean > 0.90 && mean < 1.1, format("Expected sample to be rescaled to ~1.0, but was %s", mean));
        assertTrue(stats.getVariance() > 0, "Samples all had the exact same size");
    }

View Full Code Here


    @Test
    public void testTableSampleBernoulli()
            throws Exception
    {
        DescriptiveStatistics stats = new DescriptiveStatistics();


        int total = computeExpected("SELECT orderkey FROM orders", ImmutableList.of(BIGINT)).getMaterializedRows().size();


        for (int i = 0; i < 100; i++) {
            List<MaterializedRow> values = computeActual("SELECT orderkey FROM ORDERS TABLESAMPLE BERNOULLI (50)").getMaterializedRows();


            assertEquals(values.size(), ImmutableSet.copyOf(values).size(), "TABLESAMPLE produced duplicate rows");
            stats.addValue(values.size() * 1.0 / total);
        }


        double mean = stats.getGeometricMean();
        assertTrue(mean > 0.45 && mean < 0.55, format("Expected mean sampling rate to be ~0.5, but was %s", mean));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics

cc.redberry.core.performance.ProductBijectionPerformanceTest

cc.redberry.core.performance.StableSort

com.facebook.presto.AbstractTestQueries

com.facebook.presto.operator.aggregation.AbstractTestApproximateCountDistinct

com.facebook.presto.operator.aggregation.BootstrappedAggregation$BootstrappedAccumulator

com.facebook.presto.operator.aggregation.BootstrappedAggregation$BootstrappedGroupedAccumulator

com.facebook.presto.tests.AbstractTestQueries

org.apache.commons.math3.analysis.differentiation.DerivativeStructure

org.apache.commons.math3.analysis.differentiation.MultivariateDifferentiableVectorFunction

org.apache.commons.math3.analysis.function.HarmonicOscillator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.