Examples of de.lmu.ifi.dbs.elki.math.MeanVariance

de.lmu.ifi.dbs.elki.math.MeanVariance
Do some simple statistics (mean, variance) using a numerically stable online algorithm. This class can repeatedly be fed with data using the add() methods, the resulting values for mean and average can be queried at any time using getMean() and getSampleVariance(). Make sure you have understood variance correctly when using getNaiveVariance() - since this class is fed with samples and estimates the mean from the samples, getSampleVariance() is the proper formula. Trivial code, but replicated a lot. The class is final so it should come at low cost. Related Literature:
B. P. Welford
Note on a method for calculating corrected sums of squares and products
in: Technometrics 4(3)

D.H.D. West
Updating Mean and Variance Estimates: An Improved Method
In: Communications of the ACM, Volume 22 Issue 9
@author Erich Schubert

          final double r = c.first;
          final double alpha_r = alpha * r;
          // compute n(p_i, \alpha * r) from list (note: alpha_r is different from c!)
          final int n_alphar = elementsAtRadius(cdist, alpha_r);
          // compute \hat{n}(p_i, r, \alpha) and the corresponding \simga_{MDEF}
          MeanVariance mv_n_r_alpha = new MeanVariance();
          for(DistanceResultPair<D> ne : maxneighbors) {
            // Stop at radius r
            if(ne.getDistance().doubleValue() > r) {
              break;
            }
            int rn_alphar = elementsAtRadius(interestingDistances.get(ne.getDBID()), alpha_r);
            mv_n_r_alpha.put(rn_alphar);
          }
          // We only use the average and standard deviation
          final double nhat_r_alpha = mv_n_r_alpha.getMean();
          final double sigma_nhat_r_alpha = mv_n_r_alpha.getNaiveStddev();


          // Redundant divisions removed.
          final double mdef = (nhat_r_alpha - n_alphar); // / nhat_r_alpha;
          final double sigmamdef = sigma_nhat_r_alpha; // / nhat_r_alpha;
          final double mdefnorm = mdef / sigmamdef;

View Full Code Here

      inter = meanv[1] - slope * meanv[0];
    }


    // calculate mean and variance for error
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    MeanVariance mv = new MeanVariance();
    for(DBID id : relation.iterDBIDs()) {
      // Compute the error from the linear regression
      double y_i = relation.get(id).doubleValue(1);
      double e = means.doubleValue(id) - (slope * y_i + inter);
      scores.putDouble(id, e);
      mv.put(e);
    }


    // Normalize scores
    DoubleMinMax minmax = new DoubleMinMax();
    {
      final double mean = mv.getMean();
      final double variance = mv.getNaiveStddev();
      for(DBID id : relation.iterDBIDs()) {
        double score = Math.abs((scores.doubleValue(id) - mean) / variance);
        minmax.put(score);
        scores.putDouble(id, score);
      }

View Full Code Here

   */
  public OutlierResult run(Relation<N> nrel, Relation<? extends NumberVector<?, ?>> relation) {
    final NeighborSetPredicate npred = getNeighborSetPredicateFactory().instantiate(nrel);
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);


    MeanVariance mv = new MeanVariance();
    for(DBID id : relation.iterDBIDs()) {
      DBIDs neighbors = npred.getNeighborDBIDs(id);
      final double median;
      {
        double[] fi = new double[neighbors.size()];
        // calculate and store Median of neighborhood
        int c = 0;
        for(DBID n : neighbors) {
          if(id.equals(n)) {
            continue;
          }
          fi[c] = relation.get(n).doubleValue(1);
          c++;
        }


        if(c > 0) {
          median = QuickSelect.median(fi, 0, c);
        }
        else {
          median = relation.get(id).doubleValue(1);
        }
      }
      double h = relation.get(id).doubleValue(1) - median;
      scores.putDouble(id, h);
      mv.put(h);
    }


    // Normalize scores
    final double mean = mv.getMean();
    final double stddev = mv.getNaiveStddev();
    DoubleMinMax minmax = new DoubleMinMax();
    for(DBID id : relation.iterDBIDs()) {
      double score = Math.abs((scores.doubleValue(id) - mean) / stddev);
      minmax.put(score);
      scores.putDouble(id, score);

View Full Code Here

    // preprocess kNN neighborhoods
    assert (k == this.k);
    KNNQuery<V, DoubleDistance> knnQuery = QueryUtil.getKNNQuery(relation, getDistanceFunction(), k);


    for(DBID objKey : relation.iterDBIDs()) {
      MeanVariance s = new MeanVariance();


      // System.out.println("Processing: " +objKey);
      KNNResult<DoubleDistance> neighbors = knnQuery.getKNNForDBID(objKey, k);
      Iterator<DistanceResultPair<DoubleDistance>> iter = neighbors.iterator();
      while(iter.hasNext()) {
        DBID key1 = iter.next().getDBID();
        // Iterator iter2 = data.keyIterator();
        Iterator<DistanceResultPair<DoubleDistance>> iter2 = neighbors.iterator();
        // PriorityQueue best = new PriorityQueue(false, k);
        while(iter2.hasNext()) {
          DBID key2 = iter2.next().getDBID();
          if(key2.equals(key1) || key1.equals(objKey) || key2.equals(objKey)) {
            continue;
          }
          double nenner = calcDenominator(kernelMatrix, objKey, key1, key2);


          if(nenner != 0) {
            double sqrtnenner = Math.sqrt(nenner);
            double tmp = calcNumerator(kernelMatrix, objKey, key1, key2) / nenner;
            s.put(tmp, 1 / sqrtnenner);
          }


        }
      }
      // Sample variance probably would be correct, however the numerical
      // instabilities can actually break ABOD here.
      pq.add(new DoubleObjPair<DBID>(s.getNaiveVariance(), objKey));
    }


    DoubleMinMax minmaxabod = new DoubleMinMax();
    WritableDoubleDataStore abodvalues = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DoubleObjPair<DBID> pair : pq) {

View Full Code Here

      // } else {
      // System.out.println("Best Candidate " + aKey+" : " + pq.firstPriority()
      // + " worst result: " + Double.MAX_VALUE);
      // }
      // v++;
      MeanVariance s = new MeanVariance();
      for(DBID bKey : relation.iterDBIDs()) {
        if(bKey.equals(aKey)) {
          continue;
        }
        for(DBID cKey : relation.iterDBIDs()) {
          if(cKey.equals(aKey)) {
            continue;
          }
          // double nenner = dists[y]*dists[z];
          double nenner = calcDenominator(kernelMatrix, aKey, bKey, cKey);
          if(nenner != 0) {
            double tmp = calcNumerator(kernelMatrix, aKey, bKey, cKey) / nenner;
            double sqrtNenner = Math.sqrt(nenner);
            s.put(tmp, 1 / sqrtNenner);
          }
        }
      }
      // System.out.println( aKey + "Sum " + sum + " SQRSum " +sqrSum +
      // " Counter " + counter);
      double var = s.getSampleVariance();
      // System.out.println(aKey+ " : " + approx +" " + var);
      if(resqueue.size() < k) {
        resqueue.add(new DoubleObjPair<DBID>(var, aKey));
      }
      else {

View Full Code Here

    // PQ for Outlier Ranking
    Heap<DoubleObjPair<DBID>> pq = new Heap<DoubleObjPair<DBID>>(data.size(), Collections.reverseOrder());
    HashMap<DBID, DBIDs> explaintab = new HashMap<DBID, DBIDs>();
    // test all objects
    for(DBID objKey : data.iterDBIDs()) {
      MeanVariance s = new MeanVariance();
      // Queue for the best explanation
      Heap<DoubleObjPair<DBID>> explain = new Heap<DoubleObjPair<DBID>>();
      // determine Object
      // for each pair of other objects
      Iterator<DBID> iter = data.iterDBIDs();
      // Collect Explanation Vectors
      while(iter.hasNext()) {
        MeanVariance s2 = new MeanVariance();
        DBID key1 = iter.next();
        Iterator<DBID> iter2 = data.iterDBIDs();
        if(objKey.equals(key1)) {
          continue;
        }
        while(iter2.hasNext()) {
          DBID key2 = iter2.next();
          if(key2.equals(key1) || objKey.equals(key2)) {
            continue;
          }
          double nenner = calcDenominator(kernelMatrix, objKey, key1, key2);
          if(nenner != 0) {
            double tmp = calcNumerator(kernelMatrix, objKey, key1, key2) / nenner;
            double sqr = Math.sqrt(nenner);
            s2.put(tmp, 1 / sqr);
          }
        }
        explain.add(new DoubleObjPair<DBID>(s2.getSampleVariance(), key1));
        s.put(s2);
      }
      // build variance of the observed vectors
      pq.add(new DoubleObjPair<DBID>(s.getSampleVariance(), objKey));
      //

View Full Code Here

        }
      }
    }
    // Compute PLOF values.
    WritableDoubleDataStore plofs = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    MeanVariance mvplof = new MeanVariance();
    {// compute LOOP_SCORE of each db object
      if(stepprog != null) {
        stepprog.beginStep(4, "Computing PLOF", logger);
      }


      FiniteProgress progressPLOFs = logger.isVerbose() ? new FiniteProgress("PLOFs for objects", relation.size(), logger) : null;
      for(DBID id : relation.iterDBIDs()) {
        final KNNResult<D> neighbors = knnComp.getKNNForDBID(id, kcomp);
        MeanVariance mv = new MeanVariance();
        // use first kref neighbors as comparison set.
        int ks = 0;
        for(DistanceResultPair<D> neighbor1 : neighbors) {
          if(objectIsInKNN || !neighbor1.getDBID().equals(id)) {
            mv.put(pdists.doubleValue(neighbor1.getDBID()));
            ks++;
            if(ks >= kcomp) {
              break;
            }
          }
        }
        double plof = Math.max(pdists.doubleValue(id) / mv.getMean(), 1.0);
        if(Double.isNaN(plof) || Double.isInfinite(plof)) {
          plof = 1.0;
        }
        plofs.putDouble(id, plof);
        mvplof.put((plof - 1.0) * (plof - 1.0));

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of de.lmu.ifi.dbs.elki.math.MeanVariance

de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.LMCLUS

de.lmu.ifi.dbs.elki.algorithm.outlier.ABOD

de.lmu.ifi.dbs.elki.algorithm.outlier.LOCI

de.lmu.ifi.dbs.elki.algorithm.outlier.LoOP

de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMedianAlgorithm

de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMoranScatterplotOutlier

de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuScatterplotOutlier

de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuZTestOutlier

de.lmu.ifi.dbs.elki.algorithm.statistics.DistanceStatisticsWithClasses

de.lmu.ifi.dbs.elki.algorithm.statistics.RankingQualityHistogram

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.