Package de.lmu.ifi.dbs.elki.math

Examples of de.lmu.ifi.dbs.elki.math.DoubleMinMax


   */
  public OutlierResult run(Database database, Relation<?> relation) {
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);

    Pattern colSep = Pattern.compile(AbstractParser.WHITESPACE_PATTERN);
    DoubleMinMax minmax = new DoubleMinMax();
    InputStream in;
    try {
      in = FileUtil.tryGzipInput(new FileInputStream(file));
      BufferedReader reader = new BufferedReader(new InputStreamReader(in));

      for(String line; (line = reader.readLine()) != null;) {
        if(line.startsWith(COMMENT)) {
          continue;
        }
        else if(line.length() > 0) {
          String[] cols = colSep.split(line);
          Integer id = null;
          double score = Double.NaN;
          for(String str : cols) {
            Matcher mi = idpattern.matcher(str);
            Matcher ms = scorepattern.matcher(str);
            final boolean mif = mi.find();
            final boolean msf = ms.find();
            if(mif && msf) {
              throw new AbortException("ID pattern and score pattern both match value: " + str);
            }
            if(mif) {
              if(id != null) {
                throw new AbortException("ID pattern matched twice: previous value " + id + " second value: " + str);
              }
              id = Integer.parseInt(str.substring(mi.end()));
            }
            if(msf) {
              if(!Double.isNaN(score)) {
                throw new AbortException("Score pattern matched twice: previous value " + score + " second value: " + str);
              }
              score = Double.parseDouble(str.substring(ms.end()));
            }
          }
          if(id != null && !Double.isNaN(score)) {
            scores.putDouble(DBIDUtil.importInteger(id), score);
            minmax.put(score);
          }
          else if(id == null && Double.isNaN(score)) {
            logger.warning("Line did not match either ID nor score nor comment: " + line);
          }
          else {
            throw new AbortException("Line matched only ID or only SCORE patterns: " + line);
          }
        }
      }
    }
    catch(IOException e) {
      throw new AbortException("Could not load outlier scores: " + e.getMessage() + " when loading " + file, e);
    }

    OutlierScoreMeta meta;
    if(inverted) {
      meta = new InvertedOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    }
    else {
      meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    }
    Relation<Double> scoresult = new MaterializedRelation<Double>("External Outlier", "external-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs());
    OutlierResult or = new OutlierResult(meta, scoresult);

    // Apply scaling
    if(scaling instanceof OutlierScalingFunction) {
      ((OutlierScalingFunction) scaling).prepare(or);
    }
    DoubleMinMax mm = new DoubleMinMax();
    for(DBID id : relation.iterDBIDs()) {
      double val = scoresult.get(id); // scores.get(id);
      val = scaling.getScaled(val);
      scores.putDouble(id, val);
      mm.put(val);
    }
    meta = new BasicOutlierScoreMeta(mm.getMin(), mm.getMax());
    or = new OutlierResult(meta, scoresult);

    return or;
  }
View Full Code Here


    if(logger.isVerbose()) {
      logger.verbose("Computing the kNN outlier degree (distance to the k nearest neighbor)");
    }
    FiniteProgress progressKNNDistance = logger.isVerbose() ? new FiniteProgress("kNN distance for objects", relation.size(), logger) : null;

    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore knno_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    // compute distance to the k nearest neighbor.
    for(DBID id : relation.iterDBIDs()) {
      // distance to the kth nearest neighbor
      final KNNResult<D> knns = knnQuery.getKNNForDBID(id, k);
      double dkn = knns.getKNNDistance().doubleValue();
      knno_score.putDouble(id, dkn);

      minmax.put(dkn);

      if(progressKNNDistance != null) {
        progressKNNDistance.incrementProcessed(logger);
      }
    }
    if(progressKNNDistance != null) {
      progressKNNDistance.ensureCompleted(logger);
    }
    Relation<Double> scoreres = new MaterializedRelation<Double>("kNN Outlier Score", "knn-outlier", TypeUtil.DOUBLE, knno_score, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0);
    return new OutlierResult(meta, scoreres);
  }
View Full Code Here

  public OutlierResult run(Database database, Relation<O> relation) throws IllegalStateException {
    DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction());
    KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, k);

    // track the maximum value for normalization
    DoubleMinMax ldofminmax = new DoubleMinMax();
    // compute the ldof values
    WritableDoubleDataStore ldofs = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);

    // compute LOF_SCORE of each db object
    if(logger.isVerbose()) {
      logger.verbose("Computing LDOFs");
    }
    FiniteProgress progressLDOFs = logger.isVerbose() ? new FiniteProgress("LDOF_SCORE for objects", relation.size(), logger) : null;

    for(DBID id : relation.iterDBIDs()) {
      KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k);
      int nsize = neighbors.size() - 1;
      // skip the point itself
      double dxp = 0;
      double Dxp = 0;
      for(DistanceResultPair<D> neighbor1 : neighbors) {
        if(!neighbor1.getDBID().equals(id)) {
          dxp += neighbor1.getDistance().doubleValue();
          for(DistanceResultPair<D> neighbor2 : neighbors) {
            if(!neighbor1.getDBID().equals(neighbor2.getDBID()) && !neighbor2.getDBID().equals(id)) {
              Dxp += distFunc.distance(neighbor1.getDBID(), neighbor2.getDBID()).doubleValue();
            }
          }
        }
      }
      dxp /= nsize;
      Dxp /= (nsize * (nsize - 1));
      Double ldof = dxp / Dxp;
      if(ldof.isNaN() || ldof.isInfinite()) {
        ldof = 1.0;
      }
      ldofs.putDouble(id, ldof);
      // update maximum
      ldofminmax.put(ldof);

      if(progressLDOFs != null) {
        progressLDOFs.incrementProcessed(logger);
      }
    }
    if(progressLDOFs != null) {
      progressLDOFs.ensureCompleted(logger);
    }

    // Build result representation.
    Relation<Double> scoreResult = new MaterializedRelation<Double>("LDOF Outlier Score", "ldof-outlier", TypeUtil.DOUBLE, ldofs, relation.getDBIDs());
    OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(ldofminmax.getMin(), ldofminmax.getMax(), 0.0, Double.POSITIVE_INFINITY, LDOF_BASELINE);
    return new OutlierResult(scoreMeta, scoreResult);
  }
View Full Code Here

      stepprog.beginStep(3, "Computing LOFs.", logger);
    }
    Pair<WritableDoubleDataStore, DoubleMinMax> lofsAndMax = computeLOFs(ids, lrds, kNNRefer);
    WritableDoubleDataStore lofs = lofsAndMax.getFirst();
    // track the maximum value for normalization.
    DoubleMinMax lofminmax = lofsAndMax.getSecond();

    if(stepprog != null) {
      stepprog.setCompleted(logger);
    }

    // Build result representation.
    Relation<Double> scoreResult = new MaterializedRelation<Double>("Local Outlier Factor", "lof-outlier", TypeUtil.DOUBLE, lofs, ids);
    OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(lofminmax.getMin(), lofminmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 1.0);
    OutlierResult result = new OutlierResult(scoreMeta, scoreResult);

    return new LOFResult<O, D>(result, kNNRefer, kNNReach, lrds, lofs);
  }
View Full Code Here

    }
    // Finalize covariance matrix:
    Vector mean = covmaker.getMeanVector();
    Matrix cmati = covmaker.destroyToSampleMatrix().inverse();

    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(attributes.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : attributes.iterDBIDs()) {
      Vector temp = deltas.get(id).minus(mean);
      final double score = temp.transposeTimesTimes(cmati, temp);
      minmax.put(score);
      scores.putDouble(id, score);
    }

    Relation<Double> scoreResult = new MaterializedRelation<Double>("mean multiple attributes spatial outlier", "mean-multipleattributes-outlier", TypeUtil.DOUBLE, scores, attributes.getDBIDs());
    OutlierScoreMeta scoreMeta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0);
    OutlierResult or = new OutlierResult(scoreMeta, scoreResult);
    or.addChildResult(npred);
    return or;
  }
View Full Code Here

   * @return the LOFs of the objects and the maximum LOF
   */
  protected Pair<WritableDoubleDataStore, DoubleMinMax> computeLOFs(DBIDs ids, DataStore<Double> lrds, KNNQuery<O, D> knnRefer) {
    WritableDoubleDataStore lofs = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC);
    // track the maximum value for normalization.
    DoubleMinMax lofminmax = new DoubleMinMax();

    FiniteProgress progressLOFs = logger.isVerbose() ? new FiniteProgress("LOF_SCORE for objects", ids.size(), logger) : null;
    for(DBID id : ids) {
      double lrdp = lrds.get(id);
      final double lof;
      if(lrdp > 0) {
        final KNNResult<D> neighbors = knnRefer.getKNNForDBID(id, k);
        int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1);
        // skip the point itself
        // neighbors.remove(0);
        double sum = 0;
        for(DistanceResultPair<D> neighbor : neighbors) {
          if(objectIsInKNN || !neighbor.getDBID().equals(id)) {
            sum += lrds.get(neighbor.getDBID());
          }
        }
        lof = (sum / nsize) / lrdp;
      }
      else {
        lof = 1.0;
      }
      lofs.putDouble(id, lof);
      // update minimum and maximum
      lofminmax.put(lof);

      if(progressLOFs != null) {
        progressLOFs.incrementProcessed(logger);
      }
    }
View Full Code Here

   */
  public OutlierResult run(Relation<V> relation) throws IllegalStateException {
    SimilarityQuery<V, D> snnInstance = similarityFunction.instantiate(relation);
    FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Assigning Subspace Outlier Degree", relation.size(), logger) : null;
    WritableDataStore<SODModel<?>> sod_models = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC, SODModel.class);
    DoubleMinMax minmax = new DoubleMinMax();
    for(Iterator<DBID> iter = relation.iterDBIDs(); iter.hasNext();) {
      DBID queryObject = iter.next();
      if(progress != null) {
        progress.incrementProcessed(logger);
      }
      DBIDs knnList = getNearestNeighbors(relation, snnInstance, queryObject);
      SODModel<V> model = new SODModel<V>(relation, knnList, alpha, relation.get(queryObject));
      sod_models.put(queryObject, model);
      minmax.put(model.getSod());
    }
    if(progress != null) {
      progress.ensureCompleted(logger);
    }
    // combine results.
    Relation<SODModel<?>> models = new MaterializedRelation<SODModel<?>>("Subspace Outlier Model", "sod-outlier", new SimpleTypeInformation<SODModel<?>>(SODModel.class), sod_models, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    OutlierResult sodResult = new OutlierResult(meta, new SODProxyScoreResult(models, relation.getDBIDs()));
    // also add the models.
    sodResult.addChildResult(models);
    return sodResult;
  }
View Full Code Here

      }
    }

    // Calculate INFLO for any Object
    // IF Object is pruned INFLO=1.0
    DoubleMinMax inflominmax = new DoubleMinMax();
    WritableDoubleDataStore inflos = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      if(!pruned.contains(id)) {
        ModifiableDBIDs knn = knns.get(id);
        ModifiableDBIDs rnn = rnns.get(id);

        double denP = density.doubleValue(id);
        knn.addDBIDs(rnn);
        double den = 0;
        for(DBID q : knn) {
          double denQ = density.doubleValue(q);
          den = den + denQ;
        }
        den = den / rnn.size();
        den = den / denP;
        inflos.putDouble(id, den);
        // update minimum and maximum
        inflominmax.put(den);

      }
      if(pruned.contains(id)) {
        inflos.putDouble(id, 1.0);
        inflominmax.put(1.0);
      }
    }

    // Build result representation.
    Relation<Double> scoreResult = new MaterializedRelation<Double>("Influence Outlier Score", "inflo-outlier", TypeUtil.DOUBLE, inflos, relation.getDBIDs());
    OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(inflominmax.getMin(), inflominmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 1.0);
    return new OutlierResult(scoreMeta, scoreResult);
  }
View Full Code Here

    if(logger.isVerbose()) {
      logger.verbose("computing outlier degree(sum of the distances to the k nearest neighbors");
    }
    FiniteProgress progressKNNWeight = logger.isVerbose() ? new FiniteProgress("KNNWOD_KNNWEIGHT for objects", relation.size(), logger) : null;

    DoubleMinMax minmax = new DoubleMinMax();

    // compute distance to the k nearest neighbor. n objects with the highest
    // distance are flagged as outliers
    WritableDoubleDataStore knnw_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      // compute sum of the distances to the k nearest neighbors

      final KNNResult<D> knn = knnQuery.getKNNForDBID(id, k);
      double skn = 0;
      for(DistanceResultPair<D> r : knn) {
        skn += r.getDistance().doubleValue();
      }
      knnw_score.putDouble(id, skn);
      minmax.put(skn);

      if(progressKNNWeight != null) {
        progressKNNWeight.incrementProcessed(logger);
      }
    }
    if(progressKNNWeight != null) {
      progressKNNWeight.ensureCompleted(logger);
    }

    Relation<Double> res = new MaterializedRelation<Double>("Weighted kNN Outlier Score", "knnw-outlier", TypeUtil.DOUBLE, knnw_score, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0);
    return new OutlierResult(meta, res);
  }
View Full Code Here

      reachDistance.put(id, core);
      lrds.putDouble(id, lrd);
    }

    // Pass 3
    DoubleMinMax ofminmax = new DoubleMinMax();
    WritableDoubleDataStore ofs = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      double of = 0;
      for(DistanceResultPair<D> pair : nMinPts.get(id)) {
        DBID idN = pair.getDBID();
        double lrd = lrds.doubleValue(id);
        double lrdN = lrds.doubleValue(idN);
        of = of + lrdN / lrd;
      }
      of = of / minPtsNeighborhoodSize.get(id);
      ofs.putDouble(id, of);
      // update minimum and maximum
      ofminmax.put(of);
    }
    // Build result representation.
    Relation<Double> scoreResult = new MaterializedRelation<Double>("OPTICS Outlier Scores", "optics-outlier", TypeUtil.DOUBLE, ofs, relation.getDBIDs());
    OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(ofminmax.getMin(), ofminmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 1.0);
    return new OutlierResult(scoreMeta, scoreResult);
  }
View Full Code Here

TOP

Related Classes of de.lmu.ifi.dbs.elki.math.DoubleMinMax

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.