Examples of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress

Package de.lmu.ifi.dbs.elki.logging.progress

Examples of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress

de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress
A progress object for a given overall number of items to process. The number of already processed items at a point in time can be updated. The main feature of this class is to provide a String representation of the progress suitable as a message for printing to the command line interface. @author Arthur Zimek


    ModifiableHierarchy<Cluster<Model>> hier = new HierarchyHashmapList<Cluster<Model>>();
    Map<DBID, Cluster<Model>> clusters = new HashMap<DBID, Cluster<Model>>();
    Map<DBID, ModifiableDBIDs> cids = new HashMap<DBID, ModifiableDBIDs>();


    FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Extracting clusters", ids.size(), logger) : null;


    for(DBID cur : order) {
      DBID dest = pi.get(cur);
      D l = lambda.get(cur);
      // logger.debugFine("DBID " + cur.toString() + " dist: " + l.toString());
      if(stopdist != null && stopdist.compareTo(l) > 0) {
        ModifiableDBIDs curset = cids.remove(cur);
        ModifiableDBIDs destset = cids.get(dest);
        if(destset == null) {
          if(curset != null) {
            destset = curset;
          }
          else {
            destset = DBIDUtil.newHashSet();
            destset.add(cur);
          }
          destset.add(dest);
          cids.put(dest, destset);
        }
        else {
          if(curset != null) {
            destset.addDBIDs(curset);
          }
          else {
            destset.add(cur);
          }
        }
        curdist = l;
      }
      else {
        if(curdist == null || l.compareTo(curdist) > 0) {
          // New distance level reached. Post-process the current objects
          for(Entry<DBID, ModifiableDBIDs> ent : cids.entrySet()) {
            DBID key = ent.getKey();
            ModifiableDBIDs clusids = ent.getValue();
            // Make a new cluster
            String cname = "Cluster_" + key.toString() + "_" + curdist.toString();
            Cluster<Model> cluster = new Cluster<Model>(cname, clusids, ClusterModel.CLUSTER, hier);
            // Collect child clusters and clean up the cluster ids, keeping only
            // "new" objects.
            Iterator<DBID> iter = clusids.iterator();
            while(iter.hasNext()) {
              DBID child = iter.next();
              Cluster<Model> chiclus = clusters.get(child);
              if(chiclus != null) {
                hier.add(cluster, chiclus);
                clusters.remove(child);
                iter.remove();
              }
            }
            clusters.put(key, cluster);
          }
          if(logger.isDebuggingFine()) {
            StringBuffer buf = new StringBuffer();
            buf.append("Number of clusters at depth ");
            buf.append((curdist != null ? curdist.toString() : "null"));
            buf.append(": ").append(clusters.size()).append(" ");
            buf.append("last-objects:");
            for(DBID id : clusters.keySet()) {
              buf.append(" ").append(id.toString());
            }
            logger.debugFine(buf.toString());
          }
          cids.clear();
          curdist = l;
        }
        // Add the current object to the destinations cluster
        {
          ModifiableDBIDs destset = cids.get(dest);
          if(destset == null) {
            destset = DBIDUtil.newHashSet();
            cids.put(dest, destset);
            destset.add(dest);
          }
          destset.add(cur);
        }
      }
      // Decrement counter
      if(progress != null) {
        progress.incrementProcessed(logger);
      }
    }
    if(progress != null) {
      progress.ensureCompleted(logger);
    }
    // There should be one cluster remaining at infinite distance...
    if(clusters.size() != 1) {
      logger.warning("Single-link is expected to have a single cluster at the top level!");
      return null;

View Full Code Here

   * @return Result
   */
  public Clustering<Model> run(Database database, Relation<O> relation) {
    SimilarityQuery<O, IntegerDistance> snnInstance = similarityFunction.instantiate(relation);


    FiniteProgress objprog = logger.isVerbose() ? new FiniteProgress("SNNClustering", relation.size(), logger) : null;
    IndefiniteProgress clusprog = logger.isVerbose() ? new IndefiniteProgress("Number of clusters", logger) : null;
    resultList = new ArrayList<ModifiableDBIDs>();
    noise = DBIDUtil.newHashSet();
    processedIDs = DBIDUtil.newHashSet(relation.size());
    if(relation.size() >= minpts) {
      for(DBID id : snnInstance.getRelation().iterDBIDs()) {
        if(!processedIDs.contains(id)) {
          expandCluster(snnInstance, id, objprog, clusprog);
          if(processedIDs.size() == relation.size() && noise.size() == 0) {
            break;
          }
        }
        if(objprog != null && clusprog != null) {
          objprog.setProcessed(processedIDs.size(), logger);
          clusprog.setProcessed(resultList.size(), logger);
        }
      }
    }
    else {
      for(DBID id : snnInstance.getRelation().iterDBIDs()) {
        noise.add(id);
        if(objprog != null && clusprog != null) {
          objprog.setProcessed(noise.size(), logger);
          clusprog.setProcessed(resultList.size(), logger);
        }
      }
    }
    // Finish progress logging
    if(objprog != null && clusprog != null) {
      objprog.ensureCompleted(logger);
      clusprog.setCompleted(logger);
    }


    Clustering<Model> result = new Clustering<Model>("Shared-Nearest-Neighbor Clustering", "snn-clustering");
    for(Iterator<ModifiableDBIDs> resultListIter = resultList.iterator(); resultListIter.hasNext();) {

View Full Code Here

    this.distanceFunction = distanceFunction;
    this.lambda = lambda;
  }


  public Clustering<Model> run(Database database, Relation<V> relation) throws IllegalStateException {
    FiniteProgress objprog = getLogger().isVerbose() ? new FiniteProgress("Processing objects", relation.size(), getLogger()) : null;
    IndefiniteProgress clusprog = getLogger().isVerbose() ? new IndefiniteProgress("Number of clusters", getLogger()) : null;
    resultList = new ArrayList<ModifiableDBIDs>();
    noise = DBIDUtil.newHashSet();
    processedIDs = DBIDUtil.newHashSet(relation.size());


    LocallyWeightedDistanceFunction.Instance<V> distFunc = distanceFunction.instantiate(relation);
    RangeQuery<V, DoubleDistance> rangeQuery = database.getRangeQuery(distFunc);


    if(relation.size() >= minpts) {
      for(DBID id : relation.iterDBIDs()) {
        if(!processedIDs.contains(id)) {
          expandCluster(distFunc, rangeQuery, id, objprog, clusprog);
          if(processedIDs.size() == relation.size() && noise.size() == 0) {
            break;
          }
        }
        if(objprog != null && clusprog != null) {
          objprog.setProcessed(processedIDs.size(), getLogger());
          clusprog.setProcessed(resultList.size(), getLogger());
        }
      }
    }
    else {
      for(DBID id : relation.iterDBIDs()) {
        noise.add(id);
        if(objprog != null && clusprog != null) {
          objprog.setProcessed(processedIDs.size(), getLogger());
          clusprog.setProcessed(resultList.size(), getLogger());
        }
      }
    }


    if(objprog != null && clusprog != null) {
      objprog.setProcessed(processedIDs.size(), getLogger());
      clusprog.setProcessed(resultList.size(), getLogger());
    }


    Clustering<Model> result = new Clustering<Model>(getLongResultName(), getShortResultName());
    for(Iterator<ModifiableDBIDs> resultListIter = resultList.iterator(); resultListIter.hasNext();) {
      Cluster<Model> c = new Cluster<Model>(resultListIter.next(), ClusterModel.CLUSTER);
      result.addCluster(c);
    }


    Cluster<Model> n = new Cluster<Model>(noise, true, ClusterModel.CLUSTER);
    result.addCluster(n);


    if(objprog != null && clusprog != null) {
      objprog.setProcessed(processedIDs.size(), getLogger());
      clusprog.setProcessed(resultList.size(), getLogger());
    }
    // Signal that the progress has completed.
    if(objprog != null && clusprog != null) {
      objprog.ensureCompleted(getLogger());
      clusprog.setCompleted(getLogger());
    }
    return result;
  }

View Full Code Here

    KNNQuery<O, D> knnQuery = database.getKNNQuery(distanceQuery, k);


    if(logger.isVerbose()) {
      logger.verbose("Computing the kNN outlier degree (distance to the k nearest neighbor)");
    }
    FiniteProgress progressKNNDistance = logger.isVerbose() ? new FiniteProgress("kNN distance for objects", relation.size(), logger) : null;


    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore knno_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    // compute distance to the k nearest neighbor.
    for(DBID id : relation.iterDBIDs()) {
      // distance to the kth nearest neighbor
      final KNNResult<D> knns = knnQuery.getKNNForDBID(id, k);
      double dkn = knns.getKNNDistance().doubleValue();
      knno_score.putDouble(id, dkn);


      minmax.put(dkn);


      if(progressKNNDistance != null) {
        progressKNNDistance.incrementProcessed(logger);
      }
    }
    if(progressKNNDistance != null) {
      progressKNNDistance.ensureCompleted(logger);
    }
    Relation<Double> scoreres = new MaterializedRelation<Double>("kNN Outlier Score", "knn-outlier", TypeUtil.DOUBLE, knno_score, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0);
    return new OutlierResult(meta, scoreres);
  }

View Full Code Here


    // compute LOF_SCORE of each db object
    if(logger.isVerbose()) {
      logger.verbose("Computing LDOFs");
    }
    FiniteProgress progressLDOFs = logger.isVerbose() ? new FiniteProgress("LDOF_SCORE for objects", relation.size(), logger) : null;


    for(DBID id : relation.iterDBIDs()) {
      KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k);
      int nsize = neighbors.size() - 1;
      // skip the point itself
      double dxp = 0;
      double Dxp = 0;
      for(DistanceResultPair<D> neighbor1 : neighbors) {
        if(!neighbor1.getDBID().equals(id)) {
          dxp += neighbor1.getDistance().doubleValue();
          for(DistanceResultPair<D> neighbor2 : neighbors) {
            if(!neighbor1.getDBID().equals(neighbor2.getDBID()) && !neighbor2.getDBID().equals(id)) {
              Dxp += distFunc.distance(neighbor1.getDBID(), neighbor2.getDBID()).doubleValue();
            }
          }
        }
      }
      dxp /= nsize;
      Dxp /= (nsize * (nsize - 1));
      Double ldof = dxp / Dxp;
      if(ldof.isNaN() || ldof.isInfinite()) {
        ldof = 1.0;
      }
      ldofs.putDouble(id, ldof);
      // update maximum
      ldofminmax.put(ldof);


      if(progressLDOFs != null) {
        progressLDOFs.incrementProcessed(logger);
      }
    }
    if(progressLDOFs != null) {
      progressLDOFs.ensureCompleted(logger);
    }


    // Build result representation.
    Relation<Double> scoreResult = new MaterializedRelation<Double>("LDOF Outlier Score", "ldof-outlier", TypeUtil.DOUBLE, ldofs, relation.getDBIDs());
    OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(ldofminmax.getMin(), ldofminmax.getMax(), 0.0, Double.POSITIVE_INFINITY, LDOF_BASELINE);

View Full Code Here

   *        reachability distance
   * @return the LRDs of the objects
   */
  protected WritableDoubleDataStore computeLRDs(DBIDs ids, KNNQuery<O, D> knnReach) {
    WritableDoubleDataStore lrds = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    FiniteProgress lrdsProgress = logger.isVerbose() ? new FiniteProgress("LRD", ids.size(), logger) : null;
    for(DBID id : ids) {
      double sum = 0;
      KNNResult<D> neighbors = knnReach.getKNNForDBID(id, k);
      int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1);
      for(DistanceResultPair<D> neighbor : neighbors) {
        if(objectIsInKNN || !neighbor.getDBID().equals(id)) {
          KNNResult<D> neighborsNeighbors = knnReach.getKNNForDBID(neighbor.getDBID(), k);
          sum += Math.max(neighbor.getDistance().doubleValue(), neighborsNeighbors.getKNNDistance().doubleValue());
        }
      }
      // Avoid division by 0
      double lrd = (sum > 0) ? nsize / sum : 0.0;
      lrds.putDouble(id, lrd);
      if(lrdsProgress != null) {
        lrdsProgress.incrementProcessed(logger);
      }
    }
    if(lrdsProgress != null) {
      lrdsProgress.ensureCompleted(logger);
    }
    return lrds;
  }

View Full Code Here

  protected Pair<WritableDoubleDataStore, DoubleMinMax> computeLOFs(DBIDs ids, DataStore<Double> lrds, KNNQuery<O, D> knnRefer) {
    WritableDoubleDataStore lofs = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC);
    // track the maximum value for normalization.
    DoubleMinMax lofminmax = new DoubleMinMax();


    FiniteProgress progressLOFs = logger.isVerbose() ? new FiniteProgress("LOF_SCORE for objects", ids.size(), logger) : null;
    for(DBID id : ids) {
      double lrdp = lrds.get(id);
      final double lof;
      if(lrdp > 0) {
        final KNNResult<D> neighbors = knnRefer.getKNNForDBID(id, k);
        int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1);
        // skip the point itself
        // neighbors.remove(0);
        double sum = 0;
        for(DistanceResultPair<D> neighbor : neighbors) {
          if(objectIsInKNN || !neighbor.getDBID().equals(id)) {
            sum += lrds.get(neighbor.getDBID());
          }
        }
        lof = (sum / nsize) / lrdp;
      }
      else {
        lof = 1.0;
      }
      lofs.putDouble(id, lof);
      // update minimum and maximum
      lofminmax.put(lof);


      if(progressLOFs != null) {
        progressLOFs.incrementProcessed(logger);
      }
    }
    if(progressLOFs != null) {
      progressLOFs.ensureCompleted(logger);
    }
    return new Pair<WritableDoubleDataStore, DoubleMinMax>(lofs, lofminmax);
  }

View Full Code Here

   * 
   * @param relation Data relation to process
   */
  public OutlierResult run(Relation<V> relation) throws IllegalStateException {
    SimilarityQuery<V, D> snnInstance = similarityFunction.instantiate(relation);
    FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Assigning Subspace Outlier Degree", relation.size(), logger) : null;
    WritableDataStore<SODModel<?>> sod_models = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC, SODModel.class);
    DoubleMinMax minmax = new DoubleMinMax();
    for(Iterator<DBID> iter = relation.iterDBIDs(); iter.hasNext();) {
      DBID queryObject = iter.next();
      if(progress != null) {
        progress.incrementProcessed(logger);
      }
      DBIDs knnList = getNearestNeighbors(relation, snnInstance, queryObject);
      SODModel<V> model = new SODModel<V>(relation, knnList, alpha, relation.get(queryObject));
      sod_models.put(queryObject, model);
      minmax.put(model.getSod());
    }
    if(progress != null) {
      progress.ensureCompleted(logger);
    }
    // combine results.
    Relation<SODModel<?>> models = new MaterializedRelation<SODModel<?>>("Subspace Outlier Model", "sod-outlier", new SimpleTypeInformation<SODModel<?>>(SODModel.class), sod_models, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    OutlierResult sodResult = new OutlierResult(meta, new SODProxyScoreResult(models, relation.getDBIDs()));

View Full Code Here

    KNNQuery<O, D> knnQuery = database.getKNNQuery(distanceQuery, k);


    if(logger.isVerbose()) {
      logger.verbose("computing outlier degree(sum of the distances to the k nearest neighbors");
    }
    FiniteProgress progressKNNWeight = logger.isVerbose() ? new FiniteProgress("KNNWOD_KNNWEIGHT for objects", relation.size(), logger) : null;


    DoubleMinMax minmax = new DoubleMinMax();


    // compute distance to the k nearest neighbor. n objects with the highest
    // distance are flagged as outliers
    WritableDoubleDataStore knnw_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      // compute sum of the distances to the k nearest neighbors


      final KNNResult<D> knn = knnQuery.getKNNForDBID(id, k);
      double skn = 0;
      for(DistanceResultPair<D> r : knn) {
        skn += r.getDistance().doubleValue();
      }
      knnw_score.putDouble(id, skn);
      minmax.put(skn);


      if(progressKNNWeight != null) {
        progressKNNWeight.incrementProcessed(logger);
      }
    }
    if(progressKNNWeight != null) {
      progressKNNWeight.ensureCompleted(logger);
    }


    Relation<Double> res = new MaterializedRelation<Double>("Weighted kNN Outlier Score", "knnw-outlier", TypeUtil.DOUBLE, knnw_score, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0);
    return new OutlierResult(meta, res);

View Full Code Here

    final NeighborSetPredicate npred = getNeighborSetPredicateFactory().instantiate(nrel);


    WritableDoubleDataStore errors = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP);
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);


    FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Computing trimmed means", relation.size(), logger) : null;
    for(DBID id : relation.iterDBIDs()) {
      DBIDs neighbors = npred.getNeighborDBIDs(id);
      int num = 0;
      double[] values = new double[neighbors.size()];
      // calculate trimmedMean
      for(DBID n : neighbors) {
        values[num] = relation.get(n).doubleValue(1);
        num++;
      }


      // calculate local trimmed Mean and error term
      final double tm;
      if(num > 0) {
        int left = (int) Math.floor(p * (num - 1));
        int right = (int) Math.floor((1 - p) * (num - 1));
        Arrays.sort(values, 0, num);
        Mean mean = new Mean();
        for(int i = left; i <= right; i++) {
          mean.put(values[i]);
        }
        tm = mean.getMean();
      }
      else {
        tm = relation.get(id).doubleValue(1);
      }
      // Error: deviation from trimmed mean
      errors.putDouble(id, relation.get(id).doubleValue(1) - tm);


      if(progress != null) {
        progress.incrementProcessed(logger);
      }
    }
    if(progress != null) {
      progress.ensureCompleted(logger);
    }


    if(logger.isVerbose()) {
      logger.verbose("Computing median error.");
    }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress

de.lmu.ifi.dbs.elki.algorithm.clustering.AbstractProjectedDBSCAN

de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.CASH

de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.COPAC

de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.LMCLUS

de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN

de.lmu.ifi.dbs.elki.algorithm.clustering.DeLiClu

de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICS

de.lmu.ifi.dbs.elki.algorithm.clustering.SLINK

de.lmu.ifi.dbs.elki.algorithm.clustering.SNNClustering

de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.DiSH

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.