Package de.lmu.ifi.dbs.elki.database.ids

Examples of de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs

338
339
340
341
342
343
344
345
346
347
        adjustApproximatedKNNDistances(dirEntry, knnLists);
      }
    }

    // PolynomialApproximation approx1 = node.knnDistanceApproximation();
    ArrayModifiableDBIDs ids = DBIDUtil.newArray();
    leafEntryIDs(node, ids);
    PolynomialApproximation approx = approximateKnnDistances(getMeanKNNList(ids, knnLists));
    entry.setKnnDistanceApproximation(approx);
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Processing partitions.", partitions, logger) : null;
    for(int part = 0; part < partitions; part++) {
      int size = (partitions * minsize + part >= aids.size()) ? minsize : minsize + 1;
      // Collect the ids in this node.
      ArrayModifiableDBIDs ids = DBIDUtil.newArray(size);
      for(int i = 0; i < size; i++) {
        assert (size * partitions + part < aids.size());
        ids.add(aids.get(i * partitions + part));
      }
      HashMap<DBIDPair, D> cache = new HashMap<DBIDPair, D>(size * size * 3 / 8);
      for(DBID id : ids) {
        KNNHeap<D> kNN = new KNNHeap<D>(k, distanceQuery.infiniteDistance());
        for(DBID id2 : ids) {
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
    super();
    this.positiveClassName = positive_class_name;
  }

  private ROCResult computeROCResult(int size, SetDBIDs positiveids, Iterator<DBID> iter) {
    ArrayModifiableDBIDs order = DBIDUtil.newArray(size);
    while(iter.hasNext()) {
      Object o = iter.next();
      if(!(o instanceof DBID)) {
        throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements");
      }
      else {
        order.add((DBID) o);
      }
    }
    if(order.size() != size) {
      throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    List<DoubleDoublePair> roccurve = ROC.materializeROC(size, positiveids, new ROC.SimpleAdapter(order.iterator()));
    double rocauc = ROC.computeAUC(roccurve);
    if(logger.isVerbose()) {
      logger.verbose(ROCAUC_LABEL +": " + rocauc);
    }
View Full Code Here
506
507
508
509
510
511
512
513
514
515
516
517
518
  public static ArrayModifiableDBIDs getObjectsByLabelMatch(Database database, Pattern name_pattern) {
    Relation<String> relation = guessLabelRepresentation(database);
    if(name_pattern == null) {
      return DBIDUtil.newArray();
    }
    ArrayModifiableDBIDs ret = DBIDUtil.newArray();
    for(DBID objid : relation.iterDBIDs()) {
      if(name_pattern.matcher(relation.get(objid)).matches()) {
        ret.add(objid);
      }
    }
    return ret;
  }
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    TreeSet<FCPair<Double, DBID>> minhotset = new TreeSet<FCPair<Double, DBID>>();
    TreeSet<FCPair<Double, DBID>> maxhotset = new TreeSet<FCPair<Double, DBID>>(Collections.reverseOrder());

    int randomsize = (int) Math.max(25, Math.pow(database.size(), 0.2));
    double rprob = ((double) randomsize) / size;
    ArrayModifiableDBIDs randomset = DBIDUtil.newArray(randomsize);

    Iterator<DBID> iter = database.iterDBIDs();
    if(!iter.hasNext()) {
      throw new IllegalStateException(ExceptionMessages.DATABASE_EMPTY);
    }
    DBID firstid = iter.next();
    minhotset.add(new FCPair<Double, DBID>(Double.MAX_VALUE, firstid));
    maxhotset.add(new FCPair<Double, DBID>(Double.MIN_VALUE, firstid));
    while(iter.hasNext()) {
      DBID id1 = iter.next();
      // generate candidates for min distance.
      ArrayList<FCPair<Double, DBID>> np = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2);
      for(FCPair<Double, DBID> pair : minhotset) {
        DBID id2 = pair.getSecond();
        // skip the object itself
        if(id1.compareTo(id2) == 0) {
          continue;
        }
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      for(DBID id2 : randomset) {
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      minhotset.addAll(np);
      shrinkHeap(minhotset, k);

      // generate candidates for max distance.
      ArrayList<FCPair<Double, DBID>> np2 = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2);
      for(FCPair<Double, DBID> pair : minhotset) {
        DBID id2 = pair.getSecond();
        // skip the object itself
        if(id1.compareTo(id2) == 0) {
          continue;
        }
        double d = distFunc.distance(id1, id2).doubleValue();
        np2.add(new FCPair<Double, DBID>(d, id1));
        np2.add(new FCPair<Double, DBID>(d, id2));
      }
      for(DBID id2 : randomset) {
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      maxhotset.addAll(np2);
      shrinkHeap(maxhotset, k);

      // update random set
      if(randomset.size() < randomsize) {
        randomset.add(id1);
      }
      else if(rnd.nextDouble() < rprob) {
        randomset.set((int) Math.floor(rnd.nextDouble() * randomsize), id1);
      }
    }
    return new DoubleMinMax(minhotset.first().getFirst(), maxhotset.first().getFirst());
  }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
   * @param relation Relation
   * @param iter DBID iterator
   * @return result object
   */
  private SimilarityMatrix computeSimilarityMatrixImage(Relation<O> relation, Iterator<DBID> iter) {
    ArrayModifiableDBIDs order = DBIDUtil.newArray(relation.size());
    while(iter.hasNext()) {
      Object o = iter.next();
      if(!(o instanceof DBID)) {
        throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements");
      }
      else {
        order.add((DBID) o);
      }
    }
    if(order.size() != relation.size()) {
      throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    DistanceQuery<O, ? extends NumberDistance<?, ?>> dq = distanceFunction.instantiate(relation);
    final int size = order.size();

    // When the logging is in the outer loop, it's just 2*size (providing enough
    // resolution)
    final int ltotal = 2 * size; // size * (size + 1);
    FiniteProgress prog = logger.isVerbose() ? new FiniteProgress("Similarity Matrix Image", ltotal, logger) : null;

    // Note: we assume that we have an efficient distance cache available,
    // since we are using 2*O(n*n) distance computations.
    DoubleMinMax minmax = new DoubleMinMax();
    for(int x = 0; x < size; x++) {
      DBID id1 = order.get(x);
      for(int y = x; y < size; y++) {
        DBID id2 = order.get(y);
        final double dist = dq.distance(id1, id2).doubleValue();
        if(!Double.isNaN(dist) && !Double.isInfinite(dist) /* && dist > 0.0 */) {
          if(!skipzero || dist != 0.0) {
            minmax.put(dist);
          }
        }
      }
      if(prog != null) {
        prog.incrementProcessed(logger);
      }
    }

    double zoom = minmax.getMax() - minmax.getMin();
    if(zoom > 0.0) {
      zoom = 1. / zoom;
    }
    LinearScaling scale = new LinearScaling(zoom, -minmax.getMin() * zoom);
    BufferedImage img = new BufferedImage(size, size, BufferedImage.TYPE_INT_RGB);
    for(int x = 0; x < size; x++) {
      DBID id1 = order.get(x);
      for(int y = x; y < size; y++) {
        DBID id2 = order.get(y);
        double ddist = dq.distance(id1, id2).doubleValue();
        if(ddist > 0.0) {
          ddist = scale.getScaled(ddist);
        }
        // Apply extra scaling
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
    TreeSet<FCPair<Double, DBID>> minhotset = new TreeSet<FCPair<Double, DBID>>();
    TreeSet<FCPair<Double, DBID>> maxhotset = new TreeSet<FCPair<Double, DBID>>(Collections.reverseOrder());

    int randomsize = (int) Math.max(25, Math.pow(database.size(), 0.2));
    double rprob = ((double) randomsize) / size;
    ArrayModifiableDBIDs randomset = DBIDUtil.newArray(randomsize);

    Iterator<DBID> iter = database.iterDBIDs();
    if(!iter.hasNext()) {
      throw new IllegalStateException(ExceptionMessages.DATABASE_EMPTY);
    }
    DBID firstid = iter.next();
    minhotset.add(new FCPair<Double, DBID>(Double.MAX_VALUE, firstid));
    maxhotset.add(new FCPair<Double, DBID>(Double.MIN_VALUE, firstid));
    while(iter.hasNext()) {
      DBID id1 = iter.next();
      // generate candidates for min distance.
      ArrayList<FCPair<Double, DBID>> np = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2);
      for(FCPair<Double, DBID> pair : minhotset) {
        DBID id2 = pair.getSecond();
        // skip the object itself
        if(id1.compareTo(id2) == 0) {
          continue;
        }
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      for(DBID id2 : randomset) {
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      minhotset.addAll(np);
      shrinkHeap(minhotset, k);

      // generate candidates for max distance.
      ArrayList<FCPair<Double, DBID>> np2 = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2);
      for(FCPair<Double, DBID> pair : minhotset) {
        DBID id2 = pair.getSecond();
        // skip the object itself
        if(id1.compareTo(id2) == 0) {
          continue;
        }
        double d = distFunc.distance(id1, id2).doubleValue();
        np2.add(new FCPair<Double, DBID>(d, id1));
        np2.add(new FCPair<Double, DBID>(d, id2));
      }
      for(DBID id2 : randomset) {
        double d = distFunc.distance(id1, id2).doubleValue();
        np.add(new FCPair<Double, DBID>(d, id1));
        np.add(new FCPair<Double, DBID>(d, id2));
      }
      maxhotset.addAll(np2);
      shrinkHeap(maxhotset, k);

      // update random set
      if(randomset.size() < randomsize) {
        randomset.add(id1);
      }
      else if(rnd.nextDouble() < rprob) {
        randomset.set((int) Math.floor(rnd.nextDouble() * randomsize), id1);
      }
    }
    return new DoubleMinMax(minhotset.first().getFirst(), maxhotset.first().getFirst());
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    super();
    this.positiveClassName = positive_class_name;
  }

  private ROCResult computeROCResult(int size, SetDBIDs positiveids, Iterator<DBID> iter) {
    ArrayModifiableDBIDs order = DBIDUtil.newArray(size);
    while(iter.hasNext()) {
      Object o = iter.next();
      if(!(o instanceof DBID)) {
        throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements");
      }
      else {
        order.add((DBID) o);
      }
    }
    if(order.size() != size) {
      throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    List<DoubleDoublePair> roccurve = ROC.materializeROC(size, positiveids, new ROC.SimpleAdapter(order.iterator()));
    double rocauc = ROC.computeAUC(roccurve);
    if(logger.isVerbose()) {
      logger.verbose(ROCAUC_LABEL +": " + rocauc);
    }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
   * @param relation Relation
   * @param iter DBID iterator
   * @return result object
   */
  private SimilarityMatrix computeSimilarityMatrixImage(Relation<O> relation, Iterator<DBID> iter) {
    ArrayModifiableDBIDs order = DBIDUtil.newArray(relation.size());
    while(iter.hasNext()) {
      Object o = iter.next();
      if(!(o instanceof DBID)) {
        throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements");
      }
      else {
        order.add((DBID) o);
      }
    }
    if(order.size() != relation.size()) {
      throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    DistanceQuery<O, ? extends NumberDistance<?, ?>> dq = distanceFunction.instantiate(relation);
    final int size = order.size();

    // When the logging is in the outer loop, it's just 2*size (providing enough
    // resolution)
    final int ltotal = 2 * size; // size * (size + 1);
    FiniteProgress prog = logger.isVerbose() ? new FiniteProgress("Similarity Matrix Image", ltotal, logger) : null;

    // Note: we assume that we have an efficient distance cache available,
    // since we are using 2*O(n*n) distance computations.
    DoubleMinMax minmax = new DoubleMinMax();
    for(int x = 0; x < size; x++) {
      DBID id1 = order.get(x);
      for(int y = x; y < size; y++) {
        DBID id2 = order.get(y);
        final double dist = dq.distance(id1, id2).doubleValue();
        if(!Double.isNaN(dist) && !Double.isInfinite(dist) /* && dist > 0.0 */) {
          if(!skipzero || dist != 0.0) {
            minmax.put(dist);
          }
        }
      }
      if(prog != null) {
        prog.incrementProcessed(logger);
      }
    }

    double zoom = minmax.getMax() - minmax.getMin();
    if(zoom > 0.0) {
      zoom = 1. / zoom;
    }
    LinearScaling scale = new LinearScaling(zoom, -minmax.getMin() * zoom);
    BufferedImage img = new BufferedImage(size, size, BufferedImage.TYPE_INT_RGB);
    for(int x = 0; x < size; x++) {
      DBID id1 = order.get(x);
      for(int y = x; y < size; y++) {
        DBID id2 = order.get(y);
        double ddist = dq.distance(id1, id2).doubleValue();
        if(ddist > 0.0) {
          ddist = scale.getScaled(ddist);
        }
        // Apply extra scaling
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    storage = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP, ArrayDBIDs.class);
    KNNQuery<O, D> knnquery = QueryUtil.getKNNQuery(relation, distanceFunction, numberOfNeighbors);

    FiniteProgress progress = getLogger().isVerbose() ? new FiniteProgress("assigning nearest neighbor lists", relation.size(), getLogger()) : null;
    for(DBID id : relation.iterDBIDs()) {
      ArrayModifiableDBIDs neighbors = DBIDUtil.newArray(numberOfNeighbors);
      KNNResult<D> kNN = knnquery.getKNNForDBID(id, numberOfNeighbors);
      for(int i = 0; i < kNN.size(); i++) {
        final DBID nid = kNN.get(i).getDBID();
        // if(!id.equals(nid)) {
        neighbors.add(nid);
        // }
        // Size limitation to exactly numberOfNeighbors
        if(neighbors.size() >= numberOfNeighbors) {
          break;
        }
      }
      neighbors.sort();
      storage.put(id, neighbors);
      if(progress != null) {
        progress.incrementProcessed(getLogger());
      }
    }
View Full Code Here
TOP

Related Classes of de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs

  • de.lmu.ifi.dbs.elki.algorithm.clustering.SLINK
  • de.lmu.ifi.dbs.elki.algorithm.clustering.SNNClustering
  • de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.PROCLUS
  • de.lmu.ifi.dbs.elki.algorithm.outlier.AbstractAggarwalYuOutlier
  • de.lmu.ifi.dbs.elki.algorithm.outlier.OnlineLOF$LOFKNNListener
  • de.lmu.ifi.dbs.elki.algorithm.outlier.SOD
  • de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuGLSBackwardSearchAlgorithm
  • de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.neighborhood.ExternalNeighborhood$Factory
  • de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.neighborhood.PrecomputedKNearestNeighborNeighborhood$Factory
  • de.lmu.ifi.dbs.elki.algorithm.statistics.DistanceStatisticsWithClasses
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.