Package edu.umd.cloud9.util.fd

Examples of edu.umd.cloud9.util.fd.Int2IntFrequencyDistributionEntry

399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
      String[] alPair = alignment.split("-");
      int f = Integer.parseInt(alPair[0]);
      int e = Integer.parseInt(alPair[1]);

      if(!one2manyAlign.containsKey(f)){
        one2manyAlign.put(f, new ArrayListOfInts()); 
      }
      one2manyAlign.get(f).add(e);
    }

    // for each source token id, sort ids of its translations in ascending order
    for(Integer f : one2manyAlign.keySet()) {
      ArrayListOfInts lst = one2manyAlign.get(f);
      lst.sort();
      one2manyAlign.put(f, lst);
    }

    return one2manyAlign;
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
      // Remember, token position is numbered started from one...
      if (positions.containsKey(term)) {
        positions.get(term).add(i + 1);
      } else {
        ArrayListOfInts l = new ArrayListOfInts();
        l.add(i + 1);
        positions.put(term, l);
      }
    }

    int doclength = 0;
    Iterator<Map.Entry<String, ArrayListOfInts>> it = positions.entrySet().iterator();
    Map.Entry<String, ArrayListOfInts> e;
    ArrayListOfInts positionsList;
    while (it.hasNext()) {
      e = it.next();
      positionsList = e.getValue();

      // We're storing tfs as shorts, so check for overflow...
      if (positionsList.size() >= TF_CUT) {
        // There are a few ways to handle this... If we're getting such a high tf, then it most
        // likely means that this is a junk doc.
        LOG.warn("Error: tf of " + e.getValue()
            + " will overflow max short value. docno=" + doc.getDocid() + ", term="
            + e.getKey());
        it.remove();
      } else {
        positionsList.trimToSize();
        doclength += positionsList.size();
      }
    }

    if ( positions.size() == 0 ) {
      return positions;
    }

    positions.put("", new ArrayListOfInts(new int[] { doclength }));
    return positions;
  }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
    key.set("gold");
    reader.get(key, value);
    System.out.println("Complete postings list for 'gold': " + value);

    Int2IntFrequencyDistribution goldHist = new Int2IntFrequencyDistributionEntry();
    postings = value.getRightElement();
    for (PairOfInts pair : postings) {
      goldHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for gold");
    for (PairOfInts pair : goldHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    key.set("silver");
    reader.get(key, value);
    System.out.println("Complete postings list for 'silver': " + value);

    Int2IntFrequencyDistribution silverHist = new Int2IntFrequencyDistributionEntry();
    postings = value.getRightElement();
    for (PairOfInts pair : postings) {
      silverHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for silver");
    for (PairOfInts pair : silverHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
View Full Code Here
384
385
386
387
388
389
390
391
    assertEquals(5, list.get(3).getRightElement());
  }

  @Test
  public void testGetSortedEventsEntry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    testGetSortedEventsCommon(fd);
  }
View Full Code Here
468
469
470
471
472
473
474
475
    assertEquals(2, list.get(3).getRightElement());
  }

  @Test
  public void testIterableEntry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    testIterableCommon(fd);
  }
View Full Code Here
34
35
36
37
38
39
40
41
public class Int2IntFrequencyDistributionTest {

  @Test
  public void test1Entry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    test1Common(fd);
  }
View Full Code Here
104
105
106
107
108
109
110
111
    assertEquals(Math.log((float) 2 / 4), fd.computeLogRelativeFrequency(3), 10e-6);
  }

  @Test
  public void test2Entry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    test2Common(fd);
  }
View Full Code Here
172
173
174
175
176
177
178
179
    assertEquals(5, fd.get(4));
  }

  @Test
  public void test3Entry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    test3Common(fd);
  }
View Full Code Here
202
203
204
205
206
207
208
209
    assertEquals(0, fd.getSumOfCounts());
  }

  @Test(expected = RuntimeException.class)
  public void testFailedDecrement1Entry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    testFailedDecrement1Common(fd);
  }
View Full Code Here
230
231
232
233
234
235
236
237
    fd.decrement(1);
  }

  @Test(expected = RuntimeException.class)
  public void testFailedDecrement2Entry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    testFailedDecrement2Common(fd);
  }
View Full Code Here
TOP

Related Classes of edu.umd.cloud9.util.fd.Int2IntFrequencyDistributionEntry

  • bak.pcj.IntIterator
  • edu.umd.cloud9.example.bfs.BfsNodeTest
  • edu.umd.cloud9.example.pagerank.PageRankNodeTest
  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.map.HMapIDW
  • edu.umd.cloud9.io.map.HMapIDWTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.