Package com.wcohen.ss.api

Examples of com.wcohen.ss.api.Token

44
45
46
47
48
49
50
51
52
53
54
    private int nextId = 0;
    private Map<String, Token> tokMap = new TreeMap<String, Token>();
   
    public Token intern(String s) {
        s = s.toLowerCase().intern();
        Token tok = tokMap.get(s);
        if (tok == null) {
            tok = new BasicToken(++nextId, s);
            tokMap.put(s, tok);
        }
        return tok;
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
        UnitVector tBag = asUnitVector(t);
        List<Similarity> similarities = new ArrayList<Similarity>(sBag.size());
        double sim = 0.0;
        int i = 0;
        for (Iterator<Token> ti = sBag.tokenIterator(); ti.hasNext(); i++) {
            Token tok = ti.next();
            int j = 0;
            for (Iterator<Token> tj = tBag.tokenIterator(); tj.hasNext(); j++) {
                Token tokJ = tj.next();
                double distItoJ = tokenDistance.score(tok.getValue(), tokJ.getValue());
                if (distItoJ >= tokenMatchThreshold) {
                    similarities.add(new Similarity(i, j, distItoJ * sBag.getWeight(tok) * tBag.getWeight(tokJ)));
                }
            }

View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
        BagOfTokens tBag = (BagOfTokens) t;
        StringBuilder buf = new StringBuilder("");
        PrintfFormat fmt = new PrintfFormat("%.3f");
        buf.append("Common tokens: ");
        for (Iterator<Token> i = sBag.tokenIterator(); i.hasNext();) {
            Token tok = i.next();
            if (tBag.contains(tok)) {
                buf.append(" " + tok.getValue() + ": ");
                buf.append(fmt.sprintf(sBag.getWeight(tok)));
                buf.append("*");
                buf.append(fmt.sprintf(tBag.getWeight(tok)));
            } else {
                // find best matching token
                double matchScore = tokenMatchThreshold;
                Token matchTok = null;
                for (Iterator<Token> j = tBag.tokenIterator(); j.hasNext();) {
                    Token tokJ = j.next();
                    double distItoJ = tokenDistance.score(tok.getValue(), tokJ.getValue());
                    if (distItoJ >= matchScore) {
                        matchTok = tokJ;
                        matchScore = distItoJ;
                    }
                }
View Full Code Here
TOP

Related Classes of com.wcohen.ss.api.Token

  • com.wcohen.ss.SoftTFIDF
  • edu.mit.simile.vicino.NGramTokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.