Package ch.akuhn.hapax.index

Examples of ch.akuhn.hapax.index.TermDocumentMatrix

29
30
31
32
33
34
35
    public CorpusBuilder(TermDocumentMatrix tdm) {
        this.corpus = tdm;
    }

    public CorpusBuilder() {
        this(new TermDocumentMatrix());
    }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
        return this;
    }


    public TermDocumentMatrix makeTDM() {
        TermDocumentMatrix tdm = corpus;
        if (ignoreCase) tdm = tdm.toLowerCase();
        if (rejectRareTerms) tdm = tdm.rejectHapaxes();
        if (rejectStopwords) tdm = tdm.rejectStopwords();
        // TODO if (rejectCommonTerms) tdm = tdm.rejectCommonTerms();
        return tdm.weight(local, global);
    }
View Full Code Here
10
11
12
13
14
15
16
17
18
19
20
21
22
public class SmallDocumentsTest {

    @Test
    public void corpusWithSmallDocuments() {
        TermDocumentMatrix tdm = new TermDocumentMatrix();
        tdm.putDocument("m1", new Terms("Lorem ipsum dolor."));
        tdm.putDocument("m2", new Terms("Lorem ipsum dolor."));
        tdm.putDocument("m3", new Terms("Lorem ipsum dolor."));
        LatentSemanticIndex lsi = tdm.rejectAndWeight().createIndex();
        assertEquals(3, lsi.documentCount());
        assertEquals(3, lsi.rankDocumentsByQuery("Lorem").size());
    }
View Full Code Here
21
22
23
24
25
26
27
28
29
30
        assertEquals(3, lsi.rankDocumentsByQuery("Lorem").size());
    }
   
    @Test
    public void corpusWithoutDocuments() {
        TermDocumentMatrix tdm = new TermDocumentMatrix();
        LatentSemanticIndex lsi = tdm.rejectAndWeight().createIndex();
        assertEquals(0, lsi.documentCount());
        assertEquals(0, lsi.rankDocumentsByQuery("Lorem").size());
    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
        assertEquals(0, lsi.rankDocumentsByQuery("Lorem").size());
    }
   
    @Test
    public void corpusWithoutOneDocuments() {
        TermDocumentMatrix tdm = new TermDocumentMatrix();
        tdm.putDocument("m1", new Terms("Lorem ipsum dolor."));
        LatentSemanticIndex lsi = tdm.rejectAndWeight().createIndex();
        assertEquals(1, lsi.documentCount());
        assertEquals(1, lsi.rankDocumentsByQuery("Lorem").size());
    }
View Full Code Here
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    public static void main(String[] args) throws FileNotFoundException {

     
     
        TermDocumentMatrix tdm = new TermDocumentMatrix();
        CorpusBuilderHelper importer = new CorpusBuilderHelper(tdm);
        importer.importAllFiles(new File("../ch.deif.meander"), ".java");
        importer.importAllFiles(new File("."), ".java");

        Out.puts(tdm);

        Out.puts(Get.head(tdm.documents()));
        Out.puts(tdm.getDocument(Get.head(tdm.documents())));
       
        Out.puts(tdm);

        Out.puts(tdm.density());

        tdm = tdm.rejectAndWeight();

        Out.puts(tdm);

        Out.puts(tdm.density());

        Out.puts(tdm.terms().sortedCounts());

        LatentSemanticIndex lsi = tdm.createIndex();

        Out.puts(first(10, lsi.rankDocumentsByTerm("bag")));
        Out.puts(first(10, lsi.rankTermsByTerm("bag")));
        Out.puts(first(10, lsi.rankDocumentsByTerm("codemap")));
        Out.puts(first(10, lsi.rankDocumentsByQuery("split string by lower- and upper-case")));
View Full Code Here
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
public class Examples {

  @Test
  public void hapaxExample() {
   
    TermDocumentMatrix tdm;
   
    tdm = Hapax.newCorpus()
        .useCamelCaseScanner()
        .rejectRareTerms()
        .rejectCommonTerms()
        .useTFIDF()
        .addFiles(".", ".java")
        .makeTDM();
   
    System.out.println(tdm);
   
    tdm = Hapax.newCorpus()
        .useLetterScanner()
        .ignoreCase()
        .rejectStopwords()
        .rejectRareTerms()
        .dontUseWeighting()
        .addDocument("c1", "Human machine interface for Lab ABC computer applications")
        .addDocument("c2", "A survey of user opinion of computer system response time")
        .addDocument("c3", "The EPS user interface management system")
        .addDocument("c4", "System and human system engineering testing of EPS")
        .addDocument("c5", "Relation of user-perceived response time to error measurement")
        .addDocument("m1", "The generation of random, binary, unordered trees")
        .addDocument("m2", "The intersection graph of paths in trees")
        .addDocument("m3", "Graph minors IV: Widths of trees and well-quasi-ordering")
        .addDocument("m4", "Graph minors: A survey")
        .makeTDM();
       
    System.out.println(tdm);
    
    LatentSemanticIndex lsi = tdm.createIndex(2);
   
    Ranking<String> ranking = lsi.rankDocumentsByQuery("human computer interaction");
   
    System.out.println(ranking.top(10));
   
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
        { "m4", "Graph minors: A survey" }};
   
   
    @Test
    public TermDocumentMatrix makeTermDocumentMatrix() {
        TermDocumentMatrix tdm = new TermDocumentMatrix();
        for (String[] each: DATA) {
            tdm.putDocument(each[0], new Terms(each[1]));
        }
        assertEquals(9, tdm.documentCount());
        assertEquals(45, tdm.termCount());
        return tdm;
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    }
   
    @Test
    @Given("#makeTermDocumentMatrix")
    public TermDocumentMatrix rejectStopWords(final TermDocumentMatrix matrix) {
        TermDocumentMatrix tdm = matrix;
        tdm = tdm.toLowerCase();
        assertEquals(9, tdm.documentCount());
        assertEquals(42, tdm.termCount());
        tdm = tdm.rejectHapaxes();
        assertEquals(9, tdm.documentCount());
        assertEquals(16, tdm.termCount());
        tdm = tdm.toLowerCase().rejectStopwords();
        assertEquals(9, tdm.documentCount());
        assertEquals(12, tdm.termCount());
        assertEquals(SORTED, Get.sorted(tdm.terms().elementSet()).toString());
        return tdm;
    }
View Full Code Here
TOP

Related Classes of ch.akuhn.hapax.index.TermDocumentMatrix

  • ch.akuhn.hapax.corpus.PorterStemmer
  • ch.akuhn.hapax.corpus.Terms
  • ch.akuhn.hapax.CorpusBuilder
  • ch.akuhn.hapax.linalg.SparseMatrix
  • ch.akuhn.hapax.linalg.SVD
  • ch.akuhn.util.IntArray
  • example.CurrentDirJava
  • example.Examples
  • hapax.test.DeerExample
  • hapax.test.SmallDocumentsTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.