Package uk.ac.cam.ha293.tweetlabel.util

Examples of uk.ac.cam.ha293.tweetlabel.util.Tools

513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
    Map<String,Integer> urlCategoryCounts = new HashMap<String,Integer>();
   
    for(SimpleTweet tweet : tweets) {
      //System.err.println(tweet.getText());
     
      TextwiseClassification textClassification = TextwiseClassifier.classify(tweet.getText(), false);
     
      textClassification.print();
     
      for(String category : textClassification.getCategories()) {
        if(textCategoryScores.containsKey(category)) {
          textCategoryScores.put(category, textCategoryScores.get(category) + textClassification.lookupScore(category));
          textCategoryCounts.put(category, textCategoryCounts.get(category) + 1);
        } else {
          textCategoryScores.put(category, textClassification.lookupScore(category));
          textCategoryCounts.put(category, 1);
        }
      }
     
      for(String url : tweet.getUrls()) {
       
        //System.err.println(url);
       
        TextwiseClassification urlClassification = TextwiseClassifier.classify(url, true);
       
        urlClassification.print();
       
        for(String category : urlClassification.getCategories()) {
          if(urlCategoryScores.containsKey(category)) {
            urlCategoryScores.put(category, urlCategoryScores.get(category) + urlClassification.lookupScore(category));
            urlCategoryCounts.put(category, urlCategoryCounts.get(category) + 1);
          } else {
            urlCategoryScores.put(category, urlClassification.lookupScore(category));
            urlCategoryCounts.put(category, 1);
          }
        }
      }
     
View Full Code Here
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
    }
   
    Map<String,Double> classifications = new HashMap<String,Double>();
    Map<String,Integer> classificationsCount = new HashMap<String,Integer>();
    for(int i=0; i<concat.length()-500; i+=500) {
      TextwiseClassification textClassification;
      if(i+500>=concat.length()) textClassification = TextwiseClassifier.classify(concat.substring(i,concat.length()), false);
      else textClassification = TextwiseClassifier.classify(concat.substring(i,i+500), false); 
      Map<String,Double> scores = textClassification.getCategoryScores();
      for(String cat : textClassification.getCategories()) {
        if(classifications.containsKey(cat)) {
          classifications.put(cat,classifications.get(cat)+scores.get(cat));
          classificationsCount.put(cat,classificationsCount.get(cat)+1);
        } else {
          classifications.put(cat,scores.get(cat));
View Full Code Here
713
714
715
716
717
718
719
720
721
722
723
724
725
  public static void similarityStuff() {
    String[] topicTypes = {"alchemy","calais","textwiseproper"};
    double[] alphas = {0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0};
   
    for(double alpha : alphas) {
      SimilarityMatrix lda = SimilarityMatrix.load("lda-50-1000-100-"+alpha);
      for(String topicType : topicTypes) {
        SimilarityMatrix llda = SimilarityMatrix.load("llda-"+topicType+"-"+alpha);
        System.out.println(topicType+"\t"+alpha+"\t"+SpearmanRank.srcc(lda,llda));
      }
    }

   
View Full Code Here
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
  public static void srccKFinder(String topicType, double alpha) {
    System.out.println("Finding optimal SRCC K for "+topicType+" "+alpha);
    String[] topics = Tools.getTopics(topicType);
    int maxK = topics.length;
    for(int k=1; k<=maxK; k++) {
      SimilarityMatrix baseline = new SimilarityMatrix(2506);
      baseline.fillRestricted(true, topicType, k, 0);
      SimilarityMatrix llda = new SimilarityMatrix(2506);
      llda.fillRestricted(false, topicType, k, alpha);
      System.out.println(alpha+","+topicType+","+k+","+SpearmanRank.jscSRCC(baseline, llda));
    }
  }
View Full Code Here
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
        //if(fac.getScore(topic) < scoreThreshold) break; //stop getting low-prob topics
        topics.add(topic);
        count++;
      }
    }else if(topicType.equals("liwc")) {
      FullLIWCClassification flc = new FullLIWCClassification(false,userID);
      int topTopics = 3;
      int count = 0;
      for(String topic : flc.getCategorySet()) {
        if(count == topTopics) break;
        topics.add(topic);
        count++;
      }
    } else if(topicType.equals("liwcnb")) {
      FullLIWCClassification flc = new FullLIWCClassification(true,userID);
      int topTopics = 3;
      double threshold = 0.2;
      int count = 0;
      for(String topic : flc.getCategorySet()) {
        if(count == topTopics) break;
        if(flc.getScore(topic) < threshold) break;
        topics.add(topic);
        count++;
      }
    } else {
      System.out.println("invalid topic type");
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
 
  public void fillLIWC(boolean naiveBayes) {
    System.out.println("Filling from LIWC classifications, NB="+naiveBayes);
    FullLIWCClassification[] classifications = new FullLIWCClassification[d];
    for(long id : Tools.getCSVUserIDs()) {
      classifications[indexLookup.get(id)] = new FullLIWCClassification(naiveBayes,id);
    }
   
    //cosine similarities!
    for(int m=0; m<d; m++) {
      System.out.println("On row "+m);
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
  public void fillLDAAndSave(int numTopics, int burn, int sample, double alpha) {
    //get clasifications
    System.out.println("Filling from LDA classifications");
    FullLDAClassification[] classifications = new FullLDAClassification[d];
    for(long id : Tools.getCSVUserIDs()) {
      classifications[indexLookup.get(id)] = new FullLDAClassification(id,numTopics,burn,sample,alpha);
    }
   
    //cosine similarities!
    for(int m=0; m<d; m++) {
      System.out.println("On row "+m);
View Full Code Here
351
352
353
354
355
356
357
358
359
360
361
  public void fillLDAJS(int numTopics, int burn, int sample, double alpha) {
    //get clasifications
    System.out.println("Filling from LDA classifications");
    FullLDAClassification[] classifications = new FullLDAClassification[d];
    for(long id : Tools.getCSVUserIDs()) {
      classifications[indexLookup.get(id)] = new FullLDAClassification(id,numTopics,burn,sample,alpha);
    }
   
    //cosine similarities!
    for(int m=0; m<d; m++) {
      System.out.println("On row "+m);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  }
 
  public static Set<Double> diversitySet(String topicType, double alpha, long uid) {
    Set<Double> valueSet = new HashSet<Double>();
    if(topicType.equals("lda")) {
      FullLDAClassification c = new FullLDAClassification(uid,1000,100,0,alpha);
      for(String cat : c.getCategorySet()) {
        valueSet.add(c.getScore(cat));
      }
    } else if(topicType.equals("alchemy")) {
      FullLLDAClassification c = new FullLLDAClassification("alchemy",alpha,uid);
      for(String cat : c.getCategorySet()) {
        valueSet.add(c.getScore(cat));
      }
    } else if(topicType.equals("calais")) {
      FullLLDAClassification c = new FullLLDAClassification("calais",alpha,uid);
      for(String cat : c.getCategorySet()) {
        valueSet.add(c.getScore(cat));
      }
    } else if(topicType.equals("textwise")) {
      FullLLDAClassification c = new FullLLDAClassification("textwise",alpha,uid);
      for(String cat : c.getCategorySet()) {
        valueSet.add(c.getScore(cat));
      }
    }
    return valueSet;
  }
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
 
  public void fillLLDA(String topicType, double alpha) {
    System.out.println("Filling from LLDA-inferred "+topicType+" classifications");
    FullLLDAClassification[] classifications = new FullLLDAClassification[d];
    for(long id : Tools.getCSVUserIDs()) {
      classifications[indexLookup.get(id)] = new FullLLDAClassification(topicType,alpha,id);
    }
   
    //cosine similarities!
    for(int m=0; m<d; m++) {
      System.out.println("On row "+m);
View Full Code Here
TOP

Related Classes of uk.ac.cam.ha293.tweetlabel.util.Tools

  • cc.mallet.pipe.CharSequence2TokenSequence
  • cc.mallet.pipe.Pipe
  • cc.mallet.pipe.PrintInputAndTarget
  • cc.mallet.pipe.SerialPipes
  • cc.mallet.pipe.TokenSequence2FeatureSequence
  • cc.mallet.pipe.TokenSequenceLowercase
  • cc.mallet.pipe.TokenSequenceRemoveStopwords
  • cc.mallet.topics.ParallelTopicModel
  • cc.mallet.topics.SimpleLDA
  • cc.mallet.types.Instance
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.