Package edu.umd.cloud9.util.fd

Examples of edu.umd.cloud9.util.fd.Int2IntFrequencyDistribution


    testGetSortedEventsCommon(fd);
  }

  @Test
  public void testGetSortedEventsFastutil() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionFastutil();
    testGetSortedEventsCommon(fd);
  }
View Full Code Here


    assertEquals(2, list.get(3).getRightElement());
  }

  @Test
  public void testIterableEntry() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
    testIterableCommon(fd);
  }
View Full Code Here

    testIterableCommon(fd);
  }

  @Test
  public void testIterableFastutil() {
    Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionFastutil();
    testIterableCommon(fd);
  }
View Full Code Here

  private long sumOfAllFrequencies = 0;

  @Override
  public void set(int k, int cond, int v) {
    if (!distributions.containsKey(cond)) {
      Int2IntFrequencyDistribution fd = new Int2IntFrequencyDistributionEntry();
      fd.set(k, v);
      distributions.put(cond, fd);
      marginals.increment(k, v);

      sumOfAllFrequencies += v;
    } else {
      Int2IntFrequencyDistribution fd = distributions.get(cond);
      int rv = fd.get(k);

      fd.set(k, v);
      distributions.put(cond, fd);
      marginals.increment(k, -rv + v);

      sumOfAllFrequencies = sumOfAllFrequencies - rv + v;
    }
View Full Code Here

    return sumOfAllFrequencies;
  }

  @Override
  public void check() {
    Int2IntFrequencyDistribution m = new Int2IntFrequencyDistributionEntry();

    long totalSum = 0;
    for (Int2IntFrequencyDistribution fd : distributions.values()) {
      long conditionalSum = 0;

      for (PairOfInts pair : fd) {
        conditionalSum += pair.getRightElement();
        m.increment(pair.getLeftElement(), pair.getRightElement());
      }

      if (conditionalSum != fd.getSumOfCounts()) {
        throw new RuntimeException("Internal Error!");
      }
      totalSum += fd.getSumOfCounts();
    }

    if (totalSum != getSumOfAllCounts()) {
      throw new RuntimeException("Internal Error! Got " + totalSum + ", Expected "  + getSumOfAllCounts());
    }

    for (PairOfInts e : m) {
      if ( e.getRightElement() != marginals.get(e.getLeftElement()) ) {
        throw new RuntimeException("Internal Error!");
      }
    }

    for (PairOfInts e : m) {
      if ( e.getRightElement() != m.get(e.getLeftElement()) ) {
        throw new RuntimeException("Internal Error!");
      }
    }
  }
View Full Code Here


  public static void addToTable(int curIndex, TreeSet<PairOfFloatString> topTrans, float cumProb, TTable_monolithic_IFAs table,
      Vocab trgVocab, float cumProbThreshold, HookaStats stats) {
    List<Integer> sortedIndices = new ArrayList<Integer>();
    HMapIF index2ProbMap = new HMapIF();

    float sumOfProbs = 0.0f;    //only extract the top K<15 if the mass prob. exceeds MAX_probThreshold
    while(!topTrans.isEmpty() && sumOfProbs < cumProbThreshold){
      PairOfFloatString e = topTrans.pollLast();
      String term = e.getRightElement();
      float pr = e.getLeftElement()/cumProb;    // normalize
      logger.debug(term+"-->"+pr);
      int trgIndex = trgVocab.addOrGet(term);
      sumOfProbs += e.getLeftElement();         // keep track of unnormalized cumulative prob for determining cutoff
      sortedIndices.add(trgIndex);
      index2ProbMap.put(trgIndex, pr);
    }

    // to enable faster access with binary search, we sort entries by vocabulary index.
    Collections.sort(sortedIndices);
    int numEntries = sortedIndices.size();

    // for statistics only
    stats.update(numEntries, sumOfProbs);

    // write translation list to TTable object
    int[] indices = new int[numEntries];
    float[] probs = new float[numEntries];
    int i=0;
    for(int sortedIndex : sortedIndices){
      indices[i]=sortedIndex;
      probs[i]=index2ProbMap.get(sortedIndex);
      i++;
    }     
    table.set(curIndex, new IndexedFloatArray(indices, probs, true));
  }
View Full Code Here

    DocumentVectorSlidingWindow generator = new DocumentVectorSlidingWindow(env, fs);

    //Parse queries, judgemnts and features
    HMapIV<String> parsedQueries = QueryUtility.loadQueries(queryPath);
    HMapIV<int[]> queries = QueryUtility.queryToIntegerCode(env, parsedQueries);
    HMapIF idfs = QueryUtility.loadIdf(env, parsedQueries);
    HMapIF cfs = QueryUtility.loadCf(env, parsedQueries);
    HMapIV<int[]> qrels = QrelUtility.parseQrelsFromTabDelimited(qrelPath);
    Map<String, Feature> featuresMap = FeatureUtility.parseFeatures(featurePath);
    Feature[] features = new Feature[featuresMap.size()];
    int index = 0;
    for(String key: featuresMap.keySet()) {
View Full Code Here

    DocumentVectorOnTheFlyIndexing generator = new DocumentVectorOnTheFlyIndexing(env, fs);

    //Parse queries, judgemnts and features
    HMapIV<String> parsedQueries = QueryUtility.loadQueries(queryPath);
    HMapIV<int[]> queries = QueryUtility.queryToIntegerCode(env, parsedQueries);
    HMapIF idfs = QueryUtility.loadIdf(env, parsedQueries);
    HMapIF cfs = QueryUtility.loadCf(env, parsedQueries);
    HMapIV<int[]> qrels = QrelUtility.parseQrelsFromTabDelimited(qrelPath);
    Map<String, Feature> featuresMap = FeatureUtility.parseFeatures(featurePath);
    Feature[] features = new Feature[featuresMap.size()];
    int index = 0;
    for(String key: featuresMap.keySet()) {
View Full Code Here

    RankAndFeaturesSmallAdaptive generator = new RankAndFeaturesSmallAdaptive(env, fs);

    //Parse queries and find integer codes for the query terms.
    HMapIV<String> parsedQueries = QueryUtility.loadQueries(queryPath);
    HMapIV<int[]> queries = QueryUtility.queryToIntegerCode(env, parsedQueries);
    HMapIF idfs = QueryUtility.loadIdf(env, parsedQueries);
    HMapIF cfs = QueryUtility.loadCf(env, parsedQueries);
    HMapIV<int[]> qrels = QrelUtility.parseQrelsFromTabDelimited(qrelPath);
    Map<String, Feature> featuresMap = FeatureUtility.parseFeatures(featurePath);
    Feature[] features = new Feature[featuresMap.size()];
    int index = 0;
    for(String key: featuresMap.keySet()) {
View Full Code Here

                            (float) env.getDefaultDf(), (float) env.getDefaultCf());
  }

  private void preparePostings(String postingsPath) throws Exception {
    postings = new HMapIV<CompressedPositionalPostings>();
    dfs = new HMapII();
    docLengths = new HMapII();

    FSDataInputStream input = fs.open(new Path(postingsPath));
    int termid = input.readInt();
    while(termid != -1) {
      dfs.put(termid, input.readInt());
View Full Code Here

TOP

Related Classes of edu.umd.cloud9.util.fd.Int2IntFrequencyDistribution

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.