Package edu.umd.cloud9.io.pair

Examples of edu.umd.cloud9.io.pair.PairOfLongString$Comparator

743
744
745
746
747
748
749
750
751
752
753
                String term = m2.group(1);
                if ( !term.equals("NULL") ) {
                  float prob = Float.parseFloat(m2.group(2));
                  int engIndex = trgVocab.addOrGet(term);
                  logger.debug("Added: "+term+" with index: "+engIndex+" and prob:"+prob);
                  indexProbPairs.add(new PairOfIntFloat(engIndex, prob));
                  sumOfProbs += prob;
                }
              }
            }
            // if number of translations not set, we never cut-off, so all cases are long tails
View Full Code Here
789
790
791
792
793
794
795
796
797
798
799
                String term = m2.group(1);
                if (!term.equals("NULL")) {
                  float prob = Float.parseFloat(m2.group(2));
                  int engIndex = trgVocab.addOrGet(term);
                  logger.debug("Added: "+term+" with index: "+engIndex+" and prob:"+prob);
                  indexProbPairs.add(new PairOfIntFloat(engIndex, prob));
                  sumOfProbs+=prob;
                }
              }
            }
            if(sumOfProbs > probThreshold){
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
138
    @Override
    public void reduce(Text key, Iterable<PairOfIntLong> values, Context context)
        throws IOException, InterruptedException {
      String term = key.toString();
      Iterator<PairOfIntLong> iter = values.iterator();
      PairOfIntLong p = iter.next();
      int df = p.getLeftElement();
      long cf = p.getRightElement();
      WritableUtils.writeVInt(dfByTermOut, df);
      WritableUtils.writeVLong(cfByTermOut, cf);
      if (iter.hasNext()) {
        throw new RuntimeException("More than one record for term: " + term);
      }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
      // map from the id back to text
      // sLogger.info("termid: " + key);
      String term = mTermIdMap.getTerm(key.get());
      // sLogger.info("term: " + term);
      PairOfIntLong pair = gs.getStats(term);

      if (pair == null) {
        p.setCf(-1);
        p.setDf(-1);
      } else {
        p.setCf(pair.getRightElement());
        p.setDf(pair.getLeftElement());
      }

      output.collect(key, p);
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
public class PairOfLongStringTest {

  @Test
  public void testBasic() throws IOException {
    PairOfLongString pair = new PairOfLongString(1L, "hi");

    assertEquals("hi", pair.getRightElement());
    assertEquals(1L, pair.getLeftElement());
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    assertEquals(1L, pair.getLeftElement());
  }

  @Test
  public void testSerialize() throws IOException {
    PairOfLongString origPair = new PairOfLongString(2L, "hi");

    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    DataOutputStream dataOut = new DataOutputStream(bytesOut);

    origPair.write(dataOut);

    PairOfLongString pair = new PairOfLongString();

    pair.readFields(new DataInputStream(new ByteArrayInputStream(bytesOut.toByteArray())));

    assertEquals("hi", pair.getRightElement());
    assertTrue(pair.getLeftElement() == 2L);
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    assertTrue(pair.getLeftElement() == 2L);
  }

  @Test
  public void testComparison1() throws IOException {
    PairOfLongString pair1 = new PairOfLongString(1L, "hi");
    PairOfLongString pair2 = new PairOfLongString(1L, "hi");
    PairOfLongString pair3 = new PairOfLongString(0L, "hi");
    PairOfLongString pair4 = new PairOfLongString(0L, "a");
    PairOfLongString pair5 = new PairOfLongString(2L, "hi");
   
    assertTrue(pair1.equals(pair2));
    assertFalse(pair1.equals(pair3));

    assertTrue(pair1.compareTo(pair2) == 0);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  @Test
  public void testComparison2() throws IOException {
    WritableComparator comparator = new PairOfLongString.Comparator();

    PairOfLongString pair1 = new PairOfLongString(1L, "hi");
    PairOfLongString pair2 = new PairOfLongString(1L, "hi");
    PairOfLongString pair3 = new PairOfLongString(0L, "hi");
    PairOfLongString pair4 = new PairOfLongString(0L, "a");
    PairOfLongString pair5 = new PairOfLongString(2L, "hi");
   
    assertTrue(pair1.equals(pair2));
    assertFalse(pair1.equals(pair3));

    assertTrue(WritableComparatorTestHarness.compare(comparator, pair1, pair2) == 0);
View Full Code Here
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
      float sumProb2 = 0;
      for (Entry<String> entry : probDist.entrySet()) {
        float pr = entry.getValue() / sumProb;
        if (pr > lexProbThreshold) {
          sumProb2 += pr;
          sortedFilteredProbDist.add(new PairOfStringFloat(entry.getKey(), pr));
        }
      }

      // re-normalize values after removal of low-prob entries
      float cumProb = 0;
      int cnt = 0;
      while (cnt < maxNumTrans && cumProb < cumProbThreshold && !sortedFilteredProbDist.isEmpty()) {
        PairOfStringFloat entry = sortedFilteredProbDist.pollLast();
        float pr = entry.getValue() / sumProb2;
        cumProb += pr;
        normProbDist.put(entry.getKey(), pr);
        cnt++;
      }

      probMap.put(sourceTerm, normProbDist);
    }
View Full Code Here
671
672
673
674
675
676
677
678
679
680
681
        String[] parts = rule.split("\\|\\|\\|");
        String[] lhs = parts[0].trim().split(" ");
        String[] rhs = parts[1].trim().split(" ");;
        for (String l : lhs) {
          for (String r : rhs) {
            pairsInSCFG.add(new PairOfStrings(l, r));
          }
        }
      }
    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();
View Full Code Here
TOP

Related Classes of edu.umd.cloud9.io.pair.PairOfLongString$Comparator

  • bak.pcj.IntIterator
  • edu.umd.cloud9.example.bfs.BfsNodeTest
  • edu.umd.cloud9.example.pagerank.PageRankNodeTest
  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.map.HMapIDW
  • edu.umd.cloud9.io.map.HMapIDWTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.