Package edu.umd.cloud9.io.pair

Examples of edu.umd.cloud9.io.pair.PairOfIntString

126
127
128
129
130
131
132
133
134
135
136
137
138
    @Override
    public void reduce(Text key, Iterable<PairOfIntLong> values, Context context)
        throws IOException, InterruptedException {
      String term = key.toString();
      Iterator<PairOfIntLong> iter = values.iterator();
      PairOfIntLong p = iter.next();
      int df = p.getLeftElement();
      long cf = p.getRightElement();
      WritableUtils.writeVInt(dfByTermOut, df);
      WritableUtils.writeVLong(cfByTermOut, cf);
      if (iter.hasNext()) {
        throw new RuntimeException("More than one record for term: " + term);
      }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
      // map from the id back to text
      // sLogger.info("termid: " + key);
      String term = mTermIdMap.getTerm(key.get());
      // sLogger.info("term: " + term);
      PairOfIntLong pair = gs.getStats(term);

      if (pair == null) {
        p.setCf(-1);
        p.setDf(-1);
      } else {
        p.setCf(pair.getRightElement());
        p.setDf(pair.getLeftElement());
      }

      output.collect(key, p);
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
public class PairOfIntStringTest {

  @Test
  public void testBasic() throws IOException {
    PairOfIntString pair = new PairOfIntString(1, "2");

    assertEquals(1, pair.getLeftElement());
    assertEquals("2", pair.getRightElement());
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    assertEquals("2", pair.getRightElement());
  }

  @Test
  public void testSerialize() throws IOException {
    PairOfIntString origPair = new PairOfIntString(1, "2");

    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    DataOutputStream dataOut = new DataOutputStream(bytesOut);

    origPair.write(dataOut);

    PairOfIntString pair = new PairOfIntString();

    pair.readFields(new DataInputStream(new ByteArrayInputStream(bytesOut.toByteArray())));

    assertEquals(1, pair.getLeftElement());
    assertEquals("2", pair.getRightElement());
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    assertEquals("2", pair.getRightElement());
  }

  @Test
  public void testComparison1() throws IOException {
    PairOfIntString pair1 = new PairOfIntString(1, "2");
    PairOfIntString pair2 = new PairOfIntString(1, "2");
    PairOfIntString pair3 = new PairOfIntString(1, "1");
    PairOfIntString pair4 = new PairOfIntString(0, "9");
    PairOfIntString pair5 = new PairOfIntString(9, "0");

    assertTrue(pair1.equals(pair2));
    assertFalse(pair1.equals(pair3));

    assertTrue(pair1.compareTo(pair2) == 0);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  @Test
  public void testComparison2() throws IOException {
    WritableComparator comparator = new PairOfIntString.Comparator();

    PairOfIntString pair1 = new PairOfIntString(1, "2");
    PairOfIntString pair2 = new PairOfIntString(1, "2");
    PairOfIntString pair3 = new PairOfIntString(1, "1");
    PairOfIntString pair4 = new PairOfIntString(0, "9");
    PairOfIntString pair5 = new PairOfIntString(9, "0");

    assertTrue(WritableComparatorTestHarness.compare(comparator, pair1, pair2) == 0);
    assertTrue(WritableComparatorTestHarness.compare(comparator, pair1, pair3) > 0);
    assertTrue(WritableComparatorTestHarness.compare(comparator, pair1, pair4) > 0);
    assertTrue(WritableComparatorTestHarness.compare(comparator, pair1, pair5) < 0);
View Full Code Here
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
  public static int exportTerms(SequenceFile.Reader sequenceFileReader,
      SequenceFile.Writer sequenceFileWriter) throws IOException {
    TreeSet<PairOfIntString> treeMap = new TreeSet<PairOfIntString>(new Comparator() {
      @Override
      public int compare(Object obj1, Object obj2) {
        PairOfIntString entry1 = (PairOfIntString) obj1;
        PairOfIntString entry2 = (PairOfIntString) obj2;
        if (entry1.getLeftElement() > entry2.getLeftElement()) {
          return -1;
        } else if (entry1.getLeftElement() < entry2.getLeftElement()) {
          return entry1.getRightElement().compareTo(entry2.getRightElement());
        } else {
          return 0;
        }
      }
    });

    Text text = new Text();
    PairOfInts pairOfInts = new PairOfInts();
    while (sequenceFileReader.next(text, pairOfInts)) {
      treeMap.add(new PairOfIntString(pairOfInts.getLeftElement(), text.toString()));
    }

    int index = 0;
    IntWritable intWritable = new IntWritable();
    Iterator<PairOfIntString> itr = treeMap.iterator();
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    public void configure(JobConf job) {
      sLogger.setLevel(Level.INFO);
      srcLang = job.get("fLang");     
      mJob = job;
      pwsimMapping = new HMapIV<ArrayListOfIntsWritable>();
      valOut = new PairOfIntString();
      keyOut = new PairOfInts();

      // read doc ids of sample into vectors
      String samplesFile = job.get("Ivory.SampleFile");
      if (samplesFile != null) {
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
      fTitle.clear();
      sLogger.info(docnoPair);

      int cnt = 0;
      while (titles.hasNext()) {
        PairOfIntString title = titles.next();
        sLogger.info(title);
        if (title.getLeftElement() == CLIRUtils.E) {
          eTitle.set(title.getRightElement());
          cnt++;
        } else if (title.getLeftElement() == CLIRUtils.F) {
          fTitle.set(title.getRightElement());
          cnt++;
        } else {
          throw new RuntimeException("Unknown language ID: " + title.getLeftElement());
        }
      }

      if (cnt == 2) {
        output.collect(fTitle, eTitle);
View Full Code Here
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
      float sumProb2 = 0;
      for (Entry<String> entry : probDist.entrySet()) {
        float pr = entry.getValue() / sumProb;
        if (pr > lexProbThreshold) {
          sumProb2 += pr;
          sortedFilteredProbDist.add(new PairOfStringFloat(entry.getKey(), pr));
        }
      }

      // re-normalize values after removal of low-prob entries
      float cumProb = 0;
      int cnt = 0;
      while (cnt < maxNumTrans && cumProb < cumProbThreshold && !sortedFilteredProbDist.isEmpty()) {
        PairOfStringFloat entry = sortedFilteredProbDist.pollLast();
        float pr = entry.getValue() / sumProb2;
        cumProb += pr;
        normProbDist.put(entry.getKey(), pr);
        cnt++;
      }

      probMap.put(sourceTerm, normProbDist);
    }
View Full Code Here
TOP

Related Classes of edu.umd.cloud9.io.pair.PairOfIntString

  • bak.pcj.IntIterator
  • edu.umd.cloud9.example.bfs.BfsNodeTest
  • edu.umd.cloud9.example.pagerank.PageRankNodeTest
  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.map.HMapIDW
  • edu.umd.cloud9.io.map.HMapIDWTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.