Examples of ivory.core.data.index.TermPositions

ivory.core.data.index.TermPositions

A Hadoop {@code Writable} that encodes the position of term occurrences within a document. Termoccurrences are represented as an array of ints, where each int represents a term position. These objects serve as intermediate values in building document-sorted inverted indexes.

In serialized form, term positions are represented as first-order differences (i.e., position gaps or p-gaps) using Gamma encoding. As an example, let's say a term has a term frequency of 5, at token positions [3, 53, 58, 90, 101]. Such an object would be encoded as the following sequence of ints: 3, 50, 5, 32, 11, each of which is expressed using Gamma codes. Every int except the first represents the difference between the previous term position and the current term position.
@author Jimmy Lin

    assertEquals(98921257, pos2[4]);
  }


  @Test(expected = RuntimeException.class)
  public void tesDuplicateDocnos() throws IOException {
    TermPositions tp = new TermPositions();
    // Illegal positions.
    int[] pos = { 1, 4, 5, 5, 23 };


    tp.set(pos, (short) 5);
    assertEquals(5, tp.getTf());


    tp.serialize();
  }

View Full Code Here

    tp.serialize();
  }


  @Test(expected = RuntimeException.class)
  public void testIllegalPositions() throws IOException {
    TermPositions tp = new TermPositions();
    // illegal positions
    int[] pos = { 1, 4, 5, -2, -1 };


    tp.set(pos, (short) 5);
    assertEquals(5, tp.getTf());


    tp.serialize();
  }

View Full Code Here

        }


        numPostings = 0;
        Iterator<TermPositions> iter = values.iterator();
        while (iter.hasNext()) {
          TermPositions positions = iter.next();
          numPostings += positions.getPositions()[0];
        }


        postings.setNumberOfPostings(numPostings);
        return;
      }


      Iterator<TermPositions> iter = values.iterator();
      TermPositions positions = iter.next();
      postings.add(pair.getRightElement(), positions.getTf(), positions);


      if (iter.hasNext()) {
        throw new RuntimeException(
            String.format("Error: values with the same (term, docno): docno=%d, term=%d",
                pair.getRightElement(), curTerm));

View Full Code Here


public class TermPositionsTest {


  @Test
  public void testBasic1() throws IOException {
    TermPositions tp = new TermPositions();
    int[] pos = { 1, 4, 5, 10, 23 };
    tp.set(pos, (short) 5);


    assertEquals(tp.getTf(), 5);
  }

View Full Code Here

    assertEquals(tp.getTf(), 5);
  }


  @Test
  public void testSerial1() throws IOException {
    TermPositions tp = new TermPositions();
    int[] pos = { 1, 4, 5, 10, 23 };
    tp.set(pos, (short) 5);


    assertEquals(5, tp.getTf());


    TermPositions tp2 = TermPositions.create(tp.serialize());


    assertEquals(5, tp2.getTf());
    int[] pos2 = tp2.getPositions();


    assertEquals(1, pos2[0]);
    assertEquals(4, pos2[1]);
    assertEquals(5, pos2[2]);
    assertEquals(10, pos2[3]);

View Full Code Here


      if (partialPostings.size() == 0) {
        return true;
      }


      TermPositions tp = new TermPositions();
      // Start the timer.
      long startTime = System.currentTimeMillis();
      for (MapIV.Entry<PostingsAccumulator> e : partialPostings.entrySet()) {
        // Emit a partial posting list for each term.
        TERM.set(e.getKey());
        context.setStatus("t" + TERM.get());
        PostingsAccumulator pl = e.getValue();
        postingsList.clear();
        postingsList.setCollectionDocumentCount(collectionDocumentCount);
        postingsList.setNumberOfPostings(pl.size());


        int[] docnos = pl.getDocnos();
        int[][] positions = pl.getPositions();
        QuickSort.quicksortWithStack(positions, docnos, 0, pl.size() - 1);
        for (int i = 0; i < pl.size(); i++) {
          tp.set(positions[i], (short) positions[i].length);
          postingsList.add(docnos[i], tp.getTf(), tp);
        }
        context.write(TERM, postingsList);
      }
      context.getCounter(MapTime.Spilling).increment(System.currentTimeMillis() - startTime);
      partialPostings.clear();

View Full Code Here

        }


        numPostings = 0;
        Iterator<TermPositions> iter = values.iterator();
        while (iter.hasNext()) {
          TermPositions positions = iter.next();
          numPostings += positions.getPositions()[0];
        }


        postings.setNumberOfPostings(numPostings);
        return;
      }


      Iterator<TermPositions> iter = values.iterator();
      TermPositions positions = iter.next();
      postings.add(pair.getRightElement(), positions.getTf(), positions);


      if (iter.hasNext()) {
        throw new RuntimeException(
            String.format("Error: values with the same (term, docno): docno=%d, term=%d",
                pair.getRightElement(), curTerm));

View Full Code Here

          positionsMap.clear();
          int[] data = new int[pl.getDf()];
          int index = 0;
          while (reader.nextPosting(posting)) {
            data[index] = newDocids[posting.getDocno()];
            positionsMap.put(data[index], new TermPositions(reader.getPositions(), reader.getTf()));
            docLengths.put(data[index], env.getDocumentLength(posting.getDocno()));
            index++;
          }
          Arrays.sort(data);

View Full Code Here


      int cnt = 0;
      while(r.hasMoreTerms()) {
        termids.add(r.nextTerm());
        int[] p = r.getPositions();
        positions.add(new TermPositions(p, r.getTf()));


        for(int j = 0; j < p.length; j++) {
          if(p[j] > cnt) {
            cnt = p[j];
          }

View Full Code Here

    StringBuffer s = new StringBuffer("[");
    try {
      Reader r = this.getReader();
      while (r.hasMoreTerms()) {
        int id = r.nextTerm();
        TermPositions pos = new TermPositions();
        r.getPositions(pos);
        s.append("(" + id + ", " + pos.getTf() + ", " + pos + ")");
      }
      s.append("]");
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of ivory.core.data.index.TermPositions

ivory.core.compression.BitInputStream

ivory.core.compression.BitOutputStream

ivory.core.data.document.LazyIntDocVector

ivory.core.data.document.LazyTermDocVector

ivory.core.data.index.TermPositionsTest

ivory.core.index.BuildIPInvertedIndexDocSorted$MyReducer

ivory.core.index.BuildLPInvertedIndexDocSorted$MyMapper

ivory.ffg.data.CompressedPositionalPostingsTest

ivory.ffg.data.DocumentVectorUtility

ivory.ffg.preprocessing.GenerateCompressedPositionalPostings

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.