Package ivory.core.data.document.IntDocVector

Examples of ivory.core.data.document.IntDocVector.Reader

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
      IntDocVector doc = docVecs[i];

      Map<String, Short> docTfs = new HashMap<String, Short>();
      int doclen = 0;

      Reader dvReader = doc.getReader();
      while (dvReader.hasMoreTerms()) {
        int termid = dvReader.nextTerm();
        String stem = env.getTermFromId(termid);
        short tf = dvReader.getTf();

        doclen += tf;

        if (stem != null && (stopwords == null || !stopwords.contains(stem))) {
          Integer df = vocab.get(stem);
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
    for (IntDocVector doc : docVecs) {
      HMapIV<String> termMap = new HMapIV<String>();
      int maxPos = Integer.MIN_VALUE;

      Reader dvReader = doc.getReader();
      while (dvReader.hasMoreTerms()) {
        int termid = dvReader.nextTerm();
        String stem = env.getTermFromId(termid);
        int[] pos = dvReader.getPositions();
        for (int i = 0; i < pos.length; i++) {
          termMap.put(pos[i], stem);
          if (pos[i] > maxPos) {
            maxPos = pos[i];
          }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
      IntDocVector doc = docVecs[i];

      Map<String, Short> docTfs = new HashMap<String, Short>();
      int doclen = 0;

      Reader dvReader = doc.getReader();
      while (dvReader.hasMoreTerms()) {
        int termid = dvReader.nextTerm();
        String stem = env.getTermFromId(termid);
        short tf = dvReader.getTf();

        doclen += tf;

        if (stem != null && (stopwords == null || !stopwords.contains(stem))) {
          Integer df = vocab.get(stem);
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
    for (IntDocVector doc : docVecs) {
      HMapIV<String> termMap = new HMapIV<String>();
      int maxPos = Integer.MIN_VALUE;

      Reader dvReader = doc.getReader();
      while (dvReader.hasMoreTerms()) {
        int termid = dvReader.nextTerm();
        String stem = env.getTermFromId(termid);
        int[] pos = dvReader.getPositions();
        for (int i = 0; i < pos.length; i++) {
          termMap.put(pos[i], stem);
          if (pos[i] > maxPos) {
            maxPos = pos[i];
          }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    public void map(IntWritable key, IntDocVector doc, Context context)
        throws IOException, InterruptedException {
      docno = key.get();

      long startTime = System.currentTimeMillis();
      Reader r = doc.getReader();

      int dl = 0;
      while (r.hasMoreTerms()) {
        int term = r.nextTerm();
        r.getPositions(termPositions);

        // Set up the key and value, and emit.
        pair.set(term, docno);
        context.write(pair, termPositions);
View Full Code Here
TOP

Related Classes of ivory.core.data.document.IntDocVector.Reader

  • ivory.core.index.BuildIPInvertedIndexDocSorted$MyMapper
  • ivory.smrf.model.expander.MRFExpander
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.