Examples of BufferedLoader

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    int maxId = 4096;
    int[] maxNumItems = { 1, 1, 2, 2, 3, 3, 3, 3, 1, 1 };
    int[] minNumItems = { 1, 1, 0, 1, 0, 0, 2, 3, 1, 0 };
    int[] count = new int[maxId];
    BigIntBuffer buffer = new BigIntBuffer();
    BufferedLoader loader = null;
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    Random rand = new Random();

    for (int i = 0; i < maxNumItems.length; i++) {
      loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
      for (int id = 0; id < maxId; id++) {
        count[id] = 0;
        int cnt = Math.max(rand.nextInt(maxNumItems[i] + 1), minNumItems[i]);
        for (int val = 0; val < cnt; val++) {
          if (loader.add(id, val)) count[id]++;
        }
      }

      nestedArray.load(maxId, loader);
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

250
251
252
253
254
255
256
257
258
259
260
261
262
263
  public void testCountNoReturnWithFilter() throws Throwable {
    int maxId = 20;
    int numVals = 10;
    int[] count = new int[numVals];

    BufferedLoader loader = new BufferedLoader(maxId);
    for (int val = 0; val < numVals; val++) {
      for (int i = 0; i < maxId - val; i++) {
        loader.add(i, val);
      }
    }

    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  @Override
  public void load(String fieldName, AtomicReader reader, TermListFactory<T> listFactory,
      WorkArea workArea) throws IOException {
    String field = fieldName.intern();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);
    BufferedLoader weightLoader = getBufferedLoader(maxdoc, null);

    @SuppressWarnings("unchecked")
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>) new TermStringList()
        : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();
    OpenBitSet bitset = new OpenBitSet(maxdoc + 1);
    int negativeValueCount = getNegativeValueCount(reader, field);
    int t = 1; // valid term id starts from 1
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);

    _overflow = false;

    String pre = null;

    int df = 0;
    int minID = -1;
    int maxID = -1;
    int docID = -1;
    int valId = 0;

    Terms terms = reader.terms(field);
    if (terms != null) {
      TermsEnum termsEnum = terms.iterator(null);
      BytesRef text;
      while ((text = termsEnum.next()) != null) {
        String strText = text.utf8ToString();
        String val = null;
        int weight = 0;
        String[] split = strText.split("\u0000");
        if (split.length > 1) {
          val = split[0];
          weight = Integer.parseInt(split[split.length - 1]);
        } else {
          continue;
        }

        if (pre == null || !val.equals(pre)) {
          if (pre != null) {
            freqList.add(df);
            minIDList.add(minID);
            maxIDList.add(maxID);
          }
          list.add(val);
          df = 0;
          minID = -1;
          maxID = -1;
          valId = (t - 1 < negativeValueCount) ? (negativeValueCount - t + 1) : t;
          t++;
        }

        Term term = new Term(field, strText);
        DocsEnum docsEnum = reader.termDocsEnum(term);
        if (docsEnum != null) {
          while ((docID = docsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
            df++;

            if (!loader.add(docID, valId)) {
              logOverflow(fieldName);
            } else {
              weightLoader.add(docID, weight);
            }

            if (docID < minID) minID = docID;
            bitset.fastSet(docID);
            while (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
              docID = docsEnum.docID();
              df++;
              if (!loader.add(docID, valId)) {
                logOverflow(fieldName);
              } else {
                weightLoader.add(docID, weight);
              }
              bitset.fastSet(docID);
            }
            if (docID > maxID) maxID = docID;
          }
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
   */
  public void load(String fieldName, AtomicReader reader, TermListFactory<T> listFactory,
      WorkArea workArea) throws IOException {
    String field = fieldName.intern();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);

    @SuppressWarnings("unchecked")
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>) new TermStringList()
        : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();
    OpenBitSet bitset = new OpenBitSet(maxdoc + 1);
    int negativeValueCount = getNegativeValueCount(reader, field);
    int t = 1; // valid term id starts from 1
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);

    _overflow = false;

    Terms terms = reader.terms(field);
    if (terms != null) {
      TermsEnum termsEnum = terms.iterator(null);
      BytesRef text;
      while ((text = termsEnum.next()) != null) {
        String strText = text.utf8ToString();
        list.add(strText);

        Term term = new Term(field, strText);
        DocsEnum docsEnum = reader.termDocsEnum(term);
        int df = 0;
        int minID = -1;
        int maxID = -1;
        int docID = -1;
        int valId = (t - 1 < negativeValueCount) ? (negativeValueCount - t + 1) : t;
        while ((docID = docsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
          df++;
          if (!loader.add(docID, valId)) logOverflow(fieldName);
          minID = docID;
          bitset.fastSet(docID);
          while (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
            docID = docsEnum.docID();
            df++;
            if (!loader.add(docID, valId)) logOverflow(fieldName);
            bitset.fastSet(docID);
          }
          maxID = docID;
        }
        freqList.add(df);
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
    }
  }

  protected BufferedLoader getBufferedLoader(int maxdoc, WorkArea workArea) {
    if (workArea == null) {
      return new BufferedLoader(maxdoc, _maxItems, new BigIntBuffer());
    } else {
      BigIntBuffer buffer = workArea.get(BigIntBuffer.class);
      if (buffer == null) {
        buffer = new BigIntBuffer();
        workArea.put(buffer);
      } else {
        buffer.reset();
      }

      BufferedLoader loader = workArea.get(BufferedLoader.class);
      if (loader == null || loader.capacity() < maxdoc) {
        loader = new BufferedLoader(maxdoc, _maxItems, buffer);
        workArea.put(loader);
      } else {
        loader.reset(maxdoc, _maxItems, buffer);
      }
      return loader;
    }
  }
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
                   TermListFactory<T> listFactory,
                   WorkArea workArea) throws IOException
  {
    long t0 = System.currentTimeMillis();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);
    BufferedLoader weightLoader = getBufferedLoader(maxdoc, null);

    TermEnum tenum = null;
    TermDocs tdoc = null;
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>)new TermStringList() : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();
    OpenBitSet bitset = new OpenBitSet(maxdoc + 1);
    int negativeValueCount = getNegativeValueCount(reader, fieldName.intern());
    int t = 0; // current term number
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);
    t++;
   
    _overflow = false;

    String pre = null;

    int df = 0;
    int minID = -1;
    int maxID = -1;
    int valId = 0;

    try
    {
      tdoc = reader.termDocs();
      tenum = reader.terms(new Term(fieldName, ""));
      if (tenum != null)
      {
        do
        {
          Term term = tenum.term();
          if (term == null || !fieldName.equals(term.field()))
            break;

          String val = term.text();

          if (val != null)
          {
            int weight = 0;
            String[] split = val.split("\u0000");
            if (split.length > 1)
            {
              val = split[0];
              weight = Integer.parseInt(split[split.length-1]);
            }
            if (pre == null || !val.equals(pre))
            {
              if (pre != null)
              {
                freqList.add(df);
                minIDList.add(minID);
                maxIDList.add(maxID);
              }

              list.add(val);

              df = 0;
              minID = -1;
              maxID = -1;
              valId = (t - 1 < negativeValueCount) ? (negativeValueCount - t + 1) : t;
              t++;
            }

            tdoc.seek(tenum);
            if(tdoc.next())
            {
              df++;
              int docid = tdoc.doc();

              if(!loader.add(docid, valId)) logOverflow(fieldName);
              else weightLoader.add(docid, weight);

              if (docid < minID) minID = docid;
              bitset.fastSet(docid);
              while(tdoc.next())
              {
                df++;
                docid = tdoc.doc();
               
                if(!loader.add(docid, valId)) logOverflow(fieldName);
                else weightLoader.add(docid, weight);

                bitset.fastSet(docid);
              }
              if (docid > maxID) maxID = docid;
            }
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
   */
  public void load(String fieldName, IndexReader reader, TermListFactory<T> listFactory, WorkArea workArea) throws IOException
  {
    long t0 = System.currentTimeMillis();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);

    TermEnum tenum = null;
    TermDocs tdoc = null;
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>)new TermStringList() : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();
    OpenBitSet bitset = new OpenBitSet(maxdoc + 1);
    int negativeValueCount = getNegativeValueCount(reader, fieldName.intern());
    int t = 0; // current term number
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);
    t++;
   
    _overflow = false;
    try
    {
      tdoc = reader.termDocs();
      tenum = reader.terms(new Term(fieldName, ""));
      if (tenum != null)
      {
        do
        {
          Term term = tenum.term();
          if (term == null || !fieldName.equals(term.field()))
            break;

          String val = term.text();

          if (val != null)
          {
            list.add(val);

            tdoc.seek(tenum);
            //freqList.add(tenum.docFreq()); // removed because the df doesn't take into account the num of deletedDocs
            int df = 0;
            int minID = -1;
            int maxID = -1;
            int valId = (t - 1 < negativeValueCount) ? (negativeValueCount - t + 1) : t;
            if(tdoc.next())
            {
              df++;
              int docid = tdoc.doc();
              if(!loader.add(docid, valId)) logOverflow(fieldName);
              minID = docid;
              bitset.fastSet(docid);
              while(tdoc.next())
              {
                df++;
                docid = tdoc.doc();
              
                if(!loader.add(docid, valId)) logOverflow(fieldName);
                bitset.fastSet(docid);
              }
              maxID = docid;
            }
            freqList.add(df);
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
  protected BufferedLoader getBufferedLoader(int maxdoc, WorkArea workArea)
  {
    if(workArea == null)
    {
      return new BufferedLoader(maxdoc, _maxItems, new BigIntBuffer());
    }
    else
    {
      BigIntBuffer buffer = workArea.get(BigIntBuffer.class);
      if(buffer == null)
      {
        buffer = new BigIntBuffer();
        workArea.put(buffer);
      }
      else
      {
        buffer.reset();
      }
     
      BufferedLoader loader = workArea.get(BufferedLoader.class);     
      if(loader == null || loader.capacity() < maxdoc)
      {
        loader = new BufferedLoader(maxdoc, _maxItems, buffer);
        workArea.put(loader);
      }
      else
      {
        loader.reset(maxdoc, _maxItems, buffer);
      }
      return loader;
    }
  }
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
   */
  public void load(String fieldName, IndexReader reader, TermListFactory<T> listFactory, WorkArea workArea) throws IOException
  {
    long t0 = System.currentTimeMillis();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);

    TermEnum tenum = null;
    TermDocs tdoc = null;
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>)new TermStringList() : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();
    OpenBitSet bitset = new OpenBitSet(maxdoc + 1);

    int t = 0; // current term number
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);
    t++;
   
    _overflow = false;
    try
    {
      tdoc = reader.termDocs();
      tenum = reader.terms(new Term(fieldName, ""));
      if (tenum != null)
      {
        do
        {
          Term term = tenum.term();
          if (term == null || !fieldName.equals(term.field()))
            break;

          String val = term.text();

          if (val != null)
          {
            list.add(val);

            tdoc.seek(tenum);
            //freqList.add(tenum.docFreq()); // removed because the df doesn't take into account the num of deletedDocs
            int df = 0;
            int minID = -1;
            int maxID = -1;
            if(tdoc.next())
            {
              df++;
              int docid = tdoc.doc();
              if(!loader.add(docid, t)) logOverflow(fieldName);
              minID = docid;
              bitset.fastSet(docid);
              while(tdoc.next())
              {
                df++;
                docid = tdoc.doc();
                if(!loader.add(docid, t)) logOverflow(fieldName);
                bitset.fastSet(docid);
              }
              maxID = docid;
            }
            freqList.add(df);
View Full Code Here

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
  private BufferedLoader getBufferedLoader(int maxdoc, WorkArea workArea)
  {
    if(workArea == null)
    {
      return new BufferedLoader(maxdoc, _maxItems, new BigIntBuffer());
    }
    else
    {
      BigIntBuffer buffer = workArea.get(BigIntBuffer.class);
      if(buffer == null)
      {
        buffer = new BigIntBuffer();
        workArea.put(buffer);
      }
      else
      {
        buffer.reset();
      }
     
      BufferedLoader loader = workArea.get(BufferedLoader.class);     
      if(loader == null || loader.capacity() < maxdoc)
      {
        loader = new BufferedLoader(maxdoc, _maxItems, buffer);
        workArea.put(loader);
      }
      else
      {
        loader.reset(maxdoc, _maxItems, buffer);
      }
      return loader;
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.