Package com.browseengine.bobo.util.BigNestedIntArray

Examples of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  public void testBasic() throws Throwable
  {
    int maxId = 3000;
    int[] count = new int[maxId];
    BufferedLoader loader = new BufferedLoader(maxId);
    for(int id = 0; id < maxId; id++)
    {
      for(int val = 0; val < 2000; val += (id + 1))
      {
        if(loader.add(id, val)) count[id]++;
      }
    }
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
   
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
  public void testSparseIds() throws Throwable
  {
    int maxId = 100000;
    int[] count = new int[maxId];
    BufferedLoader loader = new BufferedLoader(maxId);
    for(int id = 0; id < maxId; id += ((id >> 2) + 1))
    {
      for(int val = 0; val < 3000; val += (id + 1))
      {
        if(loader.add(id, val)) count[id]++;
      }
    }
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
   
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  {
    int maxId = 5000;
    int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 2000, 2 };
    int[][] count = new int[maxNumItems.length][maxId];
    BigIntBuffer buffer = new BigIntBuffer();
    BufferedLoader loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
    BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.length];
   
    for(int i = 0 ; i < maxNumItems.length; i++)
    {
      for(int id = 0; id < maxId; id++)
      {
        int cnt = id % (maxNumItems[i] + 1);
        for(int val = 0; val < cnt; val++)
        {
          if(loader.add(id, val)) count[i][id]++;
        }
      }
      nestedArray[i] = new BigNestedIntArray();
      nestedArray[i].load(maxId, loader);
     
      loader.reset(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
    }
   
    for(int i = 0 ; i < maxNumItems.length; i++)
    {
      int[] buf = new int[1024];
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
    int maxId = 4096;
    int[] maxNumItems = {1, 1, 2, 2, 3, 3, 3, 3, 1, 1 };
    int[] minNumItems = {1, 1, 0, 1, 0, 0, 2, 3, 1, 0 };
    int[] count = new int[maxId];
    BigIntBuffer buffer = new BigIntBuffer();
    BufferedLoader loader = null;
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    Random rand = new Random();
   
    for(int i = 0; i < maxNumItems.length; i++)
    {
      loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
      for(int id = 0; id < maxId; id++)
      {
        count[id] = 0;
        int cnt = Math.max(rand.nextInt(maxNumItems[i] + 1), minNumItems[i]);
        for(int val = 0; val < cnt; val++)
        {
          if(loader.add(id, val)) count[id]++;
        }
      }
     
      nestedArray.load(maxId, loader);
     
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
  {
    int maxId = 20;
    int numVals = 10;
    int[] count = new int[numVals];
   
    BufferedLoader loader = new BufferedLoader(maxId);
    for (int val = 0; val < numVals; val++)
    {
      for (int i = 0; i < maxId - val; i++)
      {
        loader.add(i, val);
      }
    }

    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
   */
  public void load(String fieldName, IndexReader reader, TermListFactory<T> listFactory, WorkArea workArea) throws IOException
  {
    long t0 = System.currentTimeMillis();
    int maxdoc = reader.maxDoc();
    BufferedLoader loader = getBufferedLoader(maxdoc, workArea);

    TermEnum tenum = null;
    TermDocs tdoc = null;
    TermValueList<T> list = (listFactory == null ? (TermValueList<T>)new TermStringList() : listFactory.createTermList());
    IntArrayList minIDList = new IntArrayList();
    IntArrayList maxIDList = new IntArrayList();
    IntArrayList freqList = new IntArrayList();

    int t = 0; // current term number
    list.add(null);
    minIDList.add(-1);
    maxIDList.add(-1);
    freqList.add(0);
    t++;
   
    _overflow = false;
    try
    {
      tdoc = reader.termDocs();
      tenum = reader.terms(new Term(fieldName, ""));
      if (tenum != null)
      {
        do
        {
          Term term = tenum.term();
          if (term == null || !fieldName.equals(term.field()))
            break;

          String val = term.text();

          if (val != null)
          {
            list.add(val);

            tdoc.seek(tenum);
            //freqList.add(tenum.docFreq()); // removed because the df doesn't take into account the num of deletedDocs
            int df = 0;
            int minID = -1;
            int maxID = -1;
            if(tdoc.next())
            {
              df++;
              int docid = tdoc.doc();
              if(!loader.add(docid, t)) logOverflow(fieldName);
              minID = docid;
              while(tdoc.next())
              {
                df++;
                docid = tdoc.doc();
                if(!loader.add(docid, t)) logOverflow(fieldName);
              }
              maxID = docid;
            }
            freqList.add(df);
            minIDList.add(minID);
View Full Code Here
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
  private BufferedLoader getBufferedLoader(int maxdoc, WorkArea workArea)
  {
    if(workArea == null)
    {
      return new BufferedLoader(maxdoc, _maxItems, new BigIntBuffer());
    }
    else
    {
      BigIntBuffer buffer = workArea.get(BigIntBuffer.class);
      if(buffer == null)
      {
        buffer = new BigIntBuffer();
        workArea.put(buffer);
      }
      else
      {
        buffer.reset();
      }
     
      BufferedLoader loader = workArea.get(BufferedLoader.class);     
      if(loader == null || loader.capacity() < maxdoc)
      {
        loader = new BufferedLoader(maxdoc, _maxItems, buffer);
        workArea.put(loader);
      }
      else
      {
        loader.reset(maxdoc, _maxItems, buffer);
      }
      return loader;
    }
  }
View Full Code Here
17
18
19
20
21
22
23
24
25
26
27
28
29
30
public class BigNestedIntArrayTest extends TestCase {
  public void testBasic() throws Throwable {
    int maxId = 3000;
    int[] count = new int[maxId];
    BufferedLoader loader = new BufferedLoader(maxId);
    for (int id = 0; id < maxId; id++) {
      for (int val = 0; val < 2000; val += (id + 1)) {
        if (loader.add(id, val)) count[id]++;
      }
    }
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
  }

  public void testSparseIds() throws Throwable {
    int maxId = 100000;
    int[] count = new int[maxId];
    BufferedLoader loader = new BufferedLoader(maxId);
    for (int id = 0; id < maxId; id += ((id >> 2) + 1)) {
      for (int val = 0; val < 3000; val += (id + 1)) {
        if (loader.add(id, val)) count[id]++;
      }
    }
    BigNestedIntArray nestedArray = new BigNestedIntArray();
    nestedArray.load(maxId, loader);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  public void testBufferedLoaderReuse() throws Throwable {
    int maxId = 5000;
    int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 2000, 2 };
    int[][] count = new int[maxNumItems.length][maxId];
    BigIntBuffer buffer = new BigIntBuffer();
    BufferedLoader loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
    BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.length];

    for (int i = 0; i < maxNumItems.length; i++) {
      for (int id = 0; id < maxId; id++) {
        int cnt = id % (maxNumItems[i] + 1);
        for (int val = 0; val < cnt; val++) {
          if (loader.add(id, val)) count[i][id]++;
        }
      }
      nestedArray[i] = new BigNestedIntArray();
      nestedArray[i].load(maxId, loader);

      loader.reset(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
    }

    for (int i = 0; i < maxNumItems.length; i++) {
      int[] buf = new int[1024];
      for (int id = 0; id < maxId; id++) {
View Full Code Here
TOP

Related Classes of com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader

  • com.browseengine.bobo.facets.data.MultiValueFacetDataCache
  • com.browseengine.bobo.facets.data.MultiValueWithWeightFacetDataCache
  • com.browseengine.bobo.util.test.BigNestedIntArrayTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.