Package org.apache.accumulo.core.iterators

Examples of org.apache.accumulo.core.iterators.MultiIterator

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    try {
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.length);
      for (int i = 0; i < mapfiles.length; i++)
        iters.add(mapfiles[i]);
     
      MultiIterator mmfi = new MultiIterator(iters, new KeyExtent(new Text(""), endRow, prevEndRow));
      mmfi.seek(new Range(new Key(startRow), null), LocalityGroupUtil.EMPTY_CF_SET, false);
     
      int count = 0;
     
      long t1 = System.currentTimeMillis();
     
      Key lastKey = new Key();
     
      while (mmfi.hasTop()) {
        Key key = mmfi.getTopKey();
       
        if (lastKey.compareTo(key) > 0) {
          String msg = "Not sorted : " + lastKey + " " + key;
          System.err.println(msg);
          throw new RuntimeException(msg);
        }
       
        lastKey.set(key);
       
        System.out.println(" " + key);
       
        mmfi.next();
        count++;
      }
     
      long t2 = System.currentTimeMillis();
     
View Full Code Here
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
      iters.addAll(memIters);
     
      for (SortedKeyValueIterator<Key,Value> skvi : iters)
        ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
     
      MultiIterator multiIter = new MultiIterator(iters, extent);
     
      IteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager);
     
      DeletingIterator delIter = new DeletingIterator(multiIter, false);
     
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
   
    for (TreeMap<Key,Value> map : maps) {
      iters.add(new SortedMapIterator(map));
    }
   
    MultiIterator mi;
    if (endRow == null && prevEndRow == null)
      mi = new MultiIterator(iters, init);
    else {
      Range range = new Range(prevEndRow, false, endRow, true);
      if (init)
        for (SortedKeyValueIterator<Key,Value> iter : iters)
          iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
      mi = new MultiIterator(iters, range);
     
      if (init)
        mi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
    }
   
    if (seekKey != null) {
      mi.seek(new Range(seekKey, null), EMPTY_COL_FAMS, false);
    }
   
    int i = start;
    while (mi.hasTop()) {
      if (incrRow)
        assertEquals(nk(i, 0), mi.getTopKey());
      else
        assertEquals(nk(0, i), mi.getTopKey());
     
      assertEquals("v" + i, mi.getTopValue().toString());
     
      mi.next();
      if (incrRow)
        i++;
      else
        i--;
    }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
   
    TreeMap<Key,Value> tm1 = new TreeMap<Key,Value>();
   
    List<SortedKeyValueIterator<Key,Value>> skvil = new ArrayList<SortedKeyValueIterator<Key,Value>>(1);
    skvil.add(new SortedMapIterator(tm1));
    MultiIterator mi = new MultiIterator(skvil, true);
   
    assertFalse(mi.hasTop());
   
    mi.seek(nrng(0, 6), EMPTY_COL_FAMS, false);
    assertFalse(mi.hasTop());
  }
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
    nkv(tm1, 4, 0, false, "2");
    nkv(tm1, 6, 0, false, "3");
   
    List<SortedKeyValueIterator<Key,Value>> skvil = new ArrayList<SortedKeyValueIterator<Key,Value>>(1);
    skvil.add(new SortedMapIterator(tm1));
    MultiIterator mi = new MultiIterator(skvil, true);
    mi.seek(new Range(null, true, nk(5, 9), false), EMPTY_COL_FAMS, false);
   
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(3, 0)));
    assertTrue(mi.getTopValue().toString().equals("1"));
    mi.next();
   
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(4, 0)));
    assertTrue(mi.getTopValue().toString().equals("2"));
    mi.next();
   
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 10), true, nk(5, 9), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(4, 0)));
    assertTrue(mi.getTopValue().toString().equals("2"));
    mi.next();
   
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 10), true, nk(6, 0), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(4, 0)));
    assertTrue(mi.getTopValue().toString().equals("2"));
    mi.next();
   
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 10), true, nk(6, 0), true), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(4, 0)));
    assertTrue(mi.getTopValue().toString().equals("2"));
    mi.next();
   
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(6, 0)));
    assertTrue(mi.getTopValue().toString().equals("3"));
    mi.next();
   
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 0), true, nk(6, 0), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(4, 0)));
    assertTrue(mi.getTopValue().toString().equals("2"));
    mi.next();
   
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 0), false, nk(6, 0), false), EMPTY_COL_FAMS, false);
    assertFalse(mi.hasTop());
   
    mi.seek(new Range(nk(4, 0), false, nk(6, 0), true), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertTrue(mi.getTopKey().equals(nk(6, 0)));
    assertTrue(mi.getTopValue().toString().equals("3"));
    mi.next();
    assertFalse(mi.hasTop());
   
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      String path = Constants.getMetadataTableDir() + converFileName(file);
      FileSKVIterator in = FileOperations.getInstance().openReader(path, true, fs, conf, AccumuloConfiguration.getDefaultConfiguration());
      tabletIters.add(in);
    }
   
    MultiIterator multiIter = new MultiIterator(tabletIters, ke);
    DeletingIterator delIter = new DeletingIterator(multiIter, false);
    VersioningIterator tabletIter = new VersioningIterator(delIter, 1);
    tabletIter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
   
    TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
      FileSKVIterator in = FileOperations.getInstance().openReader(fileStatus.getPath().toString(), true, fs, conf,
          AccumuloConfiguration.getDefaultConfiguration());
      rootFileIters.add(in);
    }
   
    MultiIterator multiIter = new MultiIterator(rootFileIters, true);
    DeletingIterator delIter = new DeletingIterator(multiIter, false);
    VersioningIterator rootIter = new VersioningIterator(delIter, 1);
   
    TreeMap<Key,Value> files = new TreeMap<Key,Value>();
   
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        for (String s : inFiles) {
          reader = FileOperations.getInstance().openIndex(s, fs, conf, AccumuloConfiguration.getSystemConfiguration());
          iters.add(reader);
        }
       
        MultiIterator mmfi = new MultiIterator(iters, true);
       
        while (mmfi.hasTop()) {
          Key key = mmfi.getTopKey();
         
          boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
          boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;
         
          if (gtPrevEndRow && lteEndRow)
            writer.append(key, new LongWritable(0));
         
          if (!lteEndRow)
            break;
         
          mmfi.next();
        }
      } finally {
        try {
          if (reader != null)
            reader.close();
View Full Code Here
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
        // the data just punt and return .5
        return .5;
      }
     
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
      MultiIterator mmfi = new MultiIterator(iters, true);
     
      // skip the prevendrow
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0) {
        mmfi.next();
      }
     
      int numLte = 0;
     
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(splitRow) <= 0) {
        numLte++;
        mmfi.next();
      }
     
      if (numLte > numKeys) {
        // something went wrong
        throw new RuntimeException("numLte > numKeys " + numLte + " " + numKeys + " " + prevEndRow + " " + endRow + " " + splitRow + " " + mapFiles);
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
        }
        throw new IOException("Failed to find mid point, no entries between " + prevEndRow + " and " + endRow + " for " + mapFiles);
      }
     
      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
      MultiIterator mmfi = new MultiIterator(iters, true);
     
      // skip the prevendrow
      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0)
        mmfi.next();
     
      // read half of the keys in the index
      TreeMap<Double,Key> ret = new TreeMap<Double,Key>();
      Key lastKey = null;
      long keysRead = 0;
     
      Key keyBeforeMidPoint = null;
      long keyBeforeMidPointPosition = 0;
     
      while (keysRead < numKeys / 2) {
        if (lastKey != null && !lastKey.equals(mmfi.getTopKey(), PartialKey.ROW) && (keysRead - 1) / (double) numKeys >= minSplit) {
          keyBeforeMidPoint = new Key(lastKey);
          keyBeforeMidPointPosition = keysRead - 1;
        }
       
        if (lastKey == null)
          lastKey = new Key();
       
        lastKey.set(mmfi.getTopKey());
       
        keysRead++;
       
        // consume minimum
        mmfi.next();
      }
     
      if (keyBeforeMidPoint != null)
        ret.put(keyBeforeMidPointPosition / (double) numKeys, keyBeforeMidPoint);
     
      long t2 = System.currentTimeMillis();
     
      log.debug(String.format("Found midPoint from indexes in %6.2f secs.\n", ((t2 - t1) / 1000.0)));
     
      ret.put(.5, mmfi.getTopKey());
     
      // sanity check
      for (Key key : ret.values()) {
        boolean inRange = (key.compareRow(prevEndRow) > 0 && (endRow == null || key.compareRow(endRow) < 1));
        if (!inRange) {
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.iterators.MultiIterator

  • org.apache.accumulo.core.data.Range
  • org.apache.accumulo.core.file.FileUtil
  • org.apache.accumulo.core.iterators.AggregatingIteratorTest
  • org.apache.accumulo.core.iterators.FamilyIntersectingIteratorTest
  • org.apache.accumulo.core.iterators.IntersectingIteratorTest
  • org.apache.accumulo.core.iterators.MultiIteratorTest
  • org.apache.accumulo.core.iterators.WholeRowIteratorTest
  • org.apache.accumulo.server.tabletserver.Tablet
  • org.apache.accumulo.server.tabletserver.Tablet$ScanDataSource
  • org.apache.accumulo.server.test.MMFITest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.