Package org.apache.hadoop.hbase.regionserver.compactions

Examples of org.apache.hadoop.hbase.regionserver.compactions.CompactionContext

303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
    if (this.server.isStopped()
        || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) {
      return null;
    }

    CompactionContext compaction = null;
    if (selectNow) {
      compaction = selectCompaction(r, s, priority, request);
      if (compaction == null) return null; // message logged inside
    }

    // We assume that most compactions are small. So, put system compactions into small
    // pool; we will do selection there, and move to large pool if necessary.
    long size = selectNow ? compaction.getRequest().getSize() : 0;
    ThreadPoolExecutor pool = (!selectNow && s.throttleCompaction(size))
      ? largeCompactions : smallCompactions;
    pool.execute(new CompactionRunner(s, r, compaction, pool));
    if (LOG.isDebugEnabled()) {
      String type = (pool == smallCompactions) ? "Small " : "Large ";
      LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system")
          + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this);
    }
    return selectNow ? compaction.getRequest() : null;
  }
View Full Code Here
298
299
300
301
302
303
304
305
306
307
308
309
    // Each compaction request will find one expired store file and delete it
    // by the compaction.
    for (int i = 1; i <= storeFileNum; i++) {
      // verify the expired store file.
      CompactionContext compaction = this.store.requestCompaction();
      CompactionRequest cr = compaction.getRequest();
      // the first is expired normally.
      // If not the first compaction, there is another empty store file,
      List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
      Assert.assertEquals(Math.min(i, 2), cr.getFiles().size());
      for (int j = 0; j < files.size(); j++) {
View Full Code Here
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
   *
   * @throws IOException e
   */
  public void compactStores() throws IOException {
    for (Store s : getStores().values()) {
      CompactionContext compaction = s.requestCompaction();
      if (compaction != null) {
        compact(compaction, s);
      }
    }
  }
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
    insertData(tableName, admin, t);
   
    int fileNum = store.getStorefiles().size();
    // 0, Compaction Request
    store.triggerMajorCompaction();
    CompactionContext cc = store.requestCompaction();
    assertNotNull(cc);
    // 1, A timeout split
    // 1.1 close region
    assertEquals(2, region.close(false).get(cf).size());
    // 1.2 rollback and Region initialize again
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
    if (this.server.isStopped()
        || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) {
      return null;
    }

    CompactionContext compaction = null;
    if (selectNow) {
      compaction = selectCompaction(r, s, priority, request);
      if (compaction == null) return null; // message logged inside
    }

    // We assume that most compactions are small. So, put system compactions into small
    // pool; we will do selection there, and move to large pool if necessary.
    long size = selectNow ? compaction.getRequest().getSize() : 0;
    ThreadPoolExecutor pool = (!selectNow && s.throttleCompaction(size))
      ? longCompactions : shortCompactions;
    pool.execute(new CompactionRunner(s, r, compaction, pool));
    if (LOG.isDebugEnabled()) {
      String type = (pool == shortCompactions) ? "Small " : "Large ";
      LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system")
          + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this);
    }
    return selectNow ? compaction.getRequest() : null;
  }
View Full Code Here
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
    return selectNow ? compaction.getRequest() : null;
  }

  private CompactionContext selectCompaction(final HRegion r, final Store s,
      int priority, CompactionRequest request) throws IOException {
    CompactionContext compaction = s.requestCompaction(priority, request);
    if (compaction == null) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("Not compacting " + r.getRegionNameAsString() +
            " because compaction request was cancelled");
      }
      return null;
    }
    assert compaction.hasSelection();
    if (priority != Store.NO_PRIORITY) {
      compaction.getRequest().setPriority(priority);
    }
    return compaction;
  }
View Full Code Here
538
539
540
541
542
543
544
545
546
547
    @Override
    public int compareTo(CompactionRunner o) {
      // Only compare the underlying request (if any), for queue sorting purposes.
      int compareVal = queuedPriority - o.queuedPriority; // compare priority
      if (compareVal != 0) return compareVal;
      CompactionContext tc = this.compaction, oc = o.compaction;
      // Sort pre-selected (user?) compactions before system ones with equal priority.
      return (tc == null) ? ((oc == null) ? 0 : 1)
          : ((oc == null) ? -1 : tc.getRequest().compareTo(oc.getRequest()));
    }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    // Produce 3 L0 files.
    StoreFile sf = createFile();
    ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile());
    se.getStoreFileManager().loadFiles(compactUs);
    // Create a compaction that would want to split the stripe.
    CompactionContext compaction = se.createCompaction();
    compaction.select(al(), false, false, false);
    assertEquals(3, compaction.getRequest().getFiles().size());
    // Override the file list. Granted, overriding this compaction in this manner will
    // break things in real world, but we only want to verify the override.
    compactUs.remove(sf);
    CompactionRequest req = new CompactionRequest(compactUs);
    compaction.forceSelect(req);
    assertEquals(2, compaction.getRequest().getFiles().size());
    assertFalse(compaction.getRequest().getFiles().contains(sf));
    // Make sure the correct method it called on compactor.
    compaction.compact();
    verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L,
          StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null);
  }
View Full Code Here
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
      }
    }

    @Override
    public synchronized CompactionContext selectCompaction() {
      CompactionContext ctx = new TestCompactionContext(new ArrayList<StoreFile>(notCompacting));
      compacting.addAll(notCompacting);
      notCompacting.clear();
      try {
        ctx.select(null, false, false, false);
      } catch (IOException ex) {
        fail("Shouldn't happen");
      }
      return ctx;
    }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
        " family=" + familyName);
      if (major) {
        store.triggerMajorCompaction();
      }
      do {
        CompactionContext compaction = store.requestCompaction(Store.PRIORITY_USER, null);
        if (compaction == null) break;
        List<StoreFile> storeFiles = store.compact(compaction);
        if (storeFiles != null && !storeFiles.isEmpty()) {
          if (keepCompactedFiles && deleteCompacted) {
            for (StoreFile storeFile: storeFiles) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hbase.regionserver.compactions.CompactionContext

  • org.apache.hadoop.hbase.regionserver.CompactionTool$CompactionWorker
  • org.apache.hadoop.hbase.regionserver.CompactSplitThread
  • org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner
  • org.apache.hadoop.hbase.regionserver.HRegion
  • org.apache.hadoop.hbase.regionserver.HStore
  • org.apache.hadoop.hbase.regionserver.TestCompaction$StoreMockMaker
  • org.apache.hadoop.hbase.regionserver.TestSplitTransactionOnCluster
  • org.apache.hadoop.hbase.regionserver.TestStore
  • org.apache.hadoop.hbase.regionserver.TestStripeStoreEngine
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.