Package org.apache.hadoop.hbase.regionserver.compactions

Examples of org.apache.hadoop.hbase.regionserver.compactions.CompactionContext


   *
   * @throws IOException e
   */
  public void compactStores() throws IOException {
    for (Store s : getStores().values()) {
      CompactionContext compaction = s.requestCompaction();
      if (compaction != null) {
        compact(compaction, s);
      }
    }
  }
View Full Code Here


    // Each compaction request will find one expired store file and delete it
    // by the compaction.
    for (int i = 1; i <= storeFileNum; i++) {
      // verify the expired store file.
      CompactionContext compaction = this.store.requestCompaction();
      CompactionRequest cr = compaction.getRequest();
      // the first is expired normally.
      // If not the first compaction, there is another empty store file,
      List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
      Assert.assertEquals(Math.min(i, 2), cr.getFiles().size());
      for (int j = 0; j < files.size(); j++) {
View Full Code Here

      }
    }

    @Override
    public synchronized CompactionContext selectCompaction() {
      CompactionContext ctx = new TestCompactionContext(new ArrayList<StoreFile>(notCompacting));
      compacting.addAll(notCompacting);
      notCompacting.clear();
      try {
        ctx.select(null, false, false, false);
      } catch (IOException ex) {
        fail("Shouldn't happen");
      }
      return ctx;
    }
View Full Code Here

    // Produce 3 L0 files.
    StoreFile sf = createFile();
    ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile());
    se.getStoreFileManager().loadFiles(compactUs);
    // Create a compaction that would want to split the stripe.
    CompactionContext compaction = se.createCompaction();
    compaction.select(al(), false, false, false);
    assertEquals(3, compaction.getRequest().getFiles().size());
    // Override the file list. Granted, overriding this compaction in this manner will
    // break things in real world, but we only want to verify the override.
    compactUs.remove(sf);
    CompactionRequest req = new CompactionRequest(compactUs);
    compaction.forceSelect(req);
    assertEquals(2, compaction.getRequest().getFiles().size());
    assertFalse(compaction.getRequest().getFiles().contains(sf));
    // Make sure the correct method it called on compactor.
    compaction.compact();
    verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L,
          StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null);
  }
View Full Code Here

    if (this.server.isStopped()
        || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) {
      return null;
    }

    CompactionContext compaction = null;
    if (selectNow) {
      compaction = selectCompaction(r, s, priority, request);
      if (compaction == null) return null; // message logged inside
    }

    // We assume that most compactions are small. So, put system compactions into small
    // pool; we will do selection there, and move to large pool if necessary.
    long size = selectNow ? compaction.getRequest().getSize() : 0;
    ThreadPoolExecutor pool = (!selectNow && s.throttleCompaction(size))
      ? largeCompactions : smallCompactions;
    pool.execute(new CompactionRunner(s, r, compaction, pool));
    if (LOG.isDebugEnabled()) {
      String type = (pool == smallCompactions) ? "Small " : "Large ";
      LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system")
          + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this);
    }
    return selectNow ? compaction.getRequest() : null;
  }
View Full Code Here

    return selectNow ? compaction.getRequest() : null;
  }

  private CompactionContext selectCompaction(final HRegion r, final Store s,
      int priority, CompactionRequest request) throws IOException {
    CompactionContext compaction = s.requestCompaction(priority, request);
    if (compaction == null) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("Not compacting " + r.getRegionNameAsString() +
            " because compaction request was cancelled");
      }
      return null;
    }
    assert compaction.hasSelection();
    if (priority != Store.NO_PRIORITY) {
      compaction.getRequest().setPriority(priority);
    }
    return compaction;
  }
View Full Code Here

    @Override
    public int compareTo(CompactionRunner o) {
      // Only compare the underlying request (if any), for queue sorting purposes.
      int compareVal = queuedPriority - o.queuedPriority; // compare priority
      if (compareVal != 0) return compareVal;
      CompactionContext tc = this.compaction, oc = o.compaction;
      // Sort pre-selected (user?) compactions before system ones with equal priority.
      return (tc == null) ? ((oc == null) ? 0 : 1)
          : ((oc == null) ? -1 : tc.getRequest().compareTo(oc.getRequest()));
    }
View Full Code Here

    // don't even select for compaction if writes are disabled
    if (!this.areWritesEnabled()) {
      return null;
    }

    CompactionContext compaction = storeEngine.createCompaction();
    this.lock.readLock().lock();
    try {
      synchronized (filesCompacting) {
        // First, see if coprocessor would want to override selection.
        if (this.getCoprocessorHost() != null) {
          List<StoreFile> candidatesForCoproc = compaction.preSelect(this.filesCompacting);
          boolean override = this.getCoprocessorHost().preCompactSelection(
              this, candidatesForCoproc, baseRequest);
          if (override) {
            // Coprocessor is overriding normal file selection.
            compaction.forceSelect(new CompactionRequest(candidatesForCoproc));
          }
        }

        // Normal case - coprocessor is not overriding file selection.
        if (!compaction.hasSelection()) {
          boolean isUserCompaction = priority == Store.PRIORITY_USER;
          boolean mayUseOffPeak = offPeakHours.isOffPeakHour() &&
              offPeakCompactionTracker.compareAndSet(false, true);
          try {
            compaction.select(this.filesCompacting, isUserCompaction,
              mayUseOffPeak, forceMajor && filesCompacting.isEmpty());
          } catch (IOException e) {
            if (mayUseOffPeak) {
              offPeakCompactionTracker.set(false);
            }
            throw e;
          }
          assert compaction.hasSelection();
          if (mayUseOffPeak && !compaction.getRequest().isOffPeak()) {
            // Compaction policy doesn't want to take advantage of off-peak.
            offPeakCompactionTracker.set(false);
          }
        }
        if (this.getCoprocessorHost() != null) {
          this.getCoprocessorHost().postCompactSelection(
              this, ImmutableList.copyOf(compaction.getRequest().getFiles()), baseRequest);
        }

        // Selected files; see if we have a compaction with some custom base request.
        if (baseRequest != null) {
          // Update the request with what the system thinks the request should be;
          // its up to the request if it wants to listen.
          compaction.forceSelect(
              baseRequest.combineWith(compaction.getRequest()));
        }

        // Finally, we have the resulting files list. Check if we have any files at all.
        final Collection<StoreFile> selectedFiles = compaction.getRequest().getFiles();
        if (selectedFiles.isEmpty()) {
          return null;
        }

        // Update filesCompacting (check that we do not try to compact the same StoreFile twice).
        if (!Collections.disjoint(filesCompacting, selectedFiles)) {
          Preconditions.checkArgument(false, "%s overlaps with %s",
              selectedFiles, filesCompacting);
        }
        filesCompacting.addAll(selectedFiles);
        Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);

        // If we're enqueuing a major, clear the force flag.
        boolean isMajor = selectedFiles.size() == this.getStorefilesCount();
        this.forceMajor = this.forceMajor && !isMajor;

        // Set common request properties.
        // Set priority, either override value supplied by caller or from store.
        compaction.getRequest().setPriority(
            (priority != Store.NO_PRIORITY) ? priority : getCompactPriority());
        compaction.getRequest().setIsMajor(isMajor);
        compaction.getRequest().setDescription(
            getRegionInfo().getRegionNameAsString(), getColumnFamilyName());
      }
    } finally {
      this.lock.readLock().unlock();
    }

    LOG.debug(getRegionInfo().getEncodedName() + " - " + getColumnFamilyName() + ": Initiating "
        + (compaction.getRequest().isMajor() ? "major" : "minor") + " compaction");
    this.region.reportCompactionRequestStart(compaction.getRequest().isMajor());
    return compaction;
  }
View Full Code Here

   *
   * @throws IOException e
   */
  public void compactStores() throws IOException {
    for (Store s : getStores().values()) {
      CompactionContext compaction = s.requestCompaction();
      if (compaction != null) {
        compact(compaction, s);
      }
    }
  }
View Full Code Here

        " family=" + familyName);
      if (major) {
        store.triggerMajorCompaction();
      }
      do {
        CompactionContext compaction = store.requestCompaction(Store.PRIORITY_USER, null);
        if (compaction == null) break;
        List<StoreFile> storeFiles = store.compact(compaction);
        if (storeFiles != null && !storeFiles.isEmpty()) {
          if (keepCompactedFiles && deleteCompacted) {
            for (StoreFile storeFile: storeFiles) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.compactions.CompactionContext

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.