*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A Store holds a column family in a Region.  Its a memstore and a set of zero

* or more StoreFiles, which stretch backwards over time.

*

* <p>There's no reason to consider append-logging at this level; all logging

* and locking is handled at the HRegion level.  Store just provides

* services to manage sets of StoreFiles.  One of the most important of those

* services is compaction services where files are aggregated once they pass

* a configurable threshold.

*

* <p>The only thing having to do with logs that Store needs to deal with is

* the reconstructionLog.  This is a segment of an HRegion's log that might

* NOT be present upon startup.  If the param is NULL, there's nothing to do.

* If the param is non-NULL, we need to process the log to reconstruct

* a TreeMap that might not have been written to disk before the process

* died.

*

* <p>It's assumed that after this constructor returns, the reconstructionLog

* file will be deleted (by whoever has instantiated the Store).

*

* <p>Locking and transactions are handled at a higher level.  This API should

* not be called directly but by an HRegion manager.

*/

  public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =

      "hbase.server.compactchecker.interval.multiplier";

  public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";

  public static final int DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER = 1000;

  public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;

  static final Log LOG = LogFactory.getLog(HStore.class);

  protected final MemStore memstore;

  // This stores directory in the filesystem.

  private final HRegion region;

  private final HColumnDescriptor family;

  private final HRegionFileSystem fs;

  private final Configuration conf;

  private final CacheConfig cacheConf;

  private long lastCompactSize = 0;

  volatile boolean forceMajor = false;

  /* how many bytes to write between status checks */

  static int closeCheckInterval = 0;

  private volatile long storeSize = 0L;

  private volatile long totalUncompressedBytes = 0L;

  /**

   * RWLock for store operations.

   * Locked in shared mode when the list of component stores is looked at:

   *   - all reads/writes to table data

   *   - checking for split

   * Locked in exclusive mode when the list of component stores is modified:

   *   - closing

   *   - completing a compaction

   */

  final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

  private final boolean verifyBulkLoads;

  private ScanInfo scanInfo;

  final List<StoreFile> filesCompacting = Lists.newArrayList();

  // All access must be synchronized.

  private final Set<ChangedReadersObserver> changedReaderObservers =

    Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());

  private final int blocksize;

  private HFileDataBlockEncoder dataBlockEncoder;

  /** Checksum configuration */

  private ChecksumType checksumType;

  private int bytesPerChecksum;

  // Comparing KeyValues

  private final KeyValue.KVComparator comparator;

  final StoreEngine<?, ?, ?, ?> storeEngine;

  private static final AtomicBoolean offPeakCompactionTracker = new AtomicBoolean();

  private final OffPeakHours offPeakHours;

  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;

  private int flushRetriesNumber;

  private int pauseTime;

  private long blockingFileCount;

  private int compactionCheckMultiplier;

  private Encryption.Context cryptoContext = Encryption.Context.NONE;

  /**

   * Constructor

   * @param region

   * @param family HColumnDescriptor for this column

   * @param confParam configuration object

   * failed.  Can be null.

   * @throws IOException

   */

  protected HStore(final HRegion region, final HColumnDescriptor family,

      final Configuration confParam) throws IOException {

    HRegionInfo info = region.getRegionInfo();

    this.fs = region.getRegionFileSystem();

    // Assemble the store's home directory and Ensure it exists.

    fs.createStoreDir(family.getNameAsString());

    this.region = region;

    this.family = family;

    // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor

    // CompoundConfiguration will look for keys in reverse order of addition, so we'd

    // add global config first, then table and cf overrides, then cf metadata.

    this.conf = new CompoundConfiguration()

      .add(confParam)

      .addStringMap(region.getTableDesc().getConfiguration())

      .addStringMap(family.getConfiguration())

      .addWritableMap(family.getValues());

    this.blocksize = family.getBlocksize();

    this.dataBlockEncoder =

        new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());

    this.comparator = info.getComparator();

    // used by ScanQueryMatcher

    long timeToPurgeDeletes =

        Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);

    LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes +

        "ms in store " + this);

    // Get TTL

    long ttl = determineTTLFromFamily(family);

    // Why not just pass a HColumnDescriptor in here altogether?  Even if have

    // to clone it?

    scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);

    this.memstore = new MemStore(conf, this.comparator);

    this.offPeakHours = OffPeakHours.getInstance(conf);

    // Setting up cache configuration for this family

    this.cacheConf = new CacheConfig(conf, family);

    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);

    this.blockingFileCount =

        conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);

    this.compactionCheckMultiplier = conf.getInt(

        COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY, DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);

    if (this.compactionCheckMultiplier <= 0) {

      LOG.error("Compaction check period multiplier must be positive, setting default: "

          + DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);

      this.compactionCheckMultiplier = DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER;

    }

    if (HStore.closeCheckInterval == 0) {

      HStore.closeCheckInterval = conf.getInt(

          "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);

    }

    this.storeEngine = StoreEngine.create(this, this.conf, this.comparator);

    this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());

    // Initialize checksum type from name. The names are CRC32, CRC32C, etc.

    this.checksumType = getChecksumType(conf);

    // initilize bytes per checksum

    this.bytesPerChecksum = getBytesPerChecksum(conf);

    flushRetriesNumber = conf.getInt(

        "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);

    pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE, HConstants.DEFAULT_HBASE_SERVER_PAUSE);

    if (flushRetriesNumber <= 0) {

      throw new IllegalArgumentException(

          "hbase.hstore.flush.retries.number must be > 0, not "

              + flushRetriesNumber);

    }

    // Crypto context for new store files

    String cipherName = family.getEncryptionType();

    if (cipherName != null) {

      Cipher cipher;

      Key key;

      byte[] keyBytes = family.getEncryptionKey();

      if (keyBytes != null) {

        // Family provides specific key material

        String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,

          User.getCurrent().getShortName());

        try {

          // First try the master key

          key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);

        } catch (KeyException e) {

          // If the current master key fails to unwrap, try the alternate, if

          // one is configured

          if (LOG.isDebugEnabled()) {

            LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");

          }

          String alternateKeyName =

            conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);

          if (alternateKeyName != null) {

            try {

              key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);

            } catch (KeyException ex) {

              throw new IOException(ex);

            }

          } else {

            throw new IOException(e);

          }

        }

        // Use the algorithm the key wants

        cipher = Encryption.getCipher(conf, key.getAlgorithm());

        if (cipher == null) {

          throw new RuntimeException("Cipher '" + cipher + "' is not available");

        }

        // Fail if misconfigured

        // We use the encryption type specified in the column schema as a sanity check on

        // what the wrapped key is telling us

        if (!cipher.getName().equalsIgnoreCase(cipherName)) {

          throw new RuntimeException("Encryption for family '" + family.getNameAsString() +

            "' configured with type '" + cipherName +

            "' but key specifies algorithm '" + cipher.getName() + "'");

        }

      } else {

        // Family does not provide key material, create a random key

        cipher = Encryption.getCipher(conf, cipherName);

        if (cipher == null) {

          throw new RuntimeException("Cipher '" + cipher + "' is not available");

        }

        key = cipher.getRandomKey();

      }

      cryptoContext = Encryption.newContext(conf);

      cryptoContext.setCipher(cipher);

      cryptoContext.setKey(key);

    }

  }

  /**

   * @param family

   * @return TTL in seconds of the specified family

   */

  private static long determineTTLFromFamily(final HColumnDescriptor family) {

    // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.

    long ttl = family.getTimeToLive();

    if (ttl == HConstants.FOREVER) {

      // Default is unlimited ttl.

      ttl = Long.MAX_VALUE;

    } else if (ttl == -1) {

      ttl = Long.MAX_VALUE;

    } else {

      // Second -> ms adjust for user data

      ttl *= 1000;

    }

    return ttl;

  }

  @Override

  public String getColumnFamilyName() {

    return this.family.getNameAsString();

  }

  @Override

  public TableName getTableName() {

    return this.getRegionInfo().getTable();

  }

  @Override

  public FileSystem getFileSystem() {

    return this.fs.getFileSystem();

  }

  public HRegionFileSystem getRegionFileSystem() {

    return this.fs;

  }

  /* Implementation of StoreConfigInformation */

  @Override

  public long getStoreFileTtl() {

    // TTL only applies if there's no MIN_VERSIONs setting on the column.

    return (this.scanInfo.getMinVersions() == 0) ? this.scanInfo.getTtl() : Long.MAX_VALUE;

  }

  @Override

  public long getMemstoreFlushSize() {

    return this.region.memstoreFlushSize;

  }

  @Override

  public long getCompactionCheckMultiplier() {

    return this.compactionCheckMultiplier;

  }

  @Override

  public long getBlockingFileCount() {

    return blockingFileCount;

  }

  /* End implementation of StoreConfigInformation */

  /**

   * Returns the configured bytesPerChecksum value.

   * @param conf The configuration

   * @return The bytesPerChecksum that is set in the configuration

   */

  public static int getBytesPerChecksum(Configuration conf) {

    return conf.getInt(HConstants.BYTES_PER_CHECKSUM,

                       HFile.DEFAULT_BYTES_PER_CHECKSUM);

  }

  /**

   * Returns the configured checksum algorithm.

   * @param conf The configuration

   * @return The checksum algorithm that is set in the configuration

   */

  public static ChecksumType getChecksumType(Configuration conf) {

    String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);

    if (checksumName == null) {

      return HFile.DEFAULT_CHECKSUM_TYPE;

    } else {

      return ChecksumType.nameToType(checksumName);

    }

  }

  /**

   * @return how many bytes to write between status checks

   */

  public static int getCloseCheckInterval() {

    return closeCheckInterval;

  }

  @Override

  public HColumnDescriptor getFamily() {

    return this.family;

  }

  /**

   * @return The maximum sequence id in all store files. Used for log replay.

   */

  long getMaxSequenceId(boolean includeBulkFiles) {

    return StoreFile.getMaxSequenceIdInList(this.getStorefiles(), includeBulkFiles);

  }

  @Override

  public long getMaxMemstoreTS() {

    return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());

  }

  /**

   * @param tabledir {@link Path} to where the table is being stored

   * @param hri {@link HRegionInfo} for the region.

   * @param family {@link HColumnDescriptor} describing the column family

   * @return Path to family/Store home directory.

   */

  @Deprecated

  public static Path getStoreHomedir(final Path tabledir,

      final HRegionInfo hri, final byte[] family) {

    return getStoreHomedir(tabledir, hri.getEncodedName(), family);

  }

  /**

   * @param tabledir {@link Path} to where the table is being stored

   * @param encodedName Encoded region name.

   * @param family {@link HColumnDescriptor} describing the column family

   * @return Path to family/Store home directory.

   */

  @Deprecated

  public static Path getStoreHomedir(final Path tabledir,

      final String encodedName, final byte[] family) {

    return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));

  }

  @Override

  public HFileDataBlockEncoder getDataBlockEncoder() {

    return dataBlockEncoder;

  }

  /**

   * Should be used only in tests.

   * @param blockEncoder the block delta encoder to use

   */

  void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {

    this.dataBlockEncoder = blockEncoder;

  }

  /**

   * Creates an unsorted list of StoreFile loaded in parallel

   * from the given directory.

   * @throws IOException

   */

  private List<StoreFile> loadStoreFiles() throws IOException {

    Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());

    if (files == null || files.size() == 0) {

      return new ArrayList<StoreFile>();

    }

    // initialize the thread pool for opening store files in parallel..

    ThreadPoolExecutor storeFileOpenerThreadPool =

      this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +

          this.getColumnFamilyName());

    CompletionService<StoreFile> completionService =

      new ExecutorCompletionService<StoreFile>(storeFileOpenerThreadPool);

    int totalValidStoreFile = 0;

    for (final StoreFileInfo storeFileInfo: files) {

      // open each store file in parallel

      completionService.submit(new Callable<StoreFile>() {

        @Override

        public StoreFile call() throws IOException {

          StoreFile storeFile = createStoreFileAndReader(storeFileInfo.getPath());

          return storeFile;

        }

      });

      totalValidStoreFile++;

    }

    ArrayList<StoreFile> results = new ArrayList<StoreFile>(files.size());

    IOException ioe = null;

    try {

      for (int i = 0; i < totalValidStoreFile; i++) {

        try {

          Future<StoreFile> future = completionService.take();

          StoreFile storeFile = future.get();

          long length = storeFile.getReader().length();

          this.storeSize += length;

          this.totalUncompressedBytes +=

              storeFile.getReader().getTotalUncompressedBytes();

          if (LOG.isDebugEnabled()) {

            LOG.debug("loaded " + storeFile.toStringDetailed());

          }

          results.add(storeFile);

        } catch (InterruptedException e) {

          if (ioe == null) ioe = new InterruptedIOException(e.getMessage());

        } catch (ExecutionException e) {

          if (ioe == null) ioe = new IOException(e.getCause());

        }

      }

    } finally {

      storeFileOpenerThreadPool.shutdownNow();

    }

    if (ioe != null) {

      // close StoreFile readers

      for (StoreFile file : results) {

        try {

          if (file != null) file.closeReader(true);

        } catch (IOException e) {

          LOG.warn(e.getMessage());

        }

      }

      throw ioe;

    }

    return results;

  }

  private StoreFile createStoreFileAndReader(final Path p) throws IOException {

    StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);

    info.setRegionCoprocessorHost(this.region.getCoprocessorHost());

    StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,

      this.family.getBloomFilterType());

    storeFile.createReader();

    return storeFile;

  }

  @Override

  public long add(final KeyValue kv) {

    lock.readLock().lock();

    try {

      return this.memstore.add(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  @Override

  public long timeOfOldestEdit() {

    return memstore.timeOfOldestEdit();

  }

  /**

   * Adds a value to the memstore

   *

   * @param kv

   * @return memstore size delta

   */

  protected long delete(final KeyValue kv) {

    lock.readLock().lock();

    try {

      return this.memstore.delete(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  @Override

  public void rollback(final KeyValue kv) {

    lock.readLock().lock();

    try {

      this.memstore.rollback(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  /**

   * @return All store files.

   */

  @Override

  public Collection<StoreFile> getStorefiles() {

    return this.storeEngine.getStoreFileManager().getStorefiles();

  }

  @Override

  public void assertBulkLoadHFileOk(Path srcPath) throws IOException {

    HFile.Reader reader  = null;

    try {

      LOG.info("Validating hfile at " + srcPath + " for inclusion in "

          + "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());

      reader = HFile.createReader(srcPath.getFileSystem(conf),

          srcPath, cacheConf, conf);

      reader.loadFileInfo();

      byte[] firstKey = reader.getFirstRowKey();

      Preconditions.checkState(firstKey != null, "First key can not be null");

      byte[] lk = reader.getLastKey();

      Preconditions.checkState(lk != null, "Last key can not be null");

      byte[] lastKey =  KeyValue.createKeyValueFromKey(lk).getRow();

      LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +

          " last=" + Bytes.toStringBinary(lastKey));

      LOG.debug("Region bounds: first=" +

          Bytes.toStringBinary(getRegionInfo().getStartKey()) +

          " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));

      if (!this.getRegionInfo().containsRange(firstKey, lastKey)) {

        throw new WrongRegionException(

            "Bulk load file " + srcPath.toString() + " does not fit inside region "

            + this.getRegionInfo().getRegionNameAsString());

      }

      if (verifyBulkLoads) {

        KeyValue prevKV = null;

        HFileScanner scanner = reader.getScanner(false, false, false);

        scanner.seekTo();

        do {

          KeyValue kv = scanner.getKeyValue();

          if (prevKV != null) {

            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getRowOffset(),

                prevKV.getRowLength(), kv.getBuffer(), kv.getRowOffset(),

                kv.getRowLength()) > 0) {

              throw new InvalidHFileException("Previous row is greater than"

                  + " current row: path=" + srcPath + " previous="

                  + Bytes.toStringBinary(prevKV.getKey()) + " current="

                  + Bytes.toStringBinary(kv.getKey()));

            }

            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getFamilyOffset(),

                prevKV.getFamilyLength(), kv.getBuffer(), kv.getFamilyOffset(),

                kv.getFamilyLength()) != 0) {

              throw new InvalidHFileException("Previous key had different"

                  + " family compared to current key: path=" + srcPath

                  + " previous=" + Bytes.toStringBinary(prevKV.getFamily())

                  + " current=" + Bytes.toStringBinary(kv.getFamily()));

            }

          }

          prevKV = kv;

        } while (scanner.next());

      }

    } finally {

      if (reader != null) reader.close();

    }

  }

  @Override

  public void bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {

    Path srcPath = new Path(srcPathStr);

    Path dstPath = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);

    StoreFile sf = createStoreFileAndReader(dstPath);

    StoreFile.Reader r = sf.getReader();

    this.storeSize += r.length();

    this.totalUncompressedBytes += r.getTotalUncompressedBytes();

    LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() +

        "' as " + dstPath + " - updating store file list.");

    // Append the new storefile into the list

    this.lock.writeLock().lock();

    try {

      this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf));

    } finally {

      // We need the lock, as long as we are updating the storeFiles

      // or changing the memstore. Let us release it before calling

      // notifyChangeReadersObservers. See HBASE-4485 for a possible

      // deadlock scenario that could have happened if continue to hold

      // the lock.

      this.lock.writeLock().unlock();

    }

    notifyChangedReadersObservers();

    LOG.info("Successfully loaded store file " + srcPath

        + " into store " + this + " (new location: " + dstPath + ")");

    if (LOG.isTraceEnabled()) {

      String traceMessage = "BULK LOAD time,size,store size,store files ["

          + EnvironmentEdgeManager.currentTimeMillis() + "," + r.length() + "," + storeSize

          + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";

      LOG.trace(traceMessage);

    }

  }

  @Override

  public ImmutableCollection<StoreFile> close() throws IOException {

    this.lock.writeLock().lock();

    try {

      // Clear so metrics doesn't find them.

      ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();

      if (!result.isEmpty()) {

        // initialize the thread pool for closing store files in parallel.

        ThreadPoolExecutor storeFileCloserThreadPool = this.region

            .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"

                + this.getColumnFamilyName());

        // close each store file in parallel

        CompletionService<Void> completionService =

          new ExecutorCompletionService<Void>(storeFileCloserThreadPool);

        for (final StoreFile f : result) {

          completionService.submit(new Callable<Void>() {

            @Override

            public Void call() throws IOException {

              f.closeReader(true);

              return null;

            }

          });

        }

        IOException ioe = null;

        try {

          for (int i = 0; i < result.size(); i++) {

            try {

              Future<Void> future = completionService.take();

              future.get();

            } catch (InterruptedException e) {

              if (ioe == null) {

                ioe = new InterruptedIOException();

                ioe.initCause(e);

              }

            } catch (ExecutionException e) {

              if (ioe == null) ioe = new IOException(e.getCause());

            }

          }

        } finally {

          storeFileCloserThreadPool.shutdownNow();

        }

        if (ioe != null) throw ioe;

      }

      LOG.info("Closed " + this);

      return result;

    } finally {

      this.lock.writeLock().unlock();

    }

  }

  /**

   * Snapshot this stores memstore. Call before running

   * {@link #flushCache(long, SortedSet, TimeRangeTracker, AtomicLong, MonitoredTask)}

   *  so it has some work to do.

   */

  void snapshot() {

    this.lock.writeLock().lock();

    try {

      this.memstore.snapshot();

    } finally {

      this.lock.writeLock().unlock();

    }

  }

  /**

   * Write out current snapshot.  Presumes {@link #snapshot()} has been called

   * previously.

   * @param logCacheFlushId flush sequence number

   * @param snapshot

   * @param snapshotTimeRangeTracker

   * @param flushedSize The number of bytes flushed

   * @param status

   * @return The path name of the tmp file to which the store was flushed

   * @throws IOException

   */

  protected List<Path> flushCache(final long logCacheFlushId,

      SortedSet<KeyValue> snapshot,

      TimeRangeTracker snapshotTimeRangeTracker,

      AtomicLong flushedSize,

      MonitoredTask status) throws IOException {

    // If an exception happens flushing, we let it out without clearing

    // the memstore snapshot.  The old snapshot will be returned when we say

    // 'snapshot', the next time flush comes around.

    // Retry after catching exception when flushing, otherwise server will abort

    // itself

    StoreFlusher flusher = storeEngine.getStoreFlusher();

    IOException lastException = null;

    for (int i = 0; i < flushRetriesNumber; i++) {

      try {

        List<Path> pathNames = flusher.flushSnapshot(

            snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);

        Path lastPathName = null;

        try {

          for (Path pathName : pathNames) {

            lastPathName = pathName;

            validateStoreFile(pathName);

          }

          return pathNames;

        } catch (Exception e) {

          LOG.warn("Failed validating store file " + lastPathName + ", retrying num=" + i, e);

          if (e instanceof IOException) {

            lastException = (IOException) e;

          } else {

            lastException = new IOException(e);

          }

        }

      } catch (IOException e) {

        LOG.warn("Failed flushing store file, retring num=" + i, e);

        lastException = e;

      }

      if (lastException != null && i < (flushRetriesNumber - 1)) {

        try {

          Thread.sleep(pauseTime);

        } catch (InterruptedException e) {

          IOException iie = new InterruptedIOException();

          iie.initCause(e);

          throw iie;

        }

      }

    }

    throw lastException;

  }

  /*

   * @param path The pathname of the tmp file into which the store was flushed

   * @param logCacheFlushId

   * @return StoreFile created.

   * @throws IOException

   */

  private StoreFile commitFile(final Path path,

      final long logCacheFlushId,

      TimeRangeTracker snapshotTimeRangeTracker,

      AtomicLong flushedSize,

      MonitoredTask status)

      throws IOException {

    // Write-out finished successfully, move into the right spot

    Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);

    status.setStatus("Flushing " + this + ": reopening flushed file");

    StoreFile sf = createStoreFileAndReader(dstPath);

    StoreFile.Reader r = sf.getReader();

    this.storeSize += r.length();

    this.totalUncompressedBytes += r.getTotalUncompressedBytes();

    if (LOG.isInfoEnabled()) {

      LOG.info("Added " + sf + ", entries=" + r.getEntries() +

        ", sequenceid=" + logCacheFlushId +

        ", filesize=" + StringUtils.humanReadableInt(r.length()));

    }

    return sf;

  }

  /*

   * @param maxKeyCount

   * @param compression Compression algorithm to use

   * @param isCompaction whether we are creating a new file in a compaction

   * @param includesMVCCReadPoint - whether to include MVCC or not

   * @param includesTag - includesTag or not

   * @return Writer for a new StoreFile in the tmp dir.

   */

  @Override

  public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,

      boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag)

  throws IOException {

    final CacheConfig writerCacheConf;

    if (isCompaction) {

      // Don't cache data on write on compactions.

      writerCacheConf = new CacheConfig(cacheConf);

      writerCacheConf.setCacheDataOnWrite(false);

    } else {

      writerCacheConf = cacheConf;

    }

    InetSocketAddress[] favoredNodes = null;

    if (region.getRegionServerServices() != null) {

      favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(

          region.getRegionInfo().getEncodedName());

    }

    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,

      cryptoContext);

    StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,

        this.getFileSystem())

            .withFilePath(fs.createTempName())

            .withComparator(comparator)

            .withBloomType(family.getBloomFilterType())

            .withMaxKeyCount(maxKeyCount)

            .withFavoredNodes(favoredNodes)

            .withFileContext(hFileContext)

            .build();

    return w;

  }

  private HFileContext createFileContext(Compression.Algorithm compression,

      boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {

    if (compression == null) {

      compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;

    }

    if(family.shouldCompressTags()) {

      LOG.warn("HFile tag compression attribute ignored for '" + family.getNameAsString()

          + "', see HBASE-10443.");

    }

    HFileContext hFileContext = new HFileContextBuilder()

                                .withIncludesMvcc(includeMVCCReadpoint)

                                .withIncludesTags(includesTag)

                                .withCompression(compression)

                                .withCompressTags(false)

                                .withChecksumType(checksumType)

                                .withBytesPerCheckSum(bytesPerChecksum)

                                .withBlockSize(blocksize)

                                .withHBaseCheckSum(true)

                                .withDataBlockEncoding(family.getDataBlockEncoding())

                                .withEncryptionContext(cryptoContext)

                                .build();

    return hFileContext;

  }

  /*

   * Change storeFiles adding into place the Reader produced by this new flush.

   * @param sfs Store files

   * @param set That was used to make the passed file.

   * @throws IOException

   * @return Whether compaction is required.

   */

  private boolean updateStorefiles(

      final List<StoreFile> sfs, final SortedSet<KeyValue> set) throws IOException {

    this.lock.writeLock().lock();

    try {

      this.storeEngine.getStoreFileManager().insertNewFiles(sfs);

      this.memstore.clearSnapshot(set);

    } finally {

      // We need the lock, as long as we are updating the storeFiles

      // or changing the memstore. Let us release it before calling

      // notifyChangeReadersObservers. See HBASE-4485 for a possible

      // deadlock scenario that could have happened if continue to hold

      // the lock.

      this.lock.writeLock().unlock();

    }

    // Tell listeners of the change in readers.

    notifyChangedReadersObservers();

    if (LOG.isTraceEnabled()) {

      long totalSize = 0;

      for (StoreFile sf : sfs) {

        totalSize += sf.getReader().length();

      }

      String traceMessage = "FLUSH time,count,size,store size,store files ["

          + EnvironmentEdgeManager.currentTimeMillis() + "," + sfs.size() + "," + totalSize

          + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";

      LOG.trace(traceMessage);

    }

    return needsCompaction();

  }

  /*

   * Notify all observers that set of Readers has changed.

   * @throws IOException

   */

  private void notifyChangedReadersObservers() throws IOException {

    for (ChangedReadersObserver o: this.changedReaderObservers) {

      o.updateReaders();

    }

  }

  /**

   * Get all scanners with no filtering based on TTL (that happens further down

   * the line).

   * @return all scanners for this store

   */

  @Override

  public List<KeyValueScanner> getScanners(boolean cacheBlocks, boolean isGet,

      boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,

      byte[] stopRow, long readPt) throws IOException {

    Collection<StoreFile> storeFilesToScan;

    List<KeyValueScanner> memStoreScanners;

    this.lock.readLock().lock();

    try {

      storeFilesToScan =

          this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow);

      memStoreScanners = this.memstore.getScanners(readPt);

    } finally {

      this.lock.readLock().unlock();

    }

    // First the store file scanners

    // TODO this used to get the store files in descending order,

    // but now we get them in ascending order, which I think is

    // actually more correct, since memstore get put at the end.

    List<StoreFileScanner> sfScanners = StoreFileScanner

      .getScannersForStoreFiles(storeFilesToScan, cacheBlocks, usePread, isCompaction, matcher,

        readPt);

    List<KeyValueScanner> scanners =

      new ArrayList<KeyValueScanner>(sfScanners.size()+1);

    scanners.addAll(sfScanners);

    // Then the memstore scanners

    scanners.addAll(memStoreScanners);

    return scanners;

  }

  @Override

  public void addChangedReaderObserver(ChangedReadersObserver o) {

    this.changedReaderObservers.add(o);

  }

  @Override

  public void deleteChangedReaderObserver(ChangedReadersObserver o) {

    // We don't check if observer present; it may not be (legitimately)

    this.changedReaderObservers.remove(o);

  }

  //////////////////////////////////////////////////////////////////////////////

  // Compaction

  //////////////////////////////////////////////////////////////////////////////

  /**

   * Compact the StoreFiles.  This method may take some time, so the calling

   * thread must be able to block for long periods.

   *

   * <p>During this time, the Store can work as usual, getting values from

   * StoreFiles and writing new StoreFiles from the memstore.

   *

   * Existing StoreFiles are not destroyed until the new compacted StoreFile is

   * completely written-out to disk.

   *

   * <p>The compactLock prevents multiple simultaneous compactions.

   * The structureLock prevents us from interfering with other write operations.

   *

   * <p>We don't want to hold the structureLock for the whole time, as a compact()

   * can be lengthy and we want to allow cache-flushes during this period.

   *

   * <p> Compaction event should be idempotent, since there is no IO Fencing for

   * the region directory in hdfs. A region server might still try to complete the

   * compaction after it lost the region. That is why the following events are carefully

   * ordered for a compaction:

   *  1. Compaction writes new files under region/.tmp directory (compaction output)

   *  2. Compaction atomically moves the temporary file under region directory

   *  3. Compaction appends a WAL edit containing the compaction input and output files.

   *  Forces sync on WAL.

   *  4. Compaction deletes the input files from the region directory.

   *

   * Failure conditions are handled like this:

   *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes

   *  the compaction later, it will only write the new data file to the region directory.

   *  Since we already have this data, this will be idempotent but we will have a redundant

   *  copy of the data.

   *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The

   *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.

   *  - If RS fails after 3, the region region server who opens the region will pick up the

   *  the compaction marker from the WAL and replay it by removing the compaction input files.

   *  Failed RS can also attempt to delete those files, but the operation will be idempotent

   *

   * See HBASE-2231 for details.

   *

   * @param compaction compaction details obtained from requestCompaction()

   * @throws IOException

   * @return Storefile we compacted into or null if we failed or opted out early.

   */

  @Override

  public List<StoreFile> compact(CompactionContext compaction) throws IOException {

    assert compaction != null && compaction.hasSelection();

    CompactionRequest cr = compaction.getRequest();

    Collection<StoreFile> filesToCompact = cr.getFiles();

    assert !filesToCompact.isEmpty();

    synchronized (filesCompacting) {

      // sanity check: we're compacting files that this store knows about

      // TODO: change this to LOG.error() after more debugging

      Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));

    }

    // Ready to go. Have list of files to compact.

    LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "

        + this + " of " + this.getRegionInfo().getRegionNameAsString()

        + " into tmpdir=" + fs.getTempDir() + ", totalSize="

        + StringUtils.humanReadableInt(cr.getSize()));

    long compactionStartTime = EnvironmentEdgeManager.currentTimeMillis();

    List<StoreFile> sfs = null;

    try {

      // Commence the compaction.

      List<Path> newFiles = compaction.compact();

      // TODO: get rid of this!

      if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {

        LOG.warn("hbase.hstore.compaction.complete is set to false");

        sfs = new ArrayList<StoreFile>();

        for (Path newFile : newFiles) {