* Copyright 2010 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A Store holds a column family in a Region.  Its a memstore and a set of zero

* or more StoreFiles, which stretch backwards over time.

*

* <p>There's no reason to consider append-logging at this level; all logging

* and locking is handled at the HRegion level.  Store just provides

* services to manage sets of StoreFiles.  One of the most important of those

* services is compaction services where files are aggregated once they pass

* a configurable threshold.

*

* <p>The only thing having to do with logs that Store needs to deal with is

* the reconstructionLog.  This is a segment of an HRegion's log that might

* NOT be present upon startup.  If the param is NULL, there's nothing to do.

* If the param is non-NULL, we need to process the log to reconstruct

* a TreeMap that might not have been written to disk before the process

* died.

*

* <p>It's assumed that after this constructor returns, the reconstructionLog

* file will be deleted (by whoever has instantiated the Store).

*

* <p>Locking and transactions are handled at a higher level.  This API should

* not be called directly but by an HRegion manager.

*/

  static final Log LOG = LogFactory.getLog(Store.class);

  protected final MemStore memstore;

  // This stores directory in the filesystem.

  private final Path homedir;

  private final HRegion region;

  private final HColumnDescriptor family;

  final FileSystem fs;

  final Configuration conf;

  final CacheConfig cacheConf;

  // ttl in milliseconds.

  private long ttl;

  private final int minFilesToCompact;

  private final int maxFilesToCompact;

  private final long minCompactSize;

  private final long maxCompactSize;

  private long lastCompactSize = 0;

  volatile boolean forceMajor = false;

  /* how many bytes to write between status checks */

  static int closeCheckInterval = 0;

  private final int blockingStoreFileCount;

  private volatile long storeSize = 0L;

  private volatile long totalUncompressedBytes = 0L;

  private final Object flushLock = new Object();

  final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

  private final String storeNameStr;

  private CompactionProgress progress;

  private final int compactionKVMax;

  private final boolean verifyBulkLoads;

  /* The default priority for user-specified compaction requests.

   * The user gets top priority unless we have blocking compactions. (Pri <= 0)

   */

  public static final int PRIORITY_USER = 1;

  public static final int NO_PRIORITY = Integer.MIN_VALUE;

  // not private for testing

  /* package */ScanInfo scanInfo;

  /*

   * List of store files inside this store. This is an immutable list that

   * is atomically replaced when its contents change.

   */

  private ImmutableList<StoreFile> storefiles = null;

  List<StoreFile> filesCompacting = Lists.newArrayList();

  // All access must be synchronized.

  private final CopyOnWriteArraySet<ChangedReadersObserver> changedReaderObservers =

    new CopyOnWriteArraySet<ChangedReadersObserver>();

  private final int blocksize;

  /** Compression algorithm for flush files and minor compaction */

  private final Compression.Algorithm compression;

  /** Compression algorithm for major compaction */

  private final Compression.Algorithm compactionCompression;

  private HFileDataBlockEncoder dataBlockEncoder;

  /** Checksum configuration */

  private ChecksumType checksumType;

  private int bytesPerChecksum;

  // Comparing KeyValues

  final KeyValue.KVComparator comparator;

  /**

   * Constructor

   * @param basedir qualified path under which the region directory lives;

   * generally the table subdirectory

   * @param region

   * @param family HColumnDescriptor for this column

   * @param fs file system object

   * @param confParam configuration object

   * failed.  Can be null.

   * @throws IOException

   */

  protected Store(Path basedir, HRegion region, HColumnDescriptor family,

    FileSystem fs, Configuration conf)

  throws IOException {

    super(conf, region.getTableDesc().getNameAsString(),

        Bytes.toString(family.getName()));

    HRegionInfo info = region.regionInfo;

    this.fs = fs;

    this.homedir = getStoreHomedir(basedir, info.getEncodedName(), family.getName());

    if (!this.fs.exists(this.homedir)) {

      if (!this.fs.mkdirs(this.homedir))

        throw new IOException("Failed create of: " + this.homedir.toString());

    }

    this.region = region;

    this.family = family;

    this.conf = conf;

    this.blocksize = family.getBlocksize();

    this.compression = family.getCompression();

    // avoid overriding compression setting for major compactions if the user

    // has not specified it separately

    this.compactionCompression =

      (family.getCompactionCompression() != Compression.Algorithm.NONE) ?

        family.getCompactionCompression() : this.compression;

    this.dataBlockEncoder =

        new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),

            family.getDataBlockEncoding());

    this.comparator = info.getComparator();

    // getTimeToLive returns ttl in seconds.  Convert to milliseconds.

    this.ttl = family.getTimeToLive();

    if (ttl == HConstants.FOREVER) {

      // default is unlimited ttl.

      ttl = Long.MAX_VALUE;

    } else if (ttl == -1) {

      ttl = Long.MAX_VALUE;

    } else {

      // second -> ms adjust for user data

      this.ttl *= 1000;

    }

    // used by ScanQueryMatcher

    long timeToPurgeDeletes =

        Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);

    LOG.info("time to purge deletes set to " + timeToPurgeDeletes +

        "ms in store " + this);

    scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),

        family.getMaxVersions(), ttl, family.getKeepDeletedCells(),

        timeToPurgeDeletes, this.comparator);

    this.memstore = new MemStore(conf, this.comparator);

    this.storeNameStr = getColumnFamilyName();

    // By default, compact if storefile.count >= minFilesToCompact

    this.minFilesToCompact = Math.max(2,

      conf.getInt("hbase.hstore.compaction.min",

        /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));

    // Setting up cache configuration for this family

    this.cacheConf = new CacheConfig(conf, family);

    this.blockingStoreFileCount =

      conf.getInt("hbase.hstore.blockingStoreFiles", 7);

    this.maxFilesToCompact = conf.getInt("hbase.hstore.compaction.max", 10);

    this.minCompactSize = conf.getLong("hbase.hstore.compaction.min.size",

      this.region.memstoreFlushSize);

    this.maxCompactSize

      = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);

    this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);

    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify",

        false);

    if (Store.closeCheckInterval == 0) {

      Store.closeCheckInterval = conf.getInt(

          "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);

    }

    this.storefiles = sortAndClone(loadStoreFiles());

    // Initialize checksum type from name. The names are CRC32, CRC32C, etc.

    this.checksumType = getChecksumType(conf);

    // initilize bytes per checksum

    this.bytesPerChecksum = getBytesPerChecksum(conf);

  }

  /**

   * Returns the configured bytesPerChecksum value.

   * @param conf The configuration

   * @return The bytesPerChecksum that is set in the configuration

   */

  public static int getBytesPerChecksum(Configuration conf) {

    return conf.getInt(HConstants.BYTES_PER_CHECKSUM,

                       HFile.DEFAULT_BYTES_PER_CHECKSUM);

  }

  /**

   * Returns the configured checksum algorithm.

   * @param conf The configuration

   * @return The checksum algorithm that is set in the configuration

   */

  public static ChecksumType getChecksumType(Configuration conf) {

    String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);

    if (checksumName == null) {

      return HFile.DEFAULT_CHECKSUM_TYPE;

    } else {

      return ChecksumType.nameToType(checksumName);

    }

  }

  public HColumnDescriptor getFamily() {

    return this.family;

  }

  /**

   * @return The maximum sequence id in all store files.

   */

  long getMaxSequenceId() {

    return StoreFile.getMaxSequenceIdInList(this.getStorefiles());

  }

  /**

   * @return The maximum memstoreTS in all store files.

   */

  public long getMaxMemstoreTS() {

    return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());

  }

  /**

   * @param tabledir

   * @param encodedName Encoded region name.

   * @param family

   * @return Path to family/Store home directory.

   */

  public static Path getStoreHomedir(final Path tabledir,

      final String encodedName, final byte [] family) {

    return new Path(tabledir, new Path(encodedName,

      new Path(Bytes.toString(family))));

  }

  /**

   * Return the directory in which this store stores its

   * StoreFiles

   */

  public Path getHomedir() {

    return homedir;

  }

  /**

   * @return the data block encoder

   */

  public HFileDataBlockEncoder getDataBlockEncoder() {

    return dataBlockEncoder;

  }

  /**

   * Should be used only in tests.

   * @param blockEncoder the block delta encoder to use

   */

  void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {

    this.dataBlockEncoder = blockEncoder;

  }

  /**

   * Creates an unsorted list of StoreFile loaded in parallel

   * from the given directory.

   * @throws IOException

   */

  private List<StoreFile> loadStoreFiles() throws IOException {

    ArrayList<StoreFile> results = new ArrayList<StoreFile>();

    FileStatus files[] = FSUtils.listStatus(this.fs, this.homedir, null);

    if (files == null || files.length == 0) {

      return results;

    }

    // initialize the thread pool for opening store files in parallel..

    ThreadPoolExecutor storeFileOpenerThreadPool =

      this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +

          this.family.getNameAsString());

    CompletionService<StoreFile> completionService =

      new ExecutorCompletionService<StoreFile>(storeFileOpenerThreadPool);

    int totalValidStoreFile = 0;

    for (int i = 0; i < files.length; i++) {

      // Skip directories.

      if (files[i].isDir()) {

        continue;

      }

      final Path p = files[i].getPath();

      // Check for empty file. Should never be the case but can happen

      // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646

      if (this.fs.getFileStatus(p).getLen() <= 0) {

        LOG.warn("Skipping " + p + " because its empty. HBASE-646 DATA LOSS?");

        continue;

      }

      // open each store file in parallel

      completionService.submit(new Callable<StoreFile>() {

        public StoreFile call() throws IOException {

          StoreFile storeFile = new StoreFile(fs, p, conf, cacheConf,

              family.getBloomFilterType(), dataBlockEncoder);

          passSchemaMetricsTo(storeFile);

          storeFile.createReader();

          return storeFile;

        }

      });

      totalValidStoreFile++;

    }

    try {

      for (int i = 0; i < totalValidStoreFile; i++) {

        Future<StoreFile> future = completionService.take();

        StoreFile storeFile = future.get();

        long length = storeFile.getReader().length();

        this.storeSize += length;

        this.totalUncompressedBytes +=

          storeFile.getReader().getTotalUncompressedBytes();

        if (LOG.isDebugEnabled()) {

          LOG.debug("loaded " + storeFile.toStringDetailed());

        }

        results.add(storeFile);

      }

    } catch (InterruptedException e) {

      throw new IOException(e);

    } catch (ExecutionException e) {

      throw new IOException(e.getCause());

    } finally {

      storeFileOpenerThreadPool.shutdownNow();

    }

    return results;

  }

  /**

   * Adds a value to the memstore

   *

   * @param kv

   * @return memstore size delta

   */

  protected long add(final KeyValue kv) {

    lock.readLock().lock();

    try {

      return this.memstore.add(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  /**

   * Adds a value to the memstore

   *

   * @param kv

   * @return memstore size delta

   */

  protected long delete(final KeyValue kv) {

    lock.readLock().lock();

    try {

      return this.memstore.delete(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  /**

   * Removes a kv from the memstore. The KeyValue is removed only

   * if its key & memstoreTS matches the key & memstoreTS value of the

   * kv parameter.

   *

   * @param kv

   */

  protected void rollback(final KeyValue kv) {

    lock.readLock().lock();

    try {

      this.memstore.rollback(kv);

    } finally {

      lock.readLock().unlock();

    }

  }

  /**

   * @return All store files.

   */

  List<StoreFile> getStorefiles() {

    return this.storefiles;

  }

  /**

   * This throws a WrongRegionException if the HFile does not fit in this

   * region, or an InvalidHFileException if the HFile is not valid.

   */

  void assertBulkLoadHFileOk(Path srcPath) throws IOException {

    HFile.Reader reader  = null;

    try {

      LOG.info("Validating hfile at " + srcPath + " for inclusion in "

          + "store " + this + " region " + this.region);

      reader = HFile.createReader(srcPath.getFileSystem(conf),

          srcPath, cacheConf);

      reader.loadFileInfo();

      byte[] firstKey = reader.getFirstRowKey();

      byte[] lk = reader.getLastKey();

      byte[] lastKey =

          (lk == null) ? null :

              KeyValue.createKeyValueFromKey(lk).getRow();

      LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +

          " last=" + Bytes.toStringBinary(lastKey));

      LOG.debug("Region bounds: first=" +

          Bytes.toStringBinary(region.getStartKey()) +

          " last=" + Bytes.toStringBinary(region.getEndKey()));

      HRegionInfo hri = region.getRegionInfo();

      if (!hri.containsRange(firstKey, lastKey)) {

        throw new WrongRegionException(

            "Bulk load file " + srcPath.toString() + " does not fit inside region "

            + this.region);

      }

      if (verifyBulkLoads) {

        KeyValue prevKV = null;

        HFileScanner scanner = reader.getScanner(false, false, false);

        scanner.seekTo();

        do {

          KeyValue kv = scanner.getKeyValue();

          if (prevKV != null) {

            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getRowOffset(),

                prevKV.getRowLength(), kv.getBuffer(), kv.getRowOffset(),

                kv.getRowLength()) > 0) {

              throw new InvalidHFileException("Previous row is greater than"

                  + " current row: path=" + srcPath + " previous="

                  + Bytes.toStringBinary(prevKV.getKey()) + " current="

                  + Bytes.toStringBinary(kv.getKey()));

            }

            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getFamilyOffset(),

                prevKV.getFamilyLength(), kv.getBuffer(), kv.getFamilyOffset(),

                kv.getFamilyLength()) != 0) {

              throw new InvalidHFileException("Previous key had different"

                  + " family compared to current key: path=" + srcPath

                  + " previous=" + Bytes.toStringBinary(prevKV.getFamily())

                  + " current=" + Bytes.toStringBinary(kv.getFamily()));

            }

          }

          prevKV = kv;

        } while (scanner.next());

      }

    } finally {

      if (reader != null) reader.close();

    }

  }

  /**

   * This method should only be called from HRegion.  It is assumed that the

   * ranges of values in the HFile fit within the stores assigned region.

   * (assertBulkLoadHFileOk checks this)

   */

  void bulkLoadHFile(String srcPathStr) throws IOException {

    Path srcPath = new Path(srcPathStr);

    // Move the file if it's on another filesystem

    FileSystem srcFs = srcPath.getFileSystem(conf);

    if (!srcFs.equals(fs)) {

      LOG.info("File " + srcPath + " on different filesystem than " +

          "destination store - moving to this filesystem.");

      Path tmpPath = getTmpPath();

      FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);

      LOG.info("Copied to temporary path on dst filesystem: " + tmpPath);

      srcPath = tmpPath;

    }

    Path dstPath = StoreFile.getRandomFilename(fs, homedir);

    LOG.debug("Renaming bulk load file " + srcPath + " to " + dstPath);

    StoreFile.rename(fs, srcPath, dstPath);

    StoreFile sf = new StoreFile(fs, dstPath, this.conf, this.cacheConf,

        this.family.getBloomFilterType(), this.dataBlockEncoder);

    passSchemaMetricsTo(sf);

    StoreFile.Reader r = sf.createReader();

    this.storeSize += r.length();

    this.totalUncompressedBytes += r.getTotalUncompressedBytes();

   

    LOG.info("Moved hfile " + srcPath + " into store directory " +

        homedir + " - updating store file list.");

    // Append the new storefile into the list

    this.lock.writeLock().lock();

    try {

      ArrayList<StoreFile> newFiles = new ArrayList<StoreFile>(storefiles);

      newFiles.add(sf);

      this.storefiles = sortAndClone(newFiles);

    } finally {

      // We need the lock, as long as we are updating the storefiles

      // or changing the memstore. Let us release it before calling

      // notifyChangeReadersObservers. See HBASE-4485 for a possible

      // deadlock scenario that could have happened if continue to hold

      // the lock.

      this.lock.writeLock().unlock();

    }

    notifyChangedReadersObservers();

    LOG.info("Successfully loaded store file " + srcPath

        + " into store " + this + " (new location: " + dstPath + ")");

  }

  /**

   * Get a temporary path in this region. These temporary files

   * will get cleaned up when the region is re-opened if they are

   * still around.

   */

  private Path getTmpPath() throws IOException {

    return StoreFile.getRandomFilename(

        fs, region.getTmpDir());

  }

  /**

   * Close all the readers

   *

   * We don't need to worry about subsequent requests because the HRegion holds

   * a write lock that will prevent any more reads or writes.

   *

   * @throws IOException

   */

  ImmutableList<StoreFile> close() throws IOException {

    this.lock.writeLock().lock();

    try {

      ImmutableList<StoreFile> result = storefiles;

      // Clear so metrics doesn't find them.

      storefiles = ImmutableList.of();

      if (!result.isEmpty()) {

        // initialize the thread pool for closing store files in parallel.

        ThreadPoolExecutor storeFileCloserThreadPool = this.region

            .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"

                + this.family.getNameAsString());

        // close each store file in parallel

        CompletionService<Void> completionService =

          new ExecutorCompletionService<Void>(storeFileCloserThreadPool);

        for (final StoreFile f : result) {

          completionService.submit(new Callable<Void>() {

            public Void call() throws IOException {

              f.closeReader(true);

              return null;

            }

          });

        }

        try {

          for (int i = 0; i < result.size(); i++) {

            Future<Void> future = completionService.take();

            future.get();

          }

        } catch (InterruptedException e) {

          throw new IOException(e);

        } catch (ExecutionException e) {

          throw new IOException(e.getCause());

        } finally {

          storeFileCloserThreadPool.shutdownNow();

        }

      }

      LOG.debug("closed " + this.storeNameStr);

      return result;

    } finally {

      this.lock.writeLock().unlock();

    }

  }

  /**

   * Snapshot this stores memstore.  Call before running

   * {@link #flushCache(long, SortedSet<KeyValue>)} so it has some work to do.

   */

  void snapshot() {

    this.memstore.snapshot();

  }

  /**

   * Write out current snapshot.  Presumes {@link #snapshot()} has been called

   * previously.

   * @param logCacheFlushId flush sequence number

   * @param snapshot

   * @param snapshotTimeRangeTracker

   * @param flushedSize The number of bytes flushed

   * @param status

   * @return Path The path name of the tmp file to which the store was flushed

   * @throws IOException

   */

  private Path flushCache(final long logCacheFlushId,

      SortedSet<KeyValue> snapshot,

      TimeRangeTracker snapshotTimeRangeTracker,

      AtomicLong flushedSize,

      MonitoredTask status) throws IOException {

    // If an exception happens flushing, we let it out without clearing

    // the memstore snapshot.  The old snapshot will be returned when we say

    // 'snapshot', the next time flush comes around.

    return internalFlushCache(

        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);

  }

  /*

   * @param cache

   * @param logCacheFlushId

   * @param snapshotTimeRangeTracker

   * @param flushedSize The number of bytes flushed

   * @return Path The path name of the tmp file to which the store was flushed

   * @throws IOException

   */

  private Path internalFlushCache(final SortedSet<KeyValue> set,

      final long logCacheFlushId,

      TimeRangeTracker snapshotTimeRangeTracker,

      AtomicLong flushedSize,

      MonitoredTask status)

      throws IOException {

    StoreFile.Writer writer;

    // Find the smallest read point across all the Scanners.

    long smallestReadPoint = region.getSmallestReadPoint();

    long flushed = 0;

    Path pathName;

    // Don't flush if there are no entries.

    if (set.size() == 0) {

      return null;

    }

    Scan scan = new Scan();

    scan.setMaxVersions(scanInfo.getMaxVersions());

    // Use a store scanner to find which rows to flush.

    // Note that we need to retain deletes, hence

    // treat this as a minor compaction.

    InternalScanner scanner = new StoreScanner(this, scan, Collections

        .singletonList(new CollectionBackedScanner(set, this.comparator)),

        ScanType.MINOR_COMPACT, this.region.getSmallestReadPoint(),

        HConstants.OLDEST_TIMESTAMP);

    try {

      // TODO:  We can fail in the below block before we complete adding this

      // flush to list of store files.  Add cleanup of anything put on filesystem

      // if we fail.

      synchronized (flushLock) {

        status.setStatus("Flushing " + this + ": creating writer");

        // A. Write the map out to the disk

        writer = createWriterInTmp(set.size());

        writer.setTimeRangeTracker(snapshotTimeRangeTracker);

        pathName = writer.getPath();

        try {

          List<KeyValue> kvs = new ArrayList<KeyValue>();

          boolean hasMore;

          do {

            hasMore = scanner.next(kvs);

            if (!kvs.isEmpty()) {

              for (KeyValue kv : kvs) {

                // If we know that this KV is going to be included always, then let us

                // set its memstoreTS to 0. This will help us save space when writing to disk.

                if (kv.getMemstoreTS() <= smallestReadPoint) {

                  // let us not change the original KV. It could be in the memstore

                  // changing its memstoreTS could affect other threads/scanners.

                  kv = kv.shallowCopy();

                  kv.setMemstoreTS(0);

                }

                writer.append(kv);

                flushed += this.memstore.heapSizeChange(kv, true);

              }

              kvs.clear();

            }

          } while (hasMore);

        } finally {

          // Write out the log sequence number that corresponds to this output

          // hfile.  The hfile is current up to and including logCacheFlushId.

          status.setStatus("Flushing " + this + ": appending metadata");

          writer.appendMetadata(logCacheFlushId, false);

          status.setStatus("Flushing " + this + ": closing flushed file");

          writer.close();

        }

      }

    } finally {

      flushedSize.set(flushed);

      scanner.close();

    }

    if (LOG.isInfoEnabled()) {

      LOG.info("Flushed " +

               ", sequenceid=" + logCacheFlushId +

               ", memsize=" + StringUtils.humanReadableInt(flushed) +

               ", into tmp file " + pathName);

    }

    return pathName;

  }

  /*

   * @param path The pathname of the tmp file into which the store was flushed

   * @param logCacheFlushId

   * @return StoreFile created.

   * @throws IOException

   */

  private StoreFile commitFile(final Path path,

      final long logCacheFlushId,

      TimeRangeTracker snapshotTimeRangeTracker,

      AtomicLong flushedSize,

      MonitoredTask status)

      throws IOException {

    // Write-out finished successfully, move into the right spot

    String fileName = path.getName();

    Path dstPath = new Path(homedir, fileName);

    validateStoreFile(path);

    String msg = "Renaming flushed file at " + path + " to " + dstPath;

    LOG.debug(msg);

    status.setStatus("Flushing " + this + ": " + msg);

    if (!fs.rename(path, dstPath)) {

      LOG.warn("Unable to rename " + path + " to " + dstPath);

    }

    status.setStatus("Flushing " + this + ": reopening flushed file");

    StoreFile sf = new StoreFile(this.fs, dstPath, this.conf, this.cacheConf,

        this.family.getBloomFilterType(), this.dataBlockEncoder);

    passSchemaMetricsTo(sf);

    StoreFile.Reader r = sf.createReader();

    this.storeSize += r.length();

    this.totalUncompressedBytes += r.getTotalUncompressedBytes();

    // This increments the metrics associated with total flushed bytes for this

    // family. The overall flush count is stored in the static metrics and

    // retrieved from HRegion.recentFlushes, which is set within

    // HRegion.internalFlushcache, which indirectly calls this to actually do

    // the flushing through the StoreFlusherImpl class

    getSchemaMetrics().updatePersistentStoreMetric(

        SchemaMetrics.StoreMetricType.FLUSH_SIZE, flushedSize.longValue());

    if (LOG.isInfoEnabled()) {

      LOG.info("Added " + sf + ", entries=" + r.getEntries() +

        ", sequenceid=" + logCacheFlushId +

        ", filesize=" + StringUtils.humanReadableInt(r.length()));

    }

    return sf;

  }

  /*

   * @param maxKeyCount

   * @return Writer for a new StoreFile in the tmp dir.

   */

  private StoreFile.Writer createWriterInTmp(int maxKeyCount)

  throws IOException {

    return createWriterInTmp(maxKeyCount, this.compression, false);

  }

  /*

   * @param maxKeyCount

   * @param compression Compression algorithm to use

   * @param isCompaction whether we are creating a new file in a compaction

   * @return Writer for a new StoreFile in the tmp dir.

   */

  public StoreFile.Writer createWriterInTmp(int maxKeyCount,

    Compression.Algorithm compression, boolean isCompaction)

  throws IOException {

    final CacheConfig writerCacheConf;

    if (isCompaction) {

      // Don't cache data on write on compactions.

      writerCacheConf = new CacheConfig(cacheConf);

      writerCacheConf.setCacheDataOnWrite(false);

    } else {

      writerCacheConf = cacheConf;

    }

    StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,

        fs, blocksize)

            .withOutputDir(region.getTmpDir())

            .withDataBlockEncoder(dataBlockEncoder)

            .withComparator(comparator)

            .withBloomType(family.getBloomFilterType())

            .withMaxKeyCount(maxKeyCount)

            .withChecksumType(checksumType)

            .withBytesPerChecksum(bytesPerChecksum)

            .withCompression(compression)

            .build();

    // The store file writer's path does not include the CF name, so we need

    // to configure the HFile writer directly.

    SchemaConfigured sc = (SchemaConfigured) w.writer;

    SchemaConfigured.resetSchemaMetricsConf(sc);

    passSchemaMetricsTo(sc);

    return w;

  }

  /*

   * Change storefiles adding into place the Reader produced by this new flush.

   * @param sf

   * @param set That was used to make the passed file <code>p</code>.

   * @throws IOException

   * @return Whether compaction is required.

   */

  private boolean updateStorefiles(final StoreFile sf,

                                   final SortedSet<KeyValue> set)

  throws IOException {

    this.lock.writeLock().lock();

    try {

      ArrayList<StoreFile> newList = new ArrayList<StoreFile>(storefiles);

      newList.add(sf);

      storefiles = sortAndClone(newList);

      this.memstore.clearSnapshot(set);

    } finally {

      // We need the lock, as long as we are updating the storefiles

      // or changing the memstore. Let us release it before calling

      // notifyChangeReadersObservers. See HBASE-4485 for a possible

      // deadlock scenario that could have happened if continue to hold

      // the lock.

      this.lock.writeLock().unlock();

    }

    // Tell listeners of the change in readers.

    notifyChangedReadersObservers();

    return needsCompaction();

  }

  /*

   * Notify all observers that set of Readers has changed.

   * @throws IOException

   */

  private void notifyChangedReadersObservers() throws IOException {

    for (ChangedReadersObserver o: this.changedReaderObservers) {

      o.updateReaders();

    }

  }

  /**

   * Get all scanners with no filtering based on TTL (that happens further down

   * the line).

   * @return all scanners for this store

   */

  protected List<KeyValueScanner> getScanners(boolean cacheBlocks,

      boolean isGet,

      boolean isCompaction,

      ScanQueryMatcher matcher) throws IOException {

    List<StoreFile> storeFiles;

    List<KeyValueScanner> memStoreScanners;

    this.lock.readLock().lock();

    try {

      storeFiles = this.getStorefiles();

      memStoreScanners = this.memstore.getScanners();

    } finally {

      this.lock.readLock().unlock();

    }

    // First the store file scanners

    // TODO this used to get the store files in descending order,

    // but now we get them in ascending order, which I think is

    // actually more correct, since memstore get put at the end.

    List<StoreFileScanner> sfScanners = StoreFileScanner

      .getScannersForStoreFiles(storeFiles, cacheBlocks, isGet, isCompaction, matcher);

    List<KeyValueScanner> scanners =

      new ArrayList<KeyValueScanner>(sfScanners.size()+1);