* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

    private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);

    private static final ExecutorService flushExecutor = new JMXEnabledThreadPoolExecutor(DatabaseDescriptor.getFlushWriters(),

                                                                                          StageManager.KEEPALIVE,

                                                                                          TimeUnit.SECONDS,

                                                                                          new LinkedBlockingQueue<Runnable>(),

                                                                                          new NamedThreadFactory("MemtableFlushWriter"),

                                                                                          "internal");

    // post-flush executor is single threaded to provide guarantee that any flush Future on a CF will never return until prior flushes have completed

    public static final ExecutorService postFlushExecutor = new JMXEnabledThreadPoolExecutor(1,

                                                                                             StageManager.KEEPALIVE,

                                                                                             TimeUnit.SECONDS,

                                                                                             new LinkedBlockingQueue<Runnable>(),

                                                                                             new NamedThreadFactory("MemtablePostFlush"),

                                                                                             "internal");

    public static final ExecutorService reclaimExecutor = new JMXEnabledThreadPoolExecutor(1, StageManager.KEEPALIVE,

                                                                                           TimeUnit.SECONDS,

                                                                                           new LinkedBlockingQueue<Runnable>(),

                                                                                           new NamedThreadFactory("MemtableReclaimMemory"),

                                                                                           "internal");

    public final Keyspace keyspace;

    public final String name;

    public final CFMetaData metadata;

    public final IPartitioner partitioner;

    private final String mbeanName;

    private volatile boolean valid = true;

    /**

     * Memtables and SSTables on disk for this column family.

     *

     * We synchronize on the DataTracker to ensure isolation when we want to make sure

     * that the memtable we're acting on doesn't change out from under us.  I.e., flush

     * syncronizes on it to make sure it can submit on both executors atomically,

     * so anyone else who wants to make sure flush doesn't interfere should as well.

     */

    private final DataTracker data;

    /* The read order, used to track accesses to off-heap memtable storage */

    public final OpOrder readOrdering = new OpOrder();

    /* This is used to generate the next index for a SSTable */

    private final AtomicInteger fileIndexGenerator = new AtomicInteger(0);

    public final SecondaryIndexManager indexManager;

    /* These are locally held copies to be changed from the config during runtime */

    private volatile DefaultInteger minCompactionThreshold;

    private volatile DefaultInteger maxCompactionThreshold;

    private volatile AbstractCompactionStrategy compactionStrategy;

    public final Directories directories;

    public final ColumnFamilyMetrics metric;

    public volatile long sampleLatencyNanos;

    public void reload()

    {

        // metadata object has been mutated directly. make all the members jibe with new settings.

        // only update these runtime-modifiable settings if they have not been modified.

        if (!minCompactionThreshold.isModified())

            for (ColumnFamilyStore cfs : concatWithIndexes())

                cfs.minCompactionThreshold = new DefaultInteger(metadata.getMinCompactionThreshold());

        if (!maxCompactionThreshold.isModified())

            for (ColumnFamilyStore cfs : concatWithIndexes())

                cfs.maxCompactionThreshold = new DefaultInteger(metadata.getMaxCompactionThreshold());

        maybeReloadCompactionStrategy();

        scheduleFlush();

        indexManager.reload();

        // If the CF comparator has changed, we need to change the memtable,

        // because the old one still aliases the previous comparator.

        if (data.getView().getCurrentMemtable().initialComparator != metadata.comparator)

            switchMemtable();

    }

    private void maybeReloadCompactionStrategy()

    {

        // Check if there is a need for reloading

        if (metadata.compactionStrategyClass.equals(compactionStrategy.getClass()) && metadata.compactionStrategyOptions.equals(compactionStrategy.options))

            return;

        // synchronize vs runWithCompactionsDisabled calling pause/resume.  otherwise, letting old compactions

        // finish should be harmless and possibly useful.

        synchronized (this)

        {

            compactionStrategy.shutdown();

            compactionStrategy = metadata.createCompactionStrategyInstance(this);

            compactionStrategy.startup();

        }

    }

    void scheduleFlush()

    {

        int period = metadata.getMemtableFlushPeriod();

        if (period > 0)

        {

            logger.debug("scheduling flush in {} ms", period);

            WrappedRunnable runnable = new WrappedRunnable()

            {

                protected void runMayThrow() throws Exception

                {

                    synchronized (data)

                    {

                        Memtable current = data.getView().getCurrentMemtable();

                        // if we're not expired, we've been hit by a scheduled flush for an already flushed memtable, so ignore

                        if (current.isExpired())

                        {

                            if (current.isClean())

                            {

                                // if we're still clean, instead of swapping just reschedule a flush for later

                                scheduleFlush();

                            }

                            else

                            {

                                // we'll be rescheduled by the constructor of the Memtable.

                                forceFlush();

                            }

                        }

                    }

                }

            };

            StorageService.scheduledTasks.schedule(runnable, period, TimeUnit.MILLISECONDS);

        }

    }

    public void setCompactionStrategyClass(String compactionStrategyClass)

    {

        try

        {

            metadata.compactionStrategyClass = CFMetaData.createCompactionStrategy(compactionStrategyClass);

            maybeReloadCompactionStrategy();

        }

        catch (ConfigurationException e)

        {

            throw new IllegalArgumentException(e.getMessage());

        }

    }

    public String getCompactionStrategyClass()

    {

        return metadata.compactionStrategyClass.getName();

    }

    public Map<String,String> getCompressionParameters()

    {

        return metadata.compressionParameters().asThriftOptions();

    }

    public void setCompressionParameters(Map<String,String> opts)

    {

        try

        {

            metadata.compressionParameters = CompressionParameters.create(opts);

        }

        catch (ConfigurationException e)

        {

            throw new IllegalArgumentException(e.getMessage());

        }

    }

    public void setCrcCheckChance(double crcCheckChance)

    {

        try

        {

            for (SSTableReader sstable : keyspace.getAllSSTables())

                if (sstable.compression)

                    sstable.getCompressionMetadata().parameters.setCrcCheckChance(crcCheckChance);

        }

        catch (ConfigurationException e)

        {

            throw new IllegalArgumentException(e.getMessage());

        }

    }

    private ColumnFamilyStore(Keyspace keyspace,

                              String columnFamilyName,

                              IPartitioner partitioner,

                              int generation,

                              CFMetaData metadata,

                              Directories directories,

                              boolean loadSSTables)

    {

        assert metadata != null : "null metadata for " + keyspace + ":" + columnFamilyName;

        this.keyspace = keyspace;

        name = columnFamilyName;

        this.metadata = metadata;

        this.minCompactionThreshold = new DefaultInteger(metadata.getMinCompactionThreshold());

        this.maxCompactionThreshold = new DefaultInteger(metadata.getMaxCompactionThreshold());

        this.partitioner = partitioner;

        this.directories = directories;

        this.indexManager = new SecondaryIndexManager(this);

        this.metric = new ColumnFamilyMetrics(this);

        fileIndexGenerator.set(generation);

        sampleLatencyNanos = DatabaseDescriptor.getReadRpcTimeout() / 2;

        CachingOptions caching = metadata.getCaching();

        logger.info("Initializing {}.{}", keyspace.getName(), name);

        // scan for sstables corresponding to this cf and load them

        data = new DataTracker(this);

        if (loadSSTables)

        {

            Directories.SSTableLister sstableFiles = directories.sstableLister().skipTemporary(true);

            Collection<SSTableReader> sstables = SSTableReader.openAll(sstableFiles.list().entrySet(), metadata, this.partitioner);

            data.addInitialSSTables(sstables);

        }

        if (caching.keyCache.isEnabled())

            CacheService.instance.keyCache.loadSaved(this);

        // compaction strategy should be created after the CFS has been prepared

        this.compactionStrategy = metadata.createCompactionStrategyInstance(this);

        this.compactionStrategy.startup();

        if (maxCompactionThreshold.value() <= 0 || minCompactionThreshold.value() <=0)

        {

            logger.warn("Disabling compaction strategy by setting compaction thresholds to 0 is deprecated, set the compaction option 'enabled' to 'false' instead.");

            this.compactionStrategy.disable();

        }

        // create the private ColumnFamilyStores for the secondary column indexes

        for (ColumnDefinition info : metadata.allColumns())

        {

            if (info.getIndexType() != null)

                indexManager.addIndexedColumn(info);

        }

        // register the mbean

        String type = this.partitioner instanceof LocalPartitioner ? "IndexColumnFamilies" : "ColumnFamilies";

        mbeanName = "org.apache.cassandra.db:type=" + type + ",keyspace=" + this.keyspace.getName() + ",columnfamily=" + name;

        try

        {

            MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

            ObjectName nameObj = new ObjectName(mbeanName);

            mbs.registerMBean(this, nameObj);

        }

        catch (Exception e)

        {

            throw new RuntimeException(e);

        }

        logger.debug("retryPolicy for {} is {}", name, this.metadata.getSpeculativeRetry());

        StorageService.optionalTasks.scheduleWithFixedDelay(new Runnable()

        {

            public void run()

            {

                SpeculativeRetry retryPolicy = ColumnFamilyStore.this.metadata.getSpeculativeRetry();

                switch (retryPolicy.type)

                {

                    case PERCENTILE:

                        // get percentile in nanos

                        assert metric.coordinatorReadLatency.durationUnit() == TimeUnit.MICROSECONDS;

                        sampleLatencyNanos = (long) (metric.coordinatorReadLatency.getSnapshot().getValue(retryPolicy.value) * 1000d);

                        break;

                    case CUSTOM:

                        // convert to nanos, since configuration is in millisecond

                        sampleLatencyNanos = (long) (retryPolicy.value * 1000d * 1000d);

                        break;

                    default:

                        sampleLatencyNanos = Long.MAX_VALUE;

                        break;

                }

            }

        }, DatabaseDescriptor.getReadRpcTimeout(), DatabaseDescriptor.getReadRpcTimeout(), TimeUnit.MILLISECONDS);

    }

    /** call when dropping or renaming a CF. Performs mbean housekeeping and invalidates CFS to other operations */

    public void invalidate()

    {

        valid = false;

        try

        {

            unregisterMBean();

        }

        catch (Exception e)

        {

            // this shouldn't block anything.

            logger.warn("Failed unregistering mbean: {}", mbeanName, e);

        }

        compactionStrategy.shutdown();

        SystemKeyspace.removeTruncationRecord(metadata.cfId);

        data.unreferenceSSTables();

        indexManager.invalidate();

        invalidateCaches();

    }

    /**

     * Removes every SSTable in the directory from the DataTracker's view.

     * @param directory the unreadable directory, possibly with SSTables in it, but not necessarily.

     */

    void maybeRemoveUnreadableSSTables(File directory)

    {

        data.removeUnreadableSSTables(directory);

    }

    void unregisterMBean() throws MalformedObjectNameException, InstanceNotFoundException, MBeanRegistrationException

    {

        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

        ObjectName nameObj = new ObjectName(mbeanName);

        if (mbs.isRegistered(nameObj))

            mbs.unregisterMBean(nameObj);

        // unregister metrics

        metric.release();

    }

    public long getMinRowSize()

    {

        return metric.minRowSize.value();

    }

    public long getMaxRowSize()

    {

        return metric.maxRowSize.value();

    }

    public long getMeanRowSize()

    {

        return metric.meanRowSize.value();

    }

    public int getMeanColumns()

    {

        return data.getMeanColumns();

    }

    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)

    {

        return createColumnFamilyStore(keyspace, columnFamily, StorageService.getPartitioner(), Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);

    }

    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, IPartitioner partitioner, CFMetaData metadata)

    {

        return createColumnFamilyStore(keyspace, columnFamily, partitioner, metadata, true);

    }

    private static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

                                                                         String columnFamily,

                                                                         IPartitioner partitioner,

                                                                         CFMetaData metadata,

                                                                         boolean loadSSTables)

    {

        // get the max generation number, to prevent generation conflicts

        Directories directories = new Directories(metadata);

        Directories.SSTableLister lister = directories.sstableLister().includeBackups(true);

        List<Integer> generations = new ArrayList<Integer>();

        for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())

        {

            Descriptor desc = entry.getKey();

            generations.add(desc.generation);

            if (!desc.isCompatible())

                throw new RuntimeException(String.format("Incompatible SSTable found. Current version %s is unable to read file: %s. Please run upgradesstables.",

                                                          Descriptor.Version.CURRENT, desc));

        }

        Collections.sort(generations);

        int value = (generations.size() > 0) ? (generations.get(generations.size() - 1)) : 0;

        return new ColumnFamilyStore(keyspace, columnFamily, partitioner, value, metadata, directories, loadSSTables);

    }

    /**

     * Removes unnecessary files from the cf directory at startup: these include temp files, orphans, zero-length files

     * and compacted sstables. Files that cannot be recognized will be ignored.

     */

    public static void scrubDataDirectories(CFMetaData metadata)

    {

        Directories directories = new Directories(metadata);

        // remove any left-behind SSTables from failed/stalled streaming

        FileFilter filter = new FileFilter()

        {

            public boolean accept(File pathname)

            {

                return pathname.toString().endsWith(StreamLockfile.FILE_EXT);

            }

        };

        for (File dir : directories.getCFDirectories())

        {

            File[] lockfiles = dir.listFiles(filter);

            // lock files can be null if I/O error happens

            if (lockfiles == null || lockfiles.length == 0)

                continue;

            logger.info("Removing SSTables from failed streaming session. Found {} files to cleanup.", lockfiles.length);

            for (File lockfile : lockfiles)

            {

                StreamLockfile streamLockfile = new StreamLockfile(lockfile);

                streamLockfile.cleanup();

                streamLockfile.delete();

            }

        }

        logger.debug("Removing compacted SSTable files from {} (see http://wiki.apache.org/cassandra/MemtableSSTable)", metadata.cfName);

        for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())

        {

            Descriptor desc = sstableFiles.getKey();

            Set<Component> components = sstableFiles.getValue();

            if (desc.type.isTemporary)

            {

                SSTable.delete(desc, components);

                continue;

            }

            File dataFile = new File(desc.filenameFor(Component.DATA));

            if (components.contains(Component.DATA) && dataFile.length() > 0)

                // everything appears to be in order... moving on.

                continue;

            // missing the DATA file! all components are orphaned

            logger.warn("Removing orphans for {}: {}", desc, components);

            for (Component component : components)

            {

                FileUtils.deleteWithConfirm(desc.filenameFor(component));

            }

        }

        // cleanup incomplete saved caches

        Pattern tmpCacheFilePattern = Pattern.compile(metadata.ksName + "-" + metadata.cfName + "-(Key|Row)Cache.*\\.tmp$");

        File dir = new File(DatabaseDescriptor.getSavedCachesLocation());

        if (dir.exists())

        {

            assert dir.isDirectory();

            for (File file : dir.listFiles())

                if (tmpCacheFilePattern.matcher(file.getName()).matches())

                    if (!file.delete())

                        logger.warn("could not delete {}", file.getAbsolutePath());

        }

        // also clean out any index leftovers.

        for (ColumnDefinition def : metadata.allColumns())

        {

            if (def.isIndexed())

            {

                CellNameType indexComparator = SecondaryIndex.getIndexComparator(metadata, def);

                if (indexComparator != null)

                {

                    CFMetaData indexMetadata = CFMetaData.newIndexMetadata(metadata, def, indexComparator);

                    scrubDataDirectories(indexMetadata);

                }

            }

        }

    }

    /**

     * Replacing compacted sstables is atomic as far as observers of DataTracker are concerned, but not on the

     * filesystem: first the new sstables are renamed to "live" status (i.e., the tmp marker is removed), then

     * their ancestors are removed.

     *

     * If an unclean shutdown happens at the right time, we can thus end up with both the new ones and their

     * ancestors "live" in the system.  This is harmless for normal data, but for counters it can cause overcounts.

     *

     * To prevent this, we record sstables being compacted in the system keyspace.  If we find unfinished

     * compactions, we remove the new ones (since those may be incomplete -- under LCS, we may create multiple

     * sstables from any given ancestor).

     */

    public static void removeUnfinishedCompactionLeftovers(CFMetaData metadata, Map<Integer, UUID> unfinishedCompactions)

    {

        Directories directories = new Directories(metadata);

        Set<Integer> allGenerations = new HashSet<>();

        for (Descriptor desc : directories.sstableLister().list().keySet())

            allGenerations.add(desc.generation);

        // sanity-check unfinishedCompactions

        Set<Integer> unfinishedGenerations = unfinishedCompactions.keySet();

        if (!allGenerations.containsAll(unfinishedGenerations))

        {

            HashSet<Integer> missingGenerations = new HashSet<>(unfinishedGenerations);

            missingGenerations.removeAll(allGenerations);

            logger.debug("Unfinished compactions of {}.{} reference missing sstables of generations {}",

                         metadata.ksName, metadata.cfName, missingGenerations);

        }

        // remove new sstables from compactions that didn't complete, and compute

        // set of ancestors that shouldn't exist anymore

        Set<Integer> completedAncestors = new HashSet<>();

        for (Map.Entry<Descriptor, Set<Component>> sstableFiles : directories.sstableLister().skipTemporary(true).list().entrySet())

        {

            Descriptor desc = sstableFiles.getKey();

            Set<Integer> ancestors;

            try

            {

                CompactionMetadata compactionMetadata = (CompactionMetadata) desc.getMetadataSerializer().deserialize(desc, MetadataType.COMPACTION);

                ancestors = compactionMetadata.ancestors;

            }

            catch (IOException e)

            {

                throw new FSReadError(e, desc.filenameFor(Component.STATS));

            }

            if (!ancestors.isEmpty()

                && unfinishedGenerations.containsAll(ancestors)

                && allGenerations.containsAll(ancestors))

            {

                // any of the ancestors would work, so we'll just lookup the compaction task ID with the first one

                UUID compactionTaskID = unfinishedCompactions.get(ancestors.iterator().next());

                assert compactionTaskID != null;

                logger.debug("Going to delete unfinished compaction product {}", desc);

                SSTable.delete(desc, sstableFiles.getValue());

                SystemKeyspace.finishCompaction(compactionTaskID);

            }

            else

            {

                completedAncestors.addAll(ancestors);

            }

        }

        // remove old sstables from compactions that did complete

        for (Map.Entry<Descriptor, Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())

        {

            Descriptor desc = sstableFiles.getKey();

            if (completedAncestors.contains(desc.generation))

            {

                // if any of the ancestors were participating in a compaction, finish that compaction

                logger.debug("Going to delete leftover compaction ancestor {}", desc);

                SSTable.delete(desc, sstableFiles.getValue());

                UUID compactionTaskID = unfinishedCompactions.get(desc.generation);

                if (compactionTaskID != null)

                    SystemKeyspace.finishCompaction(unfinishedCompactions.get(desc.generation));

            }

        }

    }

    // must be called after all sstables are loaded since row cache merges all row versions

    public void initRowCache()

    {

        if (!isRowCacheEnabled())

            return;

        long start = System.nanoTime();

        int cachedRowsRead = CacheService.instance.rowCache.loadSaved(this);

        if (cachedRowsRead > 0)

            logger.info("Completed loading ({} ms; {} keys) row cache for {}.{}",

                        TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start),

                        cachedRowsRead,

                        keyspace.getName(),

                        name);

    }

    public void initCounterCache()

    {

        if (!metadata.isCounter() || CacheService.instance.counterCache.getCapacity() == 0)

            return;

        long start = System.nanoTime();

        int cachedShardsRead = CacheService.instance.counterCache.loadSaved(this);

        if (cachedShardsRead > 0)

            logger.info("Completed loading ({} ms; {} shards) counter cache for {}.{}",

                        TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start),

                        cachedShardsRead,

                        keyspace.getName(),

                        name);

    }

    /**

     * See #{@code StorageService.loadNewSSTables(String, String)} for more info

     *

     * @param ksName The keyspace name

     * @param cfName The columnFamily name

     */

    public static synchronized void loadNewSSTables(String ksName, String cfName)

    {

        /** ks/cf existence checks will be done by open and getCFS methods for us */

        Keyspace keyspace = Keyspace.open(ksName);

        keyspace.getColumnFamilyStore(cfName).loadNewSSTables();

    }

    /**

     * #{@inheritDoc}

     */

    public synchronized void loadNewSSTables()

    {

        logger.info("Loading new SSTables for {}/{}...", keyspace.getName(), name);

        Set<Descriptor> currentDescriptors = new HashSet<Descriptor>();

        for (SSTableReader sstable : data.getView().sstables)