Examples of org.apache.cassandra.io.sstable.SSTableRewriter

org.apache.cassandra.io.sstable.SSTableRewriter
Wraps one or more writers as output for rewriting one or more readers: every sstable_preemptive_open_interval_in_mb we look in the summary we're collecting for the latest writer for the penultimate key that we know to have been fully flushed to the index file, and then double check that the key is fully present in the flushed data file. Then we move the starts of each reader forwards to that point, replace them in the datatracker, and attach a runnable for on-close (i.e. when all references expire) that drops the page cache prior to that key position hard-links are created for each partially written sstable so that readers opened against them continue to work past the rename of the temporary file, which is deleted once all readers against the hard-link have been closed. If for any reason the writer is rolled over, we immediately rename and fully expose the completed file in the DataTracker. On abort we restore the original lower bounds to the existing readers and delete any temporary files we had in progress, but leave any hard-links in place for the readers we opened to cleanup when they're finished as we would had we finished successfully.


        logger.info("Anticompacting {}", anticompactionGroup);
        Set<SSTableReader> sstableAsSet = new HashSet<>(anticompactionGroup);


        File destination = cfs.directories.getDirectoryForNewSSTables();
        SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, groupMaxDataAge, false);
        SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, groupMaxDataAge, false);


        long repairedKeyCount = 0;
        long unrepairedKeyCount = 0;
        AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
        try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup);
             CompactionController controller = new CompactionController(cfs, sstableAsSet, CFMetaData.DEFAULT_GC_GRACE_SECONDS))
        {
            int expectedBloomFilterSize = Math.max(cfs.metadata.getMinIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(anticompactionGroup)));


            repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, sstableAsSet));
            unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstableAsSet));


            CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners.scanners, controller, DatabaseDescriptor.getSSTableFormat());
            Iterator<AbstractCompactedRow> iter = ci.iterator();
            while(iter.hasNext())
            {
                AbstractCompactedRow row = iter.next();
                // if current range from sstable is repaired, save it into the new repaired sstable
                if (Range.isInRanges(row.key.getToken(), ranges))
                {
                    repairedSSTableWriter.append(row);
                    repairedKeyCount++;
                }
                // otherwise save into the new 'non-repaired' table
                else
                {
                    unRepairedSSTableWriter.append(row);
                    unrepairedKeyCount++;
                }
            }
            // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
            // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
            List<SSTableReader> anticompactedSSTables = new ArrayList<>();
            anticompactedSSTables.addAll(repairedSSTableWriter.finish(repairedAt));
            anticompactedSSTables.addAll(unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE));
            cfs.getDataTracker().markCompactedSSTablesReplaced(sstableAsSet, anticompactedSSTables, OperationType.ANTICOMPACTION);


            logger.debug("Repaired {} keys out of {} for {}/{} in {}", repairedKeyCount,
                                                                       repairedKeyCount + unrepairedKeyCount,
                                                                       cfs.keyspace.getName(),
                                                                       cfs.getColumnFamilyName(),
                                                                       anticompactionGroup);
            return anticompactedSSTables.size();
        }
        catch (Throwable e)
        {
            JVMStabilityInspector.inspectThrowable(e);
            logger.error("Error anticompacting " + anticompactionGroup, e);
            repairedSSTableWriter.abort();
            unRepairedSSTableWriter.abort();
        }
        return 0;
    }

View Full Code Here

                // we only need the age of the data that we're actually retaining
                long maxAge = getMaxDataAge(actuallyCompact);
                if (collector != null)
                    collector.beginCompaction(ci);
                long lastCheckObsoletion = start;
                SSTableRewriter writer = new SSTableRewriter(cfs, sstables, maxAge, offline);
                try
                {
                    if (!iter.hasNext())
                    {
                        // don't mark compacted in the finally block, since if there _is_ nondeleted data,
                        // we need to sync it (via closeAndOpen) first, so there is no period during which
                        // a crash could cause data loss.
                        cfs.markObsolete(sstables, compactionType);
                        return;
                    }


                    writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt, sstableFormat));
                    while (iter.hasNext())
                    {
                        if (ci.isStopRequested())
                            throw new CompactionInterruptedException(ci.getCompactionInfo());


                        AbstractCompactedRow row = iter.next();
                        if (writer.append(row) != null)
                        {
                            totalKeysWritten++;
                            if (newSSTableSegmentThresholdReached(writer.currentWriter()))
                            {
                                writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt, sstableFormat));
                            }
                        }


                        if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
                        {
                            controller.maybeRefreshOverlaps();
                            lastCheckObsoletion = System.nanoTime();
                        }
                    }


                    // don't replace old sstables yet, as we need to mark the compaction finished in the system table
                    newSStables = writer.finish();
                }
                catch (Throwable t)
                {
                    writer.abort();
                    throw t;
                }
                finally
                {

View Full Code Here


        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);


        metrics.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, new HashSet<>(ImmutableSet.of(sstable)), sstable.maxDataAge, OperationType.CLEANUP, false);


        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));


            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());


                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }


            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();


            writer.finish();
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
            metrics.finishCompaction(ci);
        }


        List<SSTableReader> results = writer.finished();
        if (!results.isEmpty())
        {
            String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys.  Time: %,dms.";
            long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            long startsize = sstable.onDiskLength();

View Full Code Here

            logger.info("Anticompacting {}", sstable);
            Set<SSTableReader> sstableAsSet = new HashSet<>();
            sstableAsSet.add(sstable);


            File destination = cfs.directories.getDirectoryForNewSSTables();
            SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
            SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);


            AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
            List<ICompactionScanner> scanners = strategy.getScanners(Arrays.asList(sstable));


            try (CompactionController controller = new CompactionController(cfs, new HashSet<>(Collections.singleton(sstable)), CFMetaData.DEFAULT_GC_GRACE_SECONDS))
            {
                repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable));
                unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));


                CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners, controller);


                try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
                {
                    while(iter.hasNext())
                    {
                        AbstractCompactedRow row = iter.next();
                        // if current range from sstable is repaired, save it into the new repaired sstable
                        if (Range.isInRanges(row.key.getToken(), ranges))
                        {
                            repairedSSTableWriter.append(row);
                            repairedKeyCount++;
                        }
                        // otherwise save into the new 'non-repaired' table
                        else
                        {
                            unRepairedSSTableWriter.append(row);
                            unrepairedKeyCount++;
                        }
                    }
                }
                // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
                // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
                repairedSSTableWriter.finish(false, repairedAt);
                unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE);
                // add repaired table with a non-null timestamp field to be saved in SSTableMetadata#repairedAt
                anticompactedSSTables.addAll(repairedSSTableWriter.finished());
                anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
            }
            catch (Throwable e)
            {
                logger.error("Error anticompacting " + sstable, e);
                repairedSSTableWriter.abort();
                unRepairedSSTableWriter.abort();
            }
        }
        String format = "Repaired {} keys of {} for {}/{}";
        logger.debug(format, repairedKeyCount, (repairedKeyCount + unrepairedKeyCount), cfs.keyspace, cfs.getColumnFamilyName());
        String format2 = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";

View Full Code Here

0 1

TOP

Related Classes of org.apache.cassandra.io.sstable.SSTableRewriter

org.apache.cassandra.db.compaction.CompactionManager

org.apache.cassandra.db.compaction.CompactionTask

org.apache.cassandra.db.DecoratedKey

org.apache.cassandra.db.RowIndexEntry

org.apache.cassandra.io.sstable.format.SSTableReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.