* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

    public static final String MBEAN_NAME = "org.apache.cassandra.db:type=StorageProxy";

    private static final Logger logger = LoggerFactory.getLogger(StorageProxy.class);

    static final boolean OPTIMIZE_LOCAL_REQUESTS = true; // set to false to test messagingservice path on single node

    public static final String UNREACHABLE = "UNREACHABLE";

    private static final WritePerformer standardWritePerformer;

    private static final WritePerformer counterWritePerformer;

    private static final WritePerformer counterWriteOnCoordinatorPerformer;

    public static final StorageProxy instance = new StorageProxy();

    private static volatile int maxHintsInProgress = 1024 * FBUtilities.getAvailableProcessors();

    private static final CacheLoader<InetAddress, AtomicInteger> hintsInProgress = new CacheLoader<InetAddress, AtomicInteger>()

    {

        public AtomicInteger load(InetAddress inetAddress)

        {

            return new AtomicInteger(0);

        }

    };

    private static final ClientRequestMetrics readMetrics = new ClientRequestMetrics("Read");

    private static final ClientRequestMetrics rangeMetrics = new ClientRequestMetrics("RangeSlice");

    private static final ClientRequestMetrics writeMetrics = new ClientRequestMetrics("Write");

    private static final double CONCURRENT_SUBREQUESTS_MARGIN = 0.10;

    private StorageProxy() {}

    static

    {

        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

        try

        {

            mbs.registerMBean(instance, new ObjectName(MBEAN_NAME));

        }

        catch (Exception e)

        {

            throw new RuntimeException(e);

        }

        standardWritePerformer = new WritePerformer()

        {

            public void apply(IMutation mutation,

                              Iterable<InetAddress> targets,

                              AbstractWriteResponseHandler responseHandler,

                              String localDataCenter,

                              ConsistencyLevel consistency_level)

            throws OverloadedException

            {

                assert mutation instanceof Mutation;

                sendToHintedEndpoints((Mutation) mutation, targets, responseHandler, localDataCenter);

            }

        };

        /*

         * We execute counter writes in 2 places: either directly in the coordinator node if it is a replica, or

         * in CounterMutationVerbHandler on a replica othewise. The write must be executed on the COUNTER_MUTATION stage

         * but on the latter case, the verb handler already run on the COUNTER_MUTATION stage, so we must not execute the

         * underlying on the stage otherwise we risk a deadlock. Hence two different performer.

         */

        counterWritePerformer = new WritePerformer()

        {

            public void apply(IMutation mutation,

                              Iterable<InetAddress> targets,

                              AbstractWriteResponseHandler responseHandler,

                              String localDataCenter,

                              ConsistencyLevel consistencyLevel)

            {

                counterWriteTask(mutation, targets, responseHandler, localDataCenter).run();

            }

        };

        counterWriteOnCoordinatorPerformer = new WritePerformer()

        {

            public void apply(IMutation mutation,

                              Iterable<InetAddress> targets,

                              AbstractWriteResponseHandler responseHandler,

                              String localDataCenter,

                              ConsistencyLevel consistencyLevel)

            {

                StageManager.getStage(Stage.COUNTER_MUTATION)

                            .execute(counterWriteTask(mutation, targets, responseHandler, localDataCenter));

            }

        };

    }

    /**

     * Apply @param updates if and only if the current values in the row for @param key

     * match the provided @param conditions.  The algorithm is "raw" Paxos: that is, Paxos

     * minus leader election -- any node in the cluster may propose changes for any row,

     * which (that is, the row) is the unit of values being proposed, not single columns.

     *

     * The Paxos cohort is only the replicas for the given key, not the entire cluster.

     * So we expect performance to be reasonable, but CAS is still intended to be used

     * "when you really need it," not for all your updates.

     *

     * There are three phases to Paxos:

     *  1. Prepare: the coordinator generates a ballot (timeUUID in our case) and asks replicas to (a) promise

     *     not to accept updates from older ballots and (b) tell us about the most recent update it has already

     *     accepted.

     *  2. Accept: if a majority of replicas reply, the coordinator asks replicas to accept the value of the

     *     highest proposal ballot it heard about, or a new value if no in-progress proposals were reported.

     *  3. Commit (Learn): if a majority of replicas acknowledge the accept request, we can commit the new

     *     value.

     *

     *  Commit procedure is not covered in "Paxos Made Simple," and only briefly mentioned in "Paxos Made Live,"

     *  so here is our approach:

     *   3a. The coordinator sends a commit message to all replicas with the ballot and value.

     *   3b. Because of 1-2, this will be the highest-seen commit ballot.  The replicas will note that,

     *       and send it with subsequent promise replies.  This allows us to discard acceptance records

     *       for successfully committed replicas, without allowing incomplete proposals to commit erroneously

     *       later on.

     *

     *  Note that since we are performing a CAS rather than a simple update, we perform a read (of committed

     *  values) between the prepare and accept phases.  This gives us a slightly longer window for another

     *  coordinator to come along and trump our own promise with a newer one but is otherwise safe.

     *

     * @param keyspaceName the keyspace for the CAS

     * @param cfName the column family for the CAS

     * @param key the row key for the row to CAS

     * @param conditions the conditions for the CAS to apply.

     * @param updates the value to insert if {@code condtions} matches the current values.

     * @param consistencyForPaxos the consistency for the paxos prepare and propose round. This can only be either SERIAL or LOCAL_SERIAL.

     * @param consistencyForCommit the consistency for write done during the commit phase. This can be anything, except SERIAL or LOCAL_SERIAL.

     *

     * @return null if the operation succeeds in updating the row, or the current values corresponding to conditions.

     * (since, if the CAS doesn't succeed, it means the current value do not match the conditions).

     */

    public static ColumnFamily cas(String keyspaceName,

                                   String cfName,

                                   ByteBuffer key,

                                   CASConditions conditions,

                                   ColumnFamily updates,

                                   ConsistencyLevel consistencyForPaxos,

                                   ConsistencyLevel consistencyForCommit)

    throws UnavailableException, IsBootstrappingException, ReadTimeoutException, WriteTimeoutException, InvalidRequestException

    {

        consistencyForPaxos.validateForCas();

        consistencyForCommit.validateForCasCommit(keyspaceName);

        CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName);

        long start = System.nanoTime();

        long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCasContentionTimeout());

        while (System.nanoTime() - start < timeout)

        {

            // for simplicity, we'll do a single liveness check at the start of each attempt

            Pair<List<InetAddress>, Integer> p = getPaxosParticipants(keyspaceName, key, consistencyForPaxos);

            List<InetAddress> liveEndpoints = p.left;

            int requiredParticipants = p.right;

            UUID ballot = beginAndRepairPaxos(start, key, metadata, liveEndpoints, requiredParticipants, consistencyForPaxos, consistencyForCommit);

            // read the current values and check they validate the conditions

            Tracing.trace("Reading existing values for CAS precondition");

            long timestamp = System.currentTimeMillis();

            ReadCommand readCommand = ReadCommand.create(keyspaceName, key, cfName, timestamp, conditions.readFilter());

            List<Row> rows = read(Arrays.asList(readCommand), consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM);

            ColumnFamily current = rows.get(0).cf;

            if (!conditions.appliesTo(current))

            {

                Tracing.trace("CAS precondition {} does not match current values {}", conditions, current);

                // We should not return null as this means success

                return current == null ? ArrayBackedSortedColumns.factory.create(metadata) : current;

            }

            // finish the paxos round w/ the desired updates

            // TODO turn null updates into delete?

            // Apply triggers to cas updates. A consideration here is that

            // triggers emit Mutations, and so a given trigger implementation

            // may generate mutations for partitions other than the one this

            // paxos round is scoped for. In this case, TriggerExecutor will

            // validate that the generated mutations are targetted at the same

            // partition as the initial updates and reject (via an

            // InvalidRequestException) any which aren't.

            updates = TriggerExecutor.instance.execute(key, updates);

            Commit proposal = Commit.newProposal(key, ballot, updates);

            Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot);

            if (proposePaxos(proposal, liveEndpoints, requiredParticipants, true, consistencyForPaxos))

            {

                if (consistencyForCommit == ConsistencyLevel.ANY)

                    sendCommit(proposal, liveEndpoints);

                else

                    commitPaxos(proposal, consistencyForCommit);

                Tracing.trace("CAS successful");

                return null;

            }

            Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)");

            Uninterruptibles.sleepUninterruptibly(FBUtilities.threadLocalRandom().nextInt(100), TimeUnit.MILLISECONDS);

            // continue to retry

        }

        throw new WriteTimeoutException(WriteType.CAS, consistencyForPaxos, 0, consistencyForPaxos.blockFor(Keyspace.open(keyspaceName)));

    }

    private static Predicate<InetAddress> sameDCPredicateFor(final String dc)

    {

        final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();

        return new Predicate<InetAddress>()

        {

            public boolean apply(InetAddress host)

            {

                return dc.equals(snitch.getDatacenter(host));

            }

        };

    }

    private static Pair<List<InetAddress>, Integer> getPaxosParticipants(String keyspaceName, ByteBuffer key, ConsistencyLevel consistencyForPaxos) throws UnavailableException

    {

        Token tk = StorageService.getPartitioner().getToken(key);

        List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);

        Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);

        if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL)

        {

            // Restrict naturalEndpoints and pendingEndpoints to node in the local DC only

            String localDc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());

            Predicate<InetAddress> isLocalDc = sameDCPredicateFor(localDc);

            naturalEndpoints = ImmutableList.copyOf(Iterables.filter(naturalEndpoints, isLocalDc));

            pendingEndpoints = ImmutableList.copyOf(Iterables.filter(pendingEndpoints, isLocalDc));

        }

        int requiredParticipants = pendingEndpoints.size() + 1 + naturalEndpoints.size() / 2; // See CASSANDRA-833

        List<InetAddress> liveEndpoints = ImmutableList.copyOf(Iterables.filter(Iterables.concat(naturalEndpoints, pendingEndpoints), IAsyncCallback.isAlive));

        if (liveEndpoints.size() < requiredParticipants)

            throw new UnavailableException(consistencyForPaxos, requiredParticipants, liveEndpoints.size());

        return Pair.create(liveEndpoints, requiredParticipants);

    }

    /**

     * begin a Paxos session by sending a prepare request and completing any in-progress requests seen in the replies

     *

     * @return the Paxos ballot promised by the replicas if no in-progress requests were seen and a quorum of

     * nodes have seen the mostRecentCommit.  Otherwise, return null.

     */

    private static UUID beginAndRepairPaxos(long start, ByteBuffer key, CFMetaData metadata, List<InetAddress> liveEndpoints, int requiredParticipants, ConsistencyLevel consistencyForPaxos, ConsistencyLevel consistencyForCommit)

    throws WriteTimeoutException

    {

        long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCasContentionTimeout());

        PrepareCallback summary = null;

        while (System.nanoTime() - start < timeout)

        {

            long ballotMillis = summary == null

                              ? System.currentTimeMillis()

                              : Math.max(System.currentTimeMillis(), 1 + UUIDGen.unixTimestamp(summary.mostRecentInProgressCommit.ballot));

            UUID ballot = UUIDGen.getTimeUUID(ballotMillis);

            // prepare

            Tracing.trace("Preparing {}", ballot);

            Commit toPrepare = Commit.newPrepare(key, metadata, ballot);

            summary = preparePaxos(toPrepare, liveEndpoints, requiredParticipants, consistencyForPaxos);

            if (!summary.promised)

            {

                Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");

                // sleep a random amount to give the other proposer a chance to finish

                Uninterruptibles.sleepUninterruptibly(FBUtilities.threadLocalRandom().nextInt(100), TimeUnit.MILLISECONDS);

                continue;

            }

            Commit inProgress = summary.mostRecentInProgressCommitWithUpdate;

            Commit mostRecent = summary.mostRecentCommit;

            // If we have an in-progress ballot greater than the MRC we know, then it's an in-progress round that

            // needs to be completed, so do it.

            if (!inProgress.update.isEmpty() && inProgress.isAfter(mostRecent))

            {

                Tracing.trace("Finishing incomplete paxos round {}", inProgress);

                Commit refreshedInProgress = Commit.newProposal(inProgress.key, ballot, inProgress.update);

                if (proposePaxos(refreshedInProgress, liveEndpoints, requiredParticipants, false, consistencyForPaxos))

                {

                    commitPaxos(refreshedInProgress, consistencyForCommit);

                }

                else

                {

                    Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");

                    // sleep a random amount to give the other proposer a chance to finish

                    Uninterruptibles.sleepUninterruptibly(FBUtilities.threadLocalRandom().nextInt(100), TimeUnit.MILLISECONDS);

                }

                continue;

            }

            // To be able to propose our value on a new round, we need a quorum of replica to have learn the previous one. Why is explained at:

            // https://issues.apache.org/jira/browse/CASSANDRA-5062?focusedCommentId=13619810&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13619810)

            // Since we waited for quorum nodes, if some of them haven't seen the last commit (which may just be a timing issue, but may also

            // mean we lost messages), we pro-actively "repair" those nodes, and retry.

            Iterable<InetAddress> missingMRC = summary.replicasMissingMostRecentCommit();

            if (Iterables.size(missingMRC) > 0)

            {

                Tracing.trace("Repairing replicas that missed the most recent commit");

                sendCommit(mostRecent, missingMRC);

                // TODO: provided commits don't invalid the prepare we just did above (which they don't), we could just wait

                // for all the missingMRC to acknowledge this commit and then move on with proposing our value. But that means

                // adding the ability to have commitPaxos block, which is exactly CASSANDRA-5442 will do. So once we have that

                // latter ticket, we can pass CL.ALL to the commit above and remove the 'continue'.

                continue;

            }

            return ballot;

        }

        throw new WriteTimeoutException(WriteType.CAS, consistencyForPaxos, 0, consistencyForPaxos.blockFor(Keyspace.open(metadata.ksName)));

    }

    /**

     * Unlike commitPaxos, this does not wait for replies

     */

    private static void sendCommit(Commit commit, Iterable<InetAddress> replicas)

    {

        MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_COMMIT, commit, Commit.serializer);

        for (InetAddress target : replicas)

            MessagingService.instance().sendOneWay(message, target);

    }

    private static PrepareCallback preparePaxos(Commit toPrepare, List<InetAddress> endpoints, int requiredParticipants, ConsistencyLevel consistencyForPaxos)

    throws WriteTimeoutException

    {

        PrepareCallback callback = new PrepareCallback(toPrepare.key, toPrepare.update.metadata(), requiredParticipants, consistencyForPaxos);

        MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_PREPARE, toPrepare, Commit.serializer);

        for (InetAddress target : endpoints)

            MessagingService.instance().sendRR(message, target, callback);

        callback.await();

        return callback;

    }

    private static boolean proposePaxos(Commit proposal, List<InetAddress> endpoints, int requiredParticipants, boolean timeoutIfPartial, ConsistencyLevel consistencyLevel)

    throws WriteTimeoutException

    {

        ProposeCallback callback = new ProposeCallback(endpoints.size(), requiredParticipants, !timeoutIfPartial, consistencyLevel);

        MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_PROPOSE, proposal, Commit.serializer);

        for (InetAddress target : endpoints)

            MessagingService.instance().sendRR(message, target, callback);

        callback.await();

        if (callback.isSuccessful())

            return true;

        if (timeoutIfPartial && !callback.isFullyRefused())

            throw new WriteTimeoutException(WriteType.CAS, consistencyLevel, callback.getAcceptCount(), requiredParticipants);

        return false;

    }

    private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel) throws WriteTimeoutException

    {

        Keyspace keyspace = Keyspace.open(proposal.update.metadata().ksName);

        Token tk = StorageService.getPartitioner().getToken(proposal.key);

        List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspace.getName(), tk);

        Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspace.getName());

        AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();

        AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistencyLevel, null, WriteType.SIMPLE);

        MessageOut<Commit> message = new MessageOut<Commit>(MessagingService.Verb.PAXOS_COMMIT, proposal, Commit.serializer);

        for (InetAddress destination : Iterables.concat(naturalEndpoints, pendingEndpoints))

        {

            if (FailureDetector.instance.isAlive(destination))

                MessagingService.instance().sendRR(message, destination, responseHandler);

        }

        responseHandler.get();

    }

    /**

     * Use this method to have these Mutations applied

     * across all replicas. This method will take care

     * of the possibility of a replica being down and hint

     * the data across to some other replica.

     *

     * @param mutations the mutations to be applied across the replicas

     * @param consistency_level the consistency level for the operation

     */

    public static void mutate(Collection<? extends IMutation> mutations, ConsistencyLevel consistency_level)

    throws UnavailableException, OverloadedException, WriteTimeoutException

    {

        Tracing.trace("Determining replicas for mutation");

        final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());

        long startTime = System.nanoTime();

        List<AbstractWriteResponseHandler> responseHandlers = new ArrayList<>(mutations.size());

        try

        {

            for (IMutation mutation : mutations)

            {

                mutation.retain();

                if (mutation instanceof CounterMutation)

                {

                    responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter));

                }

                else

                {

                    WriteType wt = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH;

                    responseHandlers.add(performWrite(mutation, consistency_level, localDataCenter, standardWritePerformer, null, wt));

                }

            }

            // wait for writes.  throws TimeoutException if necessary

            for (AbstractWriteResponseHandler responseHandler : responseHandlers)

            {

                responseHandler.get();

            }

        }

        catch (WriteTimeoutException ex)

        {

            if (consistency_level == ConsistencyLevel.ANY)

            {

                // hint all the mutations (except counters, which can't be safely retried).  This means

                // we'll re-hint any successful ones; doesn't seem worth it to track individual success

                // just for this unusual case.

                for (IMutation mutation : mutations)

                {

                    if (mutation instanceof CounterMutation)

                        continue;

                    Token tk = StorageService.getPartitioner().getToken(mutation.key());

                    List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk);

                    Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName());

                    for (InetAddress target : Iterables.concat(naturalEndpoints, pendingEndpoints))

                    {

                        // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and

                        // CASSANDRA-6510), so there is no need to hint or retry

                        if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target))

                            submitHint((Mutation) mutation, target, null);

                    }

                }

                Tracing.trace("Wrote hint to satisfy CL.ANY after no replicas acknowledged the write");

            }

            else

            {

                writeMetrics.timeouts.mark();

                ClientRequestMetrics.writeTimeouts.inc();

                Tracing.trace("Write timeout; received {} of {} required replies", ex.received, ex.blockFor);

                throw ex;

            }

        }

        catch (UnavailableException e)

        {

            writeMetrics.unavailables.mark();

            ClientRequestMetrics.writeUnavailables.inc();

            Tracing.trace("Unavailable");

            throw e;

        }

        catch (OverloadedException e)

        {

            ClientRequestMetrics.writeUnavailables.inc();

            Tracing.trace("Overloaded");

            throw e;

        }

        finally

        {

            //Release the mutations we dispatched so far.

            //An exception may be thrown at anytime.

            //We can infer the mutations that were dispatched from this list

            Iterator<? extends IMutation> it = mutations.iterator();

            for (int i = 0; i < responseHandlers.size(); i++)

                it.next().release();

           

            writeMetrics.addNano(System.nanoTime() - startTime);

        }

    }

    @SuppressWarnings("unchecked")

    public static void mutateWithTriggers(Collection<? extends IMutation> mutations,

                                          ConsistencyLevel consistencyLevel,

                                          boolean mutateAtomically)

    throws WriteTimeoutException, UnavailableException, OverloadedException, InvalidRequestException

    {

        Collection<Mutation> augmented = TriggerExecutor.instance.execute(mutations);

        if (augmented != null)

            mutateAtomically(augmented, consistencyLevel);

        else if (mutateAtomically)

            mutateAtomically((Collection<Mutation>) mutations, consistencyLevel);

        else

            mutate(mutations, consistencyLevel);

    }

    /**

     * See mutate. Adds additional steps before and after writing a batch.

     * Before writing the batch (but after doing availability check against the FD for the row replicas):

     *      write the entire batch to a batchlog elsewhere in the cluster.

     * After: remove the batchlog entry (after writing hints for the batch rows, if necessary).

     *

     * @param mutations the Mutations to be applied across the replicas

     * @param consistency_level the consistency level for the operation

     */

    public static void mutateAtomically(Collection<Mutation> mutations, ConsistencyLevel consistency_level)

    throws UnavailableException, OverloadedException, WriteTimeoutException

    {

        Tracing.trace("Determining replicas for atomic batch");

        long startTime = System.nanoTime();

        List<WriteResponseHandlerWrapper> wrappers = new ArrayList<WriteResponseHandlerWrapper>(mutations.size());

        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());

        try

        {

            // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet

            for (Mutation mutation : mutations)

            {

                WriteResponseHandlerWrapper wrapper = wrapResponseHandler(mutation, consistency_level, WriteType.BATCH);

                // exit early if we can't fulfill the CL at this time.

                wrapper.handler.assureSufficientLiveNodes();

                wrappers.add(wrapper);

            }

            // write to the batchlog

            Collection<InetAddress> batchlogEndpoints = getBatchlogEndpoints(localDataCenter, consistency_level);

            UUID batchUUID = UUIDGen.getTimeUUID();

            syncWriteToBatchlog(mutations, batchlogEndpoints, batchUUID);

            // now actually perform the writes and wait for them to complete

            syncWriteBatchedMutations(wrappers, localDataCenter);

            // remove the batchlog entries asynchronously

            asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID);

        }

        catch (UnavailableException e)

        {

            writeMetrics.unavailables.mark();

            ClientRequestMetrics.writeUnavailables.inc();

            Tracing.trace("Unavailable");

            throw e;

        }

        catch (WriteTimeoutException e)

        {

            writeMetrics.timeouts.mark();

            ClientRequestMetrics.writeTimeouts.inc();

            Tracing.trace("Write timeout; received {} of {} required replies", e.received, e.blockFor);

            throw e;

        }

        finally

        {

            writeMetrics.addNano(System.nanoTime() - startTime);

        }

    }

    private static void syncWriteToBatchlog(Collection<Mutation> mutations, Collection<InetAddress> endpoints, UUID uuid)

    throws WriteTimeoutException

    {

        AbstractWriteResponseHandler handler = new WriteResponseHandler(endpoints,

                                                                        Collections.<InetAddress>emptyList(),

                                                                        ConsistencyLevel.ONE,

                                                                        Keyspace.open(Keyspace.SYSTEM_KS),

                                                                        null,

                                                                        WriteType.BATCH_LOG);

        MessageOut<Mutation> message = BatchlogManager.getBatchlogMutationFor(mutations, uuid, MessagingService.current_version)

                                                      .createMessage();

        for (InetAddress target : endpoints)

        {

            int targetVersion = MessagingService.instance().getVersion(target);

            if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS)

            {

                insertLocal(message.payload, handler);

            }

            else if (targetVersion == MessagingService.current_version)

            {

                MessagingService.instance().sendRR(message, target, handler, false);

            }

            else

            {

                MessagingService.instance().sendRR(BatchlogManager.getBatchlogMutationFor(mutations, uuid, targetVersion)

                                                                  .createMessage(),

                                                   target,

                                                   handler,

                                                   false);

            }

        }

        handler.get();

    }

    private static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid)

    {

        AbstractWriteResponseHandler handler = new WriteResponseHandler(endpoints,

                                                                        Collections.<InetAddress>emptyList(),

                                                                        ConsistencyLevel.ANY,

                                                                        Keyspace.open(Keyspace.SYSTEM_KS),

                                                                        null,

                                                                        WriteType.SIMPLE);

        Mutation mutation = new Mutation(Keyspace.SYSTEM_KS, UUIDType.instance.decompose(uuid));

        mutation.delete(SystemKeyspace.BATCHLOG_CF, FBUtilities.timestampMicros());

        MessageOut<Mutation> message = mutation.createMessage();

        for (InetAddress target : endpoints)

        {

            if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS)

                insertLocal(message.payload, handler);

            else

                MessagingService.instance().sendRR(message, target, handler, false);

        }

    }

    private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter)

    throws WriteTimeoutException, OverloadedException

    {

        for (WriteResponseHandlerWrapper wrapper : wrappers)

        {

            Iterable<InetAddress> endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints);

            sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter);

        }

        for (WriteResponseHandlerWrapper wrapper : wrappers)

            wrapper.handler.get();

    }

    /**

     * Perform the write of a mutation given a WritePerformer.

     * Gather the list of write endpoints, apply locally and/or forward the mutation to

     * said write endpoint (deletaged to the actual WritePerformer) and wait for the

     * responses based on consistency level.

     *

     * @param mutation the mutation to be applied

     * @param consistency_level the consistency level for the write operation

     * @param performer the WritePerformer in charge of appliying the mutation

     * given the list of write endpoints (either standardWritePerformer for

     * standard writes or counterWritePerformer for counter writes).

     * @param callback an optional callback to be run if and when the write is

     * successful.

     */

    public static AbstractWriteResponseHandler performWrite(IMutation mutation,

                                                            ConsistencyLevel consistency_level,

                                                            String localDataCenter,

                                                            WritePerformer performer,

                                                            Runnable callback,

                                                            WriteType writeType)

    throws UnavailableException, OverloadedException

    {

        String keyspaceName = mutation.getKeyspaceName();

        AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy();

        Token tk = StorageService.getPartitioner().getToken(mutation.key());

        List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);

        Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);

        AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, callback, writeType);

        // exit early if we can't fulfill the CL at this time

        responseHandler.assureSufficientLiveNodes();

        performer.apply(mutation, Iterables.concat(naturalEndpoints, pendingEndpoints), responseHandler, localDataCenter, consistency_level);

        return responseHandler;

    }

    // same as above except does not initiate writes (but does perform availability checks).

    private static WriteResponseHandlerWrapper wrapResponseHandler(Mutation mutation, ConsistencyLevel consistency_level, WriteType writeType)

    {

        AbstractReplicationStrategy rs = Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy();

        String keyspaceName = mutation.getKeyspaceName();

        Token tk = StorageService.getPartitioner().getToken(mutation.key());

        List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);

        Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);

        AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType);

        return new WriteResponseHandlerWrapper(responseHandler, mutation);

    }

    // used by atomic_batch_mutate to decouple availability check from the write itself, caches consistency level and endpoints.

    private static class WriteResponseHandlerWrapper

    {

        final AbstractWriteResponseHandler handler;

        final Mutation mutation;

        WriteResponseHandlerWrapper(AbstractWriteResponseHandler handler, Mutation mutation)

        {

            this.handler = handler;

            this.mutation = mutation;

        }

    }

    /*

     * Replicas are picked manually:

     * - replicas should be alive according to the failure detector

     * - replicas should be in the local datacenter

     * - choose min(2, number of qualifying candiates above)

     * - allow the local node to be the only replica only if it's a single-node DC

     */

    private static Collection<InetAddress> getBatchlogEndpoints(String localDataCenter, ConsistencyLevel consistencyLevel)

    throws UnavailableException

    {

        TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology();

        Multimap<String, InetAddress> localEndpoints = HashMultimap.create(topology.getDatacenterRacks().get(localDataCenter));

        String localRack = DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress());

        Collection<InetAddress> chosenEndpoints = new BatchlogManager.EndpointFilter(localRack, localEndpoints).filter();

        if (chosenEndpoints.isEmpty())

        {

            if (consistencyLevel == ConsistencyLevel.ANY)

                return Collections.singleton(FBUtilities.getBroadcastAddress());

            throw new UnavailableException(ConsistencyLevel.ONE, 1, 0);

        }

        return chosenEndpoints;

    }

    /**

     * Send the mutations to the right targets, write it locally if it corresponds or writes a hint when the node

     * is not available.

     *

     * Note about hints:

     *

     * | Hinted Handoff | Consist. Level |

     * | on             |       >=1      | --> wait for hints. We DO NOT notify the handler with handler.response() for hints;

     * | on             |       ANY      | --> wait for hints. Responses count towards consistency.

     * | off            |       >=1      | --> DO NOT fire hints. And DO NOT wait for them to complete.

     * | off            |       ANY      | --> DO NOT fire hints. And DO NOT wait for them to complete.

     *

     * @throws OverloadedException if the hints cannot be written/enqueued

     */

    public static void sendToHintedEndpoints(final Mutation mutation,

                                             Iterable<InetAddress> targets,

                                             AbstractWriteResponseHandler responseHandler,

                                             String localDataCenter)

    throws OverloadedException

    {

        // extra-datacenter replicas, grouped by dc

        Map<String, Collection<InetAddress>> dcGroups = null;

        // only need to create a Message for non-local writes

        MessageOut<Mutation> message = null;

        boolean insertLocal = false;

        mutation.retain();

        try       

        {

            for (InetAddress destination : targets)

            {

                // avoid OOMing due to excess hints.  we need to do this check even for "live" nodes, since we can

                // still generate hints for those if it's overloaded or simply dead but not yet known-to-be-dead.

                // The idea is that if we have over maxHintsInProgress hints in flight, this is probably due to

                // a small number of nodes causing problems, so we should avoid shutting down writes completely to

                // healthy nodes.  Any node with no hintsInProgress is considered healthy.

                if (StorageMetrics.totalHintsInProgress.count() > maxHintsInProgress

                        && (getHintsInProgressFor(destination).get() > 0 && shouldHint(destination)))

                {

                    throw new OverloadedException("Too many in flight hints: " + StorageMetrics.totalHintsInProgress.count());

                }

                if (FailureDetector.instance.isAlive(destination))

                {

                    if (destination.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS)

                    {

                        insertLocal = true;

                    } else

                    {

                        // belongs on a different server

                        if (message == null)

                            message = mutation.createMessage();

                        String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination);

                        // direct writes to local DC or old Cassandra versions

                        // (1.1 knows how to forward old-style String message IDs; updated to int in 2.0)

                        if (localDataCenter.equals(dc))

                        {

                            MessagingService.instance().sendRR(message, destination, responseHandler, true);

                        } else

                        {

                            Collection<InetAddress> messages = (dcGroups != null) ? dcGroups.get(dc) : null;

                            if (messages == null)

                            {

                                messages = new ArrayList<InetAddress>(3); // most DCs will have <= 3 replicas

                                if (dcGroups == null)

                                    dcGroups = new HashMap<String, Collection<InetAddress>>();

                                dcGroups.put(dc, messages);

                            }

                            messages.add(destination);

                        }

                    }

                } else

                {

                    if (!shouldHint(destination))

                        continue;

                    // Schedule a local hint

                    submitHint(mutation, destination, responseHandler);

                }

            }

            if (insertLocal)

                insertLocal(mutation, responseHandler);

            if (dcGroups != null)

            {

                // for each datacenter, send the message to one node to relay the write to other replicas

                if (message == null)

                    message = mutation.createMessage();

                for (Collection<InetAddress> dcTargets : dcGroups.values())

                    sendMessagesToNonlocalDC(message, dcTargets, responseHandler);

            }

        }

        finally

        {

            mutation.release();

        }

    }

    private static AtomicInteger getHintsInProgressFor(InetAddress destination)

    {

        try

        {

            return hintsInProgress.load(destination);

        }

        catch (Exception e)

        {

            throw new AssertionError(e);

        }

    }

    public static Future<Void> submitHint(final Mutation mutation,

                                          final InetAddress target,

                                          final AbstractWriteResponseHandler responseHandler)

    {

        // local write that time out should be handled by LocalMutationRunnable

        assert !target.equals(FBUtilities.getBroadcastAddress()) : target;

        mutation.retain();

        HintRunnable runnable = new HintRunnable(target)

        {

            public void runMayThrow()

            {

                try

                {

                    int ttl = HintedHandOffManager.calculateHintTTL(mutation);

                    if (ttl > 0)

                    {

                        logger.debug("Adding hint for {}", target);

                        writeHintForMutation(mutation, System.currentTimeMillis(), ttl, target);

                        // Notify the handler only for CL == ANY

                        if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY)

                            responseHandler.response(null);

                    } else

                    {

                        logger.debug("Skipped writing hint for {} (ttl {})", target, ttl);

                    }

                }

                finally

                {

                    mutation.release();

                }

            }

        };

        return submitHint(runnable);

    }

    private static Future<Void> submitHint(HintRunnable runnable)

    {

        StorageMetrics.totalHintsInProgress.inc();

        getHintsInProgressFor(runnable.target).incrementAndGet();

        return (Future<Void>) StageManager.getStage(Stage.MUTATION).submit(runnable);

    }

    /**

     * @param now current time in milliseconds - relevant for hint replay handling of truncated CFs

     */

    public static void writeHintForMutation(Mutation mutation, long now, int ttl, InetAddress target)

    {

        assert ttl > 0;

        UUID hostId = StorageService.instance.getTokenMetadata().getHostId(target);

        assert hostId != null : "Missing host ID for " + target.getHostAddress();

        HintedHandOffManager.instance.hintFor(mutation, now, ttl, hostId).apply();

        StorageMetrics.totalHints.inc();

    }

    private static void sendMessagesToNonlocalDC(MessageOut<? extends IMutation> message, Collection<InetAddress> targets, AbstractWriteResponseHandler handler)

    {

        Iterator<InetAddress> iter = targets.iterator();

        InetAddress target = iter.next();

        // Add the other destinations of the same message as a FORWARD_HEADER entry

        DataOutputBuffer out = new DataOutputBuffer();

        try

        {

            out.writeInt(targets.size() - 1);

            while (iter.hasNext())

            {

                InetAddress destination = iter.next();

                CompactEndpointSerializationHelper.serialize(destination, out);

                int id = MessagingService.instance().addCallback(handler,

                                                                 message,

                                                                 destination,

                                                                 message.getTimeout(),

                                                                 handler.consistencyLevel,

                                                                 true);

                out.writeInt(id);

                logger.trace("Adding FWD message to {}@{}", id, destination);

            }

            message = message.withParameter(Mutation.FORWARD_TO, out.getData());

            // send the combined message + forward headers

            int id = MessagingService.instance().sendRR(message, target, handler, true);

            logger.trace("Sending message to {}@{}", id, target);

        }

        catch (IOException e)

        {

            // DataOutputBuffer is in-memory, doesn't throw IOException

            throw new AssertionError(e);

        }

    }

    private static void insertLocal(final Mutation mutation, final AbstractWriteResponseHandler responseHandler)

    {

        mutation.retain();

        StageManager.getStage(Stage.MUTATION).maybeExecuteImmediately(new LocalMutationRunnable()

        {

            public void runMayThrow()

            {

                try

                {

                    IMutation processed = SinkManager.processWriteRequest(mutation);

                    if (processed != null)

                    {

                        ((Mutation) processed).apply();

                        responseHandler.response(null);

                    }

                }

                finally

                {

                    mutation.release();

                }

            }

        });

    }

    /**

     * Handle counter mutation on the coordinator host.

     *

     * A counter mutation needs to first be applied to a replica (that we'll call the leader for the mutation) before being

     * replicated to the other endpoint. To achieve so, there is two case:

     *   1) the coordinator host is a replica: we proceed to applying the update locally and replicate throug

     *   applyCounterMutationOnCoordinator

     *   2) the coordinator is not a replica: we forward the (counter)mutation to a chosen replica (that will proceed through

     *   applyCounterMutationOnLeader upon receive) and wait for its acknowledgment.

     *

     * Implementation note: We check if we can fulfill the CL on the coordinator host even if he is not a replica to allow

     * quicker response and because the WriteResponseHandlers don't make it easy to send back an error. We also always gather

     * the write latencies at the coordinator node to make gathering point similar to the case of standard writes.

     */

    public static AbstractWriteResponseHandler mutateCounter(CounterMutation cm, String localDataCenter) throws UnavailableException, OverloadedException

    {

        InetAddress endpoint = findSuitableEndpoint(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency());

        if (endpoint.equals(FBUtilities.getBroadcastAddress()))

        {

            return applyCounterMutationOnCoordinator(cm, localDataCenter);

        }

        else

        {

            // Exit now if we can't fulfill the CL here instead of forwarding to the leader replica

            String keyspaceName = cm.getKeyspaceName();

            AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy();

            Token tk = StorageService.getPartitioner().getToken(cm.key());

            List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);

            Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);

            rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, cm.consistency(), null, WriteType.COUNTER).assureSufficientLiveNodes();

            // Forward the actual update to the chosen leader replica

            AbstractWriteResponseHandler responseHandler = new WriteResponseHandler(endpoint, WriteType.COUNTER);

            Tracing.trace("Enqueuing counter update to {}", endpoint);

            MessagingService.instance().sendRR(cm.makeMutationMessage(), endpoint, responseHandler, false);

            return responseHandler;

        }

    }

    /**

     * Find a suitable replica as leader for counter update.

     * For now, we pick a random replica in the local DC (or ask the snitch if

     * there is no replica alive in the local DC).

     * TODO: if we track the latency of the counter writes (which makes sense

     * contrarily to standard writes since there is a read involved), we could

     * trust the dynamic snitch entirely, which may be a better solution. It