Examples of org.apache.cassandra.db.context.CounterContext

org.apache.cassandra.db.context.CounterContext
An implementation of a partitioned counter context. A context is primarily a list of tuples (node id, clock, count) -- called shard in the following. But with some shard are flagged as delta (with special resolution rules in merge()). The data structure has two parts: a) a header containing the lists of "delta" (a list of references to the second parts) b) a list of shard -- (node id, logical clock, count) tuples -- (the so-called 'body' below) The exact layout is: | header | body | context : |--|------|----------| ^ ^ | list of indices in the body list (2*#elt bytes) #elt in rest of header (2 bytes) The body layout being: body: |----|----|----|----|----|----|.... ^ ^ ^ ^ ^ ^ | | count_1 | | count_2 | clock_1 | clock_2 nodeid_1 nodeid_2 The rules when merging two shard with the same nodeid are: - delta + delta = sum counts (and logical clock) - delta + other = keep the delta one - other + other = keep the shard with highest logical clock

    @Test
    public void testTransferTableCounter() throws Exception
    {
        final Table table = Table.open("Keyspace1");
        final ColumnFamilyStore cfs = table.getColumnFamilyStore("Counter1");
        final CounterContext cc = new CounterContext();


        final Map<String, ColumnFamily> cleanedEntries = new HashMap<String, ColumnFamily>();


        List<String> keys = createAndTransfer(table, cfs, new Mutator()
        {
            /** Creates a new SSTable per key: all will be merged before streaming. */
            public void mutate(String key, String col, long timestamp) throws Exception
            {
                Map<String, ColumnFamily> entries = new HashMap<String, ColumnFamily>();
                ColumnFamily cf = ColumnFamily.create(cfs.metadata);
                ColumnFamily cfCleaned = ColumnFamily.create(cfs.metadata);
                CounterContext.ContextState state = CounterContext.ContextState.allocate(4, 1);
                state.writeElement(NodeId.fromInt(2), 9L, 3L, true);
                state.writeElement(NodeId.fromInt(4), 4L, 2L);
                state.writeElement(NodeId.fromInt(6), 3L, 3L);
                state.writeElement(NodeId.fromInt(8), 2L, 4L);
                cf.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                                               state.context,
                                               timestamp));
                cfCleaned.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                                                      cc.clearAllDelta(state.context),
                                                      timestamp));


                entries.put(key, cf);
                cleanedEntries.put(key, cfCleaned);
                cfs.addSSTable(SSTableUtils.prepare()

View Full Code Here

    }


    @Test
    public void testRemoveOldShardFixCorrupted() throws IOException
    {
        CounterContext ctx = CounterContext.instance();
        int now = (int) (System.currentTimeMillis() / 1000);


        // Check that corrupted context created prior to #2968 are fixed by removeOldShards
        CounterId id1 = CounterId.getLocalId();
        CounterId.renewLocalId();
        CounterId id2 = CounterId.getLocalId();


        ContextState state = ContextState.allocate(3, 2);
        state.writeElement(CounterId.fromInt(1), 1, 4, false);
        state.writeElement(id1, 3, 2, true);
        state.writeElement(id2, -100, 5, true); // corrupted!


        assert ctx.total(state.context) == 11;


        try
        {
            ByteBuffer merger = ctx.computeOldShardMerger(state.context, Collections.<CounterId.CounterIdRecord>emptyList(), 0);
            ctx.removeOldShards(ctx.merge(state.context, merger, HeapAllocator.instance), now);
            fail("RemoveOldShards should throw an exception if the current id is non-sensical");
        }
        catch (RuntimeException e) {}


        CounterId.renewLocalId();
        ByteBuffer merger = ctx.computeOldShardMerger(state.context, Collections.<CounterId.CounterIdRecord>emptyList(), 0);
        ByteBuffer cleaned = ctx.removeOldShards(ctx.merge(state.context, merger, HeapAllocator.instance), now);
        assert ctx.total(cleaned) == 11;


        // Check it is not corrupted anymore
        ContextState state2 = new ContextState(cleaned);
        while (state2.hasRemaining())
        {
            assert state2.getClock() >= 0 || state2.getCount() == 0;
            state2.moveToNext();
        }


        // Check that if we merge old and clean on another node, we keep the right count
        ByteBuffer onRemote = ctx.merge(ctx.clearAllDelta(state.context), ctx.clearAllDelta(cleaned), HeapAllocator.instance);
        assert ctx.total(onRemote) == 11;
    }

View Full Code Here

    @Test
    public void testTransferTableCounter() throws Exception
    {
        final Keyspace keyspace = Keyspace.open("Keyspace1");
        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Counter1");
        final CounterContext cc = new CounterContext();


        final Map<String, ColumnFamily> cleanedEntries = new HashMap<>();


        List<String> keys = createAndTransfer(cfs, new Mutator()
        {
            /** Creates a new SSTable per key: all will be merged before streaming. */
            public void mutate(String key, String col, long timestamp) throws Exception
            {
                Map<String, ColumnFamily> entries = new HashMap<>();
                ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                ColumnFamily cfCleaned = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                CounterContext.ContextState state = CounterContext.ContextState.allocate(0, 1, 3, HeapAllocator.instance);
                state.writeLocal(CounterId.fromInt(2), 9L, 3L);
                state.writeRemote(CounterId.fromInt(4), 4L, 2L);
                state.writeRemote(CounterId.fromInt(6), 3L, 3L);
                state.writeRemote(CounterId.fromInt(8), 2L, 4L);
                cf.addColumn(new CounterColumn(ByteBufferUtil.bytes(col), state.context, timestamp));
                cfCleaned.addColumn(new CounterColumn(ByteBufferUtil.bytes(col), cc.clearAllLocal(state.context), timestamp));


                entries.put(key, cf);
                cleanedEntries.put(key, cfCleaned);
                cfs.addSSTable(SSTableUtils.prepare()
                    .ks(keyspace.getName())

View Full Code Here

    @Test
    public void testTransferTableCounter() throws Exception
    {
        final Keyspace keyspace = Keyspace.open("Keyspace1");
        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Counter1");
        final CounterContext cc = new CounterContext();


        final Map<String, ColumnFamily> cleanedEntries = new HashMap<>();


        List<String> keys = createAndTransfer(cfs, new Mutator()
        {
            /** Creates a new SSTable per key: all will be merged before streaming. */
            public void mutate(String key, String col, long timestamp) throws Exception
            {
                Map<String, ColumnFamily> entries = new HashMap<>();
                ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                ColumnFamily cfCleaned = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                CounterContext.ContextState state = CounterContext.ContextState.allocate(4, 1);
                state.writeElement(CounterId.fromInt(2), 9L, 3L, true);
                state.writeElement(CounterId.fromInt(4), 4L, 2L);
                state.writeElement(CounterId.fromInt(6), 3L, 3L);
                state.writeElement(CounterId.fromInt(8), 2L, 4L);
                cf.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                        state.context,
                        timestamp));
                cfCleaned.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                        cc.clearAllDelta(state.context),
                        timestamp));


                entries.put(key, cf);
                cleanedEntries.put(key, cfCleaned);
                cfs.addSSTable(SSTableUtils.prepare()

View Full Code Here

    @Test
    public void testTransferTableCounter() throws Exception
    {
        final Keyspace keyspace = Keyspace.open("Keyspace1");
        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Counter1");
        final CounterContext cc = new CounterContext();


        final Map<String, ColumnFamily> cleanedEntries = new HashMap<>();


        List<String> keys = createAndTransfer(cfs, new Mutator()
        {
            /** Creates a new SSTable per key: all will be merged before streaming. */
            public void mutate(String key, String col, long timestamp) throws Exception
            {
                Map<String, ColumnFamily> entries = new HashMap<>();
                ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
                ColumnFamily cfCleaned = ArrayBackedSortedColumns.factory.create(cfs.metadata);
                CounterContext.ContextState state = CounterContext.ContextState.allocate(0, 1, 3);
                state.writeLocal(CounterId.fromInt(2), 9L, 3L);
                state.writeRemote(CounterId.fromInt(4), 4L, 2L);
                state.writeRemote(CounterId.fromInt(6), 3L, 3L);
                state.writeRemote(CounterId.fromInt(8), 2L, 4L);
                cf.addColumn(new BufferCounterCell(cellname(col), state.context, timestamp));
                cfCleaned.addColumn(new BufferCounterCell(cellname(col), cc.clearAllLocal(state.context), timestamp));


                entries.put(key, cf);
                cleanedEntries.put(key, cfCleaned);
                cfs.addSSTable(SSTableUtils.prepare()
                    .ks(keyspace.getName())

View Full Code Here

    @Test
    public void testTransferTableCounter() throws Exception
    {
        final Keyspace keyspace = Keyspace.open("Keyspace1");
        final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Counter1");
        final CounterContext cc = new CounterContext();


        final Map<String, ColumnFamily> cleanedEntries = new HashMap<>();


        List<String> keys = createAndTransfer(cfs, new Mutator()
        {
            /** Creates a new SSTable per key: all will be merged before streaming. */
            public void mutate(String key, String col, long timestamp) throws Exception
            {
                Map<String, ColumnFamily> entries = new HashMap<>();
                ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                ColumnFamily cfCleaned = TreeMapBackedSortedColumns.factory.create(cfs.metadata);
                CounterContext.ContextState state = CounterContext.ContextState.allocate(4, 1);
                state.writeElement(CounterId.fromInt(2), 9L, 3L, true);
                state.writeElement(CounterId.fromInt(4), 4L, 2L);
                state.writeElement(CounterId.fromInt(6), 3L, 3L);
                state.writeElement(CounterId.fromInt(8), 2L, 4L);
                cf.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                                               state.context,
                                               timestamp));
                cfCleaned.addColumn(new CounterColumn(ByteBufferUtil.bytes(col),
                                                      cc.clearAllDelta(state.context),
                                                      timestamp));


                entries.put(key, cf);
                cleanedEntries.put(key, cfCleaned);
                cfs.addSSTable(SSTableUtils.prepare()

View Full Code Here

    }


    @Test
    public void testRemoveOldShardFixCorrupted() throws IOException
    {
        CounterContext ctx = CounterContext.instance();


        // Check that corrupted context created prior to #2968 are fixed by removeOldShards
        NodeId id1 = NodeId.getLocalId();
        NodeId.renewLocalId();
        NodeId id2 = NodeId.getLocalId();


        ContextState state = ContextState.allocate(3, 2);
        state.writeElement(NodeId.fromInt(1), 1, 4, false);
        state.writeElement(id1, 3, 2, true);
        state.writeElement(id2, -System.currentTimeMillis(), 5, true); // corrupted!


        assert ctx.total(state.context) == 11;


        try
        {
            ctx.removeOldShards(state.context, Integer.MAX_VALUE);
            fail("RemoveOldShards should throw an exception if the current id is non-sensical");
        }
        catch (RuntimeException e) {}


        NodeId.renewLocalId();
        ByteBuffer cleaned = ctx.removeOldShards(state.context, Integer.MAX_VALUE);
        assert ctx.total(cleaned) == 11;


        // Check it is not corrupted anymore
        ContextState state2 = new ContextState(cleaned);
        while (state2.hasRemaining())
        {
            assert state2.getClock() >= 0 || state2.getCount() == 0;
            state2.moveToNext();
        }


        // Check that if we merge old and clean on another node, we keep the right count
        ByteBuffer onRemote = ctx.merge(ctx.clearAllDelta(state.context), ctx.clearAllDelta(cleaned));
        assert ctx.total(onRemote) == 11;
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.cassandra.db.context.CounterContext

org.apache.cassandra.db.CounterMutationTest

org.apache.cassandra.streaming.StreamingTransferTest

org.apache.cassandra.utils.NodeId

org.apache.cassandra.serializers.MarshalException

org.apache.cassandra.db.marshal.MarshalException

java.nio.ByteBuffer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.