Package com.taobao.metamorphosis.server.store

Source Code of com.taobao.metamorphosis.server.store.MessageStoreUnitTest

/*
* (C) 2007-2012 Alibaba Group Holding Limited.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* Authors:
*   wuhua <wq163@163.com> , boyan <killme2008@gmail.com>
*/
package com.taobao.metamorphosis.server.store;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.taobao.metamorphosis.Message;
import com.taobao.metamorphosis.MessageAccessor;
import com.taobao.metamorphosis.exception.InvalidMessageException;
import com.taobao.metamorphosis.network.PutCommand;
import com.taobao.metamorphosis.server.store.MessageStore.Segment;
import com.taobao.metamorphosis.server.store.MessageStore.SegmentList;
import com.taobao.metamorphosis.server.utils.MetaConfig;
import com.taobao.metamorphosis.utils.IdWorker;
import com.taobao.metamorphosis.utils.MessageUtils;
import com.taobao.metamorphosis.utils.MessageUtils.DecodedMessage;
import com.taobao.metamorphosis.utils.test.ConcurrentTestCase;
import com.taobao.metamorphosis.utils.test.ConcurrentTestTask;


public class MessageStoreUnitTest {
    private static final int MSG_COUNT = 10;
    private MessageStore messageStore;
    private final String topic = "test";
    private final int partition = 1;
    private MetaConfig metaConfig;
    private DeletePolicy deletePolicy;
    private IdWorker idWorker;


    @Before
    public void setUp() throws Exception {
        final String tmpPath = System.getProperty("java.io.tmpdir");
        this.metaConfig = new MetaConfig();
        this.metaConfig.setDataPath(tmpPath);
        final PutCommand cmd1 = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        this.metaConfig.setUnflushThreshold(1);
        // ���ƴ�10����Ϣ��roll�ļ�
        this.metaConfig.setMaxSegmentSize(MessageUtils.makeMessageBuffer(1, cmd1).capacity() * MSG_COUNT);
        this.idWorker = new IdWorker(0);
        this.clearTopicPartDir();
        this.deletePolicy = new DiscardDeletePolicy();
        ((DiscardDeletePolicy) this.deletePolicy).setMaxReservedTime(Integer.MAX_VALUE);
        this.messageStore = new MessageStore(this.topic, this.partition, this.metaConfig, this.deletePolicy);
    }


    @After
    public void clearTopicPartDir() throws Exception {
        if (this.messageStore != null) {
            this.messageStore.close();
        }
        final File topicPartDir =
                new File(this.metaConfig.getDataPath() + File.separator + this.topic + "-" + this.partition);
        if (topicPartDir.exists()) {
            for (final File file : topicPartDir.listFiles()) {
                file.delete();
            }
        }
    }


    @Test
    public void testSegmentContants() throws Exception {
        final File file = new File("testSegmentContants.test");
        if (!file.exists()) {
            file.createNewFile();
        }
        final Segment segment = new Segment(0, file);
        try {
            segment.fileMessageSet.setHighWaterMarker(1024);
            assertTrue(segment.contains(0));
            assertFalse(segment.contains(1024));
            assertFalse(segment.contains(1025));
            assertFalse(segment.contains(2048));
            assertTrue(segment.contains(1));
            assertTrue(segment.contains(100));
            assertTrue(segment.contains(512));
            assertTrue(segment.contains(1023));
        }
        finally {
            if (segment != null) {
                segment.fileMessageSet.close();
            }
            file.delete();
        }
    }


    @Test
    public void testAppendSegmentDeleteSegment() throws Exception {
        final SegmentList segmentList = new SegmentList();
        assertEquals(0, segmentList.contents.get().length);
        final File file = new File("testAppendSegmentDeleteSegment.test");
        if (!file.exists()) {
            file.createNewFile();
        }
        final Segment segment1 = new Segment(0, file);
        final Segment segment2 = new Segment(1024, file);
        final Segment segment3 = new Segment(2048, file);
        try {
            segmentList.append(segment1);
            assertEquals(1, segmentList.contents.get().length);
            assertSame(segment1, segmentList.first());
            assertSame(segment1, segmentList.last());
            segmentList.append(segment2);
            assertEquals(2, segmentList.contents.get().length);
            assertSame(segment1, segmentList.first());
            assertSame(segment2, segmentList.last());
            segmentList.append(segment3);
            assertEquals(3, segmentList.contents.get().length);
            assertSame(segment1, segmentList.first());
            assertSame(segment3, segmentList.last());

            segmentList.delete(segment1);
            assertEquals(2, segmentList.contents.get().length);
            assertSame(segment2, segmentList.first());
            assertSame(segment3, segmentList.last());

            segmentList.delete(segment3);
            assertEquals(1, segmentList.contents.get().length);
            assertSame(segment2, segmentList.first());
            assertSame(segment2, segmentList.last());
            // delete not existing
            segmentList.delete(segment3);
            assertEquals(1, segmentList.contents.get().length);
            assertSame(segment2, segmentList.first());
            assertSame(segment2, segmentList.last());

            segmentList.delete(segment2);
            assertEquals(0, segmentList.contents.get().length);
            assertNull(segmentList.first());
            assertNull(segmentList.last());
        }
        finally {
            if (segment1 != null) {
                segment1.fileMessageSet.close();
            }
            file.delete();
        }
    }


    @Test
    public void testAppendMessages() throws Exception {
        final PutCommand cmd1 = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        final PutCommand cmd2 = new PutCommand(this.topic, this.partition, "world".getBytes(), null, 0, 0);
        final long id1 = this.idWorker.nextId();
        final long id2 = this.idWorker.nextId();
        this.messageStore.append(id1, cmd1, new AppendCallback() {
            @Override
            public void appendComplete(final Location location) {
                if (0 != location.getOffset()) {
                    throw new RuntimeException();
                }
            }
        });
        this.messageStore.flush();
        final long size = this.messageStore.getSegments().last().size();
        this.messageStore.append(id2, cmd2, new AppendCallback() {
            @Override
            public void appendComplete(final Location location) {
                assertEquals(size, location.getOffset());
                if (size != location.getOffset()) {
                    throw new RuntimeException();
                }
            }
        });
        this.messageStore.flush();

        this.assertMessages(id1, id2);

    }


    @Test
    public void testConcurrentAppendMessages() throws Exception {
        System.out.println("Begin concurrent test....");
        this.metaConfig.setMaxSegmentSize(1024 * 1024 * 16);
        ConcurrentTestCase testCase = new ConcurrentTestCase(80, 1000, new ConcurrentTestTask() {

            @Override
            public void run(int index, int times) throws Exception {
                final PutCommand cmd =
                        new PutCommand(MessageStoreUnitTest.this.topic, MessageStoreUnitTest.this.partition,
                            new byte[1024], null, 0, 0);
                final long id = MessageStoreUnitTest.this.idWorker.nextId();
                final CountDownLatch latch = new CountDownLatch(1);
                MessageStoreUnitTest.this.messageStore.append(id, cmd, new AppendCallback() {
                    @Override
                    public void appendComplete(final Location location) {
                        if (location.getOffset() < 0) {
                            throw new RuntimeException();
                        }
                        else {
                            latch.countDown();
                        }
                    }

                });
                try {
                    latch.await();
                }
                catch (InterruptedException e) {
                    // ignore
                }
            }
        });
        testCase.start();
        System.out.println("Appended 80000 messages,cost:" + testCase.getDurationInMillis() / 1000 + " seconds");
        assertEquals(80000, this.messageStore.getMessageCount());

    }


    @Test
    // ����һ��ָ��offset��MessageStore����д��Ϣ
    public void testAppendMessages_toOffset() throws Exception {

        // �����һ��
        this.clearTopicPartDir();

        final int offset = 2048;
        this.messageStore = new MessageStore(this.topic, this.partition, this.metaConfig, this.deletePolicy, offset);
        final PutCommand cmd1 = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        final PutCommand cmd2 = new PutCommand(this.topic, this.partition, "world".getBytes(), null, 0, 0);
        final long id1 = this.idWorker.nextId();
        final long id2 = this.idWorker.nextId();
        this.messageStore.append(id1, cmd1, new AppendCallback() {
            @Override
            public void appendComplete(final Location location) {
                assertEquals(2048, location.getOffset());
                if (2048 != location.getOffset()) {
                    throw new RuntimeException();
                }
            }
        });
        this.messageStore.flush();
        final long size = this.messageStore.getSegments().last().size();
        this.messageStore.append(id2, cmd2, new AppendCallback() {

            @Override
            public void appendComplete(final Location location) {
                assertEquals(offset + size, location.getOffset());
                if (offset + size != location.getOffset()) {
                    throw new RuntimeException();
                }
            }
        });
        this.messageStore.flush();

        final File partDir = this.messageStore.getPartitionDir();
        final File[] logs = partDir.listFiles();
        assertEquals(1, logs.length);
        assertTrue(logs[0].exists());
        assertEquals(this.messageStore.nameFromOffset(offset), logs[0].getName());
        final FileChannel channel = new RandomAccessFile(logs[0], "rw").getChannel();
        try {
            final ByteBuffer buf = ByteBuffer.allocate((int) channel.size());
            while (buf.hasRemaining()) {
                channel.read(buf);
            }
            buf.flip();

            final DecodedMessage decodedMessage1 = MessageUtils.decodeMessage(this.topic, buf.array(), 0);
            final DecodedMessage decodedMessage2 =
                    MessageUtils.decodeMessage(this.topic, buf.array(), decodedMessage1.newOffset);
            final Message msg1 = new Message(this.topic, "hello".getBytes());
            MessageAccessor.setId(msg1, id1);
            final Message msg2 = new Message(this.topic, "world".getBytes());
            MessageAccessor.setId(msg2, id2);
            assertEquals(msg1, decodedMessage1.message);
            assertEquals(msg2, decodedMessage2.message);
        }
        finally {
            channel.close();
        }
    }


    @Test
    public void testGetNearestOffset() throws Exception {
        final SegmentList segmentList = new SegmentList();
        assertEquals(0, segmentList.contents.get().length);
        final File file = new File("testGetNearestOffset.test");
        if (!file.exists()) {
            file.createNewFile();
        }
        final Segment segment1 = new Segment(0, file);
        segment1.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment2 = new Segment(1024, file);
        segment2.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment3 = new Segment(2048, file);
        segment3.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment4 = new Segment(3072, file);
        segment4.fileMessageSet.setHighWaterMarker(1024);

        segmentList.append(segment1);
        segmentList.append(segment2);
        segmentList.append(segment3);
        segmentList.append(segment4);

        try {
            assertEquals(0, this.messageStore.getNearestOffset(-100, segmentList));
            assertEquals(0, this.messageStore.getNearestOffset(0, segmentList));
            assertEquals(0, this.messageStore.getNearestOffset(100, segmentList));
            assertEquals(0, this.messageStore.getNearestOffset(1023, segmentList));
            assertEquals(1024, this.messageStore.getNearestOffset(1024, segmentList));
            assertEquals(1024, this.messageStore.getNearestOffset(2000, segmentList));
            assertEquals(2048, this.messageStore.getNearestOffset(2048, segmentList));
            assertEquals(2048, this.messageStore.getNearestOffset(2049, segmentList));
            assertEquals(2048, this.messageStore.getNearestOffset(2536, segmentList));
            assertEquals(3072, this.messageStore.getNearestOffset(3072, segmentList));
            assertEquals(3072, this.messageStore.getNearestOffset(3073, segmentList));
            assertEquals(3072 + 1024, this.messageStore.getNearestOffset(4096, segmentList));
            assertEquals(3072 + 1024, this.messageStore.getNearestOffset(16 * 1024, segmentList));
        }
        finally {
            file.delete();
            segment1.fileMessageSet.close();
            segment2.fileMessageSet.close();
            segment3.fileMessageSet.close();
            segment4.fileMessageSet.close();
        }
    }


    @Test
    public void testFindSegment() throws Exception {
        final SegmentList segmentList = new SegmentList();
        assertEquals(0, segmentList.contents.get().length);
        final File file = new File("testFindSegment.test");
        if (!file.exists()) {
            file.createNewFile();
        }
        final Segment segment1 = new Segment(0, file);
        segment1.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment2 = new Segment(1024, file);
        segment2.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment3 = new Segment(2048, file);
        segment3.fileMessageSet.setHighWaterMarker(1024);
        final Segment segment4 = new Segment(3072, file);
        segment4.fileMessageSet.setHighWaterMarker(1024);

        segmentList.append(segment1);
        segmentList.append(segment2);
        segmentList.append(segment3);
        segmentList.append(segment4);
        try {
            assertSame(segment1, this.messageStore.findSegment(segmentList.view(), 0));
            assertSame(segment1, this.messageStore.findSegment(segmentList.view(), 1));
            assertSame(segment1, this.messageStore.findSegment(segmentList.view(), 1023));
            assertSame(segment2, this.messageStore.findSegment(segmentList.view(), 1024));
            assertSame(segment2, this.messageStore.findSegment(segmentList.view(), 1536));
            assertSame(segment3, this.messageStore.findSegment(segmentList.view(), 2048));
            assertSame(segment3, this.messageStore.findSegment(segmentList.view(), 2049));
            assertSame(segment3, this.messageStore.findSegment(segmentList.view(), 3000));
            assertSame(segment4, this.messageStore.findSegment(segmentList.view(), 3072));
            assertSame(segment4, this.messageStore.findSegment(segmentList.view(), 3073));
            assertNull(this.messageStore.findSegment(segmentList.view(), 4097));
            assertNull(this.messageStore.findSegment(segmentList.view(), 4098));
            assertNull(this.messageStore.findSegment(segmentList.view(), 16 * 1024));
            try {
                this.messageStore.findSegment(segmentList.view(), -1);
                fail();
            }
            catch (final ArrayIndexOutOfBoundsException e) {
                assertTrue(true);
            }
        }
        finally {
            file.delete();
            segment1.fileMessageSet.close();
            segment2.fileMessageSet.close();
            segment3.fileMessageSet.close();
            segment4.fileMessageSet.close();
        }

    }


    @Test
    public void testAppendMessagesRoll() throws Exception {
        final PutCommand req = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        final AtomicLong size = new AtomicLong(0);
        for (int i = 0; i < MSG_COUNT; i++) {
            if (i == 0) {
                this.messageStore.append(this.idWorker.nextId(), req, new AppendCallback() {

                    @Override
                    public void appendComplete(final Location location) {
                        assertEquals(0, location.getOffset());
                        if (0 != location.getOffset()) {
                            throw new RuntimeException();
                        }
                    }
                });
                this.messageStore.flush();
                size.set(this.messageStore.getSegments().last().size());
            }
            else {
                final int j = i;
                this.messageStore.append(this.idWorker.nextId(), req, new AppendCallback() {

                    @Override
                    public void appendComplete(final Location location) {
                        assertEquals(size.get() * j, location.getOffset());
                        if (size.get() * j != location.getOffset()) {
                            throw new RuntimeException();
                        }
                    }
                });
                this.messageStore.flush();
                // assertEquals(size * i,
                // this.messageStore.append(this.idWorker.nextId(),
                // req).getOffset());
            }

        }

        final File partDir = this.messageStore.getPartitionDir();
        final File[] logs = partDir.listFiles();
        assertEquals(2, logs.length);
        Arrays.sort(logs, new Comparator<File>() {

            @Override
            public int compare(final File o1, final File o2) {
                final long l =
                        Long.parseLong(o1.getName().split("\\.")[0]) - Long.parseLong(o2.getName().split("\\.")[0]);
                return l == 0 ? 0 : l > 0 ? 1 : -1;
            }

        });
        assertEquals(this.messageStore.nameFromOffset(0), logs[0].getName());
        assertEquals(this.messageStore.nameFromOffset(this.metaConfig.getMaxSegmentSize()), logs[1].getName());

        final FileChannel channel1 = new RandomAccessFile(logs[0], "rw").getChannel();
        final FileChannel channel2 = new RandomAccessFile(logs[1], "rw").getChannel();
        try {
            assertEquals(this.metaConfig.getMaxSegmentSize(), channel1.size());
            assertEquals(0, channel2.size());
        }
        finally {
            channel1.close();
            channel2.close();
        }
    }


    @Test
    public void testAppendMessagesCloseRecover() throws Exception {
        final PutCommand req = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        final AtomicLong size = new AtomicLong(0);
        for (int i = 0; i < 100; i++) {
            if (i == 0) {
                this.messageStore.append(this.idWorker.nextId(), req, new AppendCallback() {

                    @Override
                    public void appendComplete(final Location location) {
                        assertEquals(0, location.getOffset());
                        if (0 != location.getOffset()) {
                            throw new RuntimeException();
                        }
                    }
                });
                this.messageStore.flush();
                size.set(this.messageStore.getSegments().last().size());
            }
            else {
                final int j = i;
                this.messageStore.append(this.idWorker.nextId(), req, new AppendCallback() {

                    @Override
                    public void appendComplete(final Location location) {
                        assertEquals(size.get() * j, location.getOffset());
                        if (size.get() * j != location.getOffset()) {
                            throw new RuntimeException();
                        }
                    }
                });
                this.messageStore.flush();
            }
        }
        this.messageStore.flush();
        final Segment[] segments = this.messageStore.getSegments().view();
        final int oldSegmentCount = segments.length;
        final String lastSegmentName = segments[segments.length - 1].file.getName();
        final long lastSegmentSize = segments[segments.length - 1].size();

        this.messageStore.close();
        // recover
        this.messageStore = new MessageStore(this.topic, this.partition, this.metaConfig, this.deletePolicy);

        final Segment[] newSegments = this.messageStore.getSegments().view();
        assertEquals(oldSegmentCount, newSegments.length);
        assertEquals(lastSegmentName, newSegments[newSegments.length - 1].file.getName());
        assertEquals(lastSegmentSize, newSegments[newSegments.length - 1].size());

        Segment prev = null;
        for (final Segment s : newSegments) {
            if (prev != null) {
                assertEquals(s.start, prev.start + prev.size());
            }
            prev = s;
        }
    }


    @Test
    public void testSlice() throws Exception {
        final PutCommand req = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        for (int i = 0; i < MSG_COUNT; i++) {
            this.messageStore.append(this.idWorker.nextId(), req, null);
        }
        FileMessageSet subSet = (FileMessageSet) this.messageStore.slice(0, this.metaConfig.getMaxSegmentSize() - 10);
        assertEquals(0, subSet.getOffset());
        assertEquals(this.metaConfig.getMaxSegmentSize() - 10, subSet.getSizeInBytes());

        subSet = (FileMessageSet) this.messageStore.slice(0, this.metaConfig.getMaxSegmentSize() + 10);
        assertEquals(0, subSet.getOffset());
        assertEquals(this.metaConfig.getMaxSegmentSize(), subSet.getSizeInBytes());

        subSet =
                (FileMessageSet) this.messageStore.slice(this.metaConfig.getMaxSegmentSize() - 10,
                    this.metaConfig.getMaxSegmentSize() + 10);
        assertEquals(this.metaConfig.getMaxSegmentSize() - 10, subSet.getOffset());
        assertEquals(10, subSet.getSizeInBytes());

        assertNull(this.messageStore.slice(10000, 1024));
    }


    @Test(expected = ArrayIndexOutOfBoundsException.class)
    public void testSliceIndexOutOfBounds() throws Exception {
        this.messageStore.slice(-1, 1024);
    }


    @Test
    public void testRunDeletePolicy() throws Exception {
        ((DiscardDeletePolicy) this.deletePolicy).setMaxReservedTime(1000);
        final PutCommand req = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        for (int i = 0; i < MSG_COUNT; i++) {
            this.messageStore.append(this.idWorker.nextId(), req, null);
        }

        final File partDir = this.messageStore.getPartitionDir();
        assertEquals(2, partDir.listFiles().length);
        assertEquals(2, this.messageStore.getSegments().view().length);
        Thread.sleep(1500);
        this.messageStore.runDeletePolicy();
        assertEquals(1, partDir.listFiles().length);
        assertEquals(1, this.messageStore.getSegments().view().length);

        Thread.sleep(1500);
        this.messageStore.runDeletePolicy();
        assertEquals(1, partDir.listFiles().length);
        assertEquals(1, this.messageStore.getSegments().view().length);
    }


    @Test
    public void testAppendMany() throws Exception {
        final PutCommand cmd1 = new PutCommand(this.topic, this.partition, "hello".getBytes(), null, 0, 0);
        final PutCommand cmd2 = new PutCommand(this.topic, this.partition, "world".getBytes(), null, 0, 0);
        final long id1 = this.idWorker.nextId();
        final long id2 = this.idWorker.nextId();

        final List<Long> ids = new ArrayList<Long>();
        ids.add(id1);
        ids.add(id2);
        final List<PutCommand> cmds = new ArrayList<PutCommand>();
        cmds.add(cmd1);
        cmds.add(cmd2);

        this.messageStore.append(ids, cmds, new AppendCallback() {

            @Override
            public void appendComplete(final Location location) {
                assertEquals(0, location.getOffset());
                if (0 != location.getOffset()) {
                    throw new RuntimeException();
                }
            }
        });
        this.messageStore.flush();

        this.assertMessages(id1, id2);
    }


    private void assertMessages(final long id1, final long id2) throws FileNotFoundException, IOException,
    InvalidMessageException {
        final File partDir = this.messageStore.getPartitionDir();
        final File[] logs = partDir.listFiles();
        assertEquals(1, logs.length);
        assertEquals(this.messageStore.nameFromOffset(0), logs[0].getName());
        final FileChannel channel = new RandomAccessFile(logs[0], "rw").getChannel();
        try {
            final ByteBuffer buf = ByteBuffer.allocate((int) channel.size());
            while (buf.hasRemaining()) {
                channel.read(buf);
            }
            buf.flip();

            final DecodedMessage decodedMessage1 = MessageUtils.decodeMessage(this.topic, buf.array(), 0);
            final DecodedMessage decodedMessage2 =
                    MessageUtils.decodeMessage(this.topic, buf.array(), decodedMessage1.newOffset);
            final Message msg1 = new Message(this.topic, "hello".getBytes());
            MessageAccessor.setId(msg1, id1);
            final Message msg2 = new Message(this.topic, "world".getBytes());
            MessageAccessor.setId(msg2, id2);
            assertEquals(msg1, decodedMessage1.message);
            assertEquals(msg2, decodedMessage2.message);
        }
        finally {
            channel.close();
        }
    }

}
TOP

Related Classes of com.taobao.metamorphosis.server.store.MessageStoreUnitTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.