Package com.taobao.metamorphosis.server.store

Examples of com.taobao.metamorphosis.server.store.MessageStore$SegmentList


        final int maxSize = 1024;
        final long offset = 10;
        final ByteBuffer msgBuf =
                MessageUtils.makeMessageBuffer(999, new PutCommand(this.topic, partition, "hello world".getBytes(),
                    null, 0, opaque));
        final MessageStore store = this.mocksControl.createMock(MessageStore.class);
        EasyMock.expect(this.storeManager.getMessageStore(this.topic, partition)).andReturn(store);
        final MessageSet set = this.mocksControl.createMock(MessageSet.class);
        EasyMock.expect(store.slice(offset, maxSize)).andReturn(set);
        final GetCommand request = new GetCommand(this.topic, this.group, partition, offset, maxSize, opaque);
        set.read((ByteBuffer) EasyMock.anyObject());
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>() {

            @Override
View Full Code Here


        final int maxSize = 1024;
        final long offset = 10;
        final ByteBuffer msgBuf =
                MessageUtils.makeMessageBuffer(999, new PutCommand(this.topic, partition, "hello world".getBytes(),
                    null, 0, opaque));
        final MessageStore store = this.mocksControl.createMock(MessageStore.class);
        EasyMock.expect(this.storeManager.getMessageStore(this.topic, partition)).andReturn(store);
        final MessageSet set = this.mocksControl.createMock(MessageSet.class);
        EasyMock.expect(store.slice(offset, maxSize)).andReturn(set);
        final GetCommand request = new GetCommand(this.topic, this.group, partition, offset, maxSize, opaque);
        set.read((ByteBuffer) EasyMock.anyObject());
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>() {

            @Override
View Full Code Here

        final byte[] data = new byte[1024];
        final long msgId = 100000L;
        final int flag = MessageFlagUtils.getFlag(null);
        final SyncCommand request = new SyncCommand(this.topic, partition, data, flag, msgId, -1, opaque);

        final MessageStore store = this.mocksControl.createMock(MessageStore.class);
        EasyMock.expect(this.storeManager.getOrCreateMessageStore(this.topic, partition)).andReturn(store);
        final BooleanCommand expectResp =
                new BooleanCommand(HttpStatus.Success, msgId + " " + partition + " " + offset, opaque);
        final AtomicBoolean invoked = new AtomicBoolean(false);
        final PutCallback cb = new PutCallback() {
            @Override
            public void putComplete(final ResponseCommand resp) {
                invoked.set(true);
                if (!expectResp.equals(resp)) {
                    throw new RuntimeException();
                }
            }
        };
        store.append(msgId, request,
            this.commandProcessor.new StoreAppendCallback(partition, this.metaConfig.getBrokerId() + "-" + partition,
                request, msgId, cb));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
View Full Code Here

                return;
            }

            // ʹ��master�������ķ���
            final int partition = request.getPartition();
            final MessageStore store = this.storeManager.getOrCreateMessageStore(request.getTopic(), partition);
            // ʹ��master��������id
            final long messageId = request.getMsgId();
            store.append(messageId, request,
                new StoreAppendCallback(partition, partitionString, request, messageId, cb));
        }
        catch (final Exception e) {
            this.statsManager.statsPutFailed(request.getTopic(), partitionString, 1);
            log.error("Put message failed", e);
View Full Code Here

                }
                return;
            }

            partition = this.getPartition(request);
            final MessageStore store = this.storeManager.getOrCreateMessageStore(request.getTopic(), partition);
            // ����Ƕ�̬��ӵ�topic����Ҫע�ᵽzk
            this.brokerZooKeeper.registerTopicInZk(request.getTopic(), false);
            // ����Ψһid
            final long messageId = this.idWorker.nextId();
            store.append(messageId, request,
                new StoreAppendCallback(partition, partitionString, request, messageId, cb));
        }
        catch (final Exception e) {
            this.statsManager.statsPutFailed(request.getTopic(), partitionString, 1);
            log.error("Put message failed", e);
View Full Code Here

                + ",it closed,");
            return new BooleanCommand(HttpStatus.Forbidden, "Partition[" + this.metaConfig.getBrokerId() + "-"
                    + request.getPartition() + "] has been closed", request.getOpaque());
        }

        final MessageStore store = this.storeManager.getMessageStore(topic, request.getPartition());
        if (store == null) {
            this.statsManager.statsGetMiss(topic, group, 1);
            return new BooleanCommand(HttpStatus.NotFound, "The topic `" + topic + "` in partition `"
                    + request.getPartition() + "` is not exists", request.getOpaque());
        }
        if (request.getMaxSize() <= 0) {
            return new BooleanCommand(HttpStatus.BadRequest, "Bad request,invalid max size:" + request.getMaxSize(),
                request.getOpaque());
        }
        try {
            final MessageSet set =
                    store.slice(request.getOffset(),
                        Math.min(this.metaConfig.getMaxTransferSize(), request.getMaxSize()));
            ConsumerMessageFilter filter = this.consumerFilterManager.findFilter(topic, group);
            if (set != null) {
                if (zeroCopy && filter == null) {
                    set.write(request, ctx);
                    return null;
                }
                else {
                    // refer to the code of line 440 in MessageStore
                    // create two copies of byte array including the byteBuffer
                    // and new bytes
                    // this may not a good use case of Buffer
                    final ByteBuffer byteBuffer =
                            ByteBuffer.allocate(Math.min(this.metaConfig.getMaxTransferSize(), request.getMaxSize()));
                    set.read(byteBuffer);
                    byte[] bytes = this.getBytesFromBuffer(byteBuffer);
                    // If filter is not null,we filter the messages by it.
                    if (filter != null) {
                        MessageIterator it = new MessageIterator(topic, bytes);
                        // reuse the buffer.
                        byteBuffer.clear();
                        while (it.hasNext()) {
                            Message msg = it.next();
                            try {
                                if (filter.accept(group, msg)) {
                                    ByteBuffer msgBuf = it.getCurrentMsgBuf();
                                    // Append current message buffer to result
                                    // buffer.
                                    byteBuffer.put(msgBuf);
                                }
                            }
                            catch (Exception e) {
                                log.error("Filter message for consumer failed,topic=" + topic + ",group=" + group
                                    + ",filterClass=" + filter.getClass().getCanonicalName(), e);
                            }
                        }
                        // re-new the byte array.
                        bytes = this.getBytesFromBuffer(byteBuffer);
                        // All these messages are not acceptable,move forward
                        // offset.
                        if (bytes.length == 0) {
                            return new BooleanCommand(HttpStatus.Moved, String.valueOf(request.getOffset()
                                + it.getOffset()), request.getOpaque());
                        }
                    }
                    return new DataCommand(bytes, request.getOpaque(), true);
                }
            }
            else {
                this.statsManager.statsGetMiss(topic, group, 1);
                this.statsManager.statsGetFailed(topic, group, 1);

                // �������ƫ��������ʵ�����ֵʱ,���ظ��ͻ���ʵ������ƫ����.
                final long maxOffset = store.getMaxOffset();
                final long requestOffset = request.getOffset();
                if (requestOffset > maxOffset
                        && (this.metaConfig.isUpdateConsumerOffsets() || requestOffset == Long.MAX_VALUE)) {
                    log.info("offset[" + requestOffset + "] is exceeded,tell the client real max offset: " + maxOffset
                        + ",topic=" + topic + ",group=" + group);
                    this.statsManager.statsOffset(topic, group, 1);
                    return new BooleanCommand(HttpStatus.Moved, String.valueOf(maxOffset), request.getOpaque());
                }
                else {
                    return new BooleanCommand(HttpStatus.NotFound, "Could not find message at position "
                            + requestOffset, request.getOpaque());
                }
            }
        }
        catch (final ArrayIndexOutOfBoundsException e) {
            log.error("Could not get message from position " + request.getOffset() + ",it is out of bounds,topic="
                    + topic);
            // ��֪������õ�offset
            this.statsManager.statsGetMiss(topic, group, 1);
            this.statsManager.statsGetFailed(topic, group, 1);
            final long validOffset = store.getNearestOffset(request.getOffset());
            this.statsManager.statsOffset(topic, group, 1);
            return new BooleanCommand(HttpStatus.Moved, String.valueOf(validOffset), request.getOpaque());

        }
        catch (final Throwable e) {
View Full Code Here


    @Override
    public ResponseCommand processOffsetCommand(final OffsetCommand request, final SessionContext ctx) {
        this.statsManager.statsOffset(request.getTopic(), request.getGroup(), 1);
        final MessageStore store = this.storeManager.getMessageStore(request.getTopic(), request.getPartition());
        if (store == null) {
            return new BooleanCommand(HttpStatus.NotFound, "The topic `" + request.getTopic() + "` in partition `"
                    + request.getPartition() + "` is not exists", request.getOpaque());
        }
        final long offset = store.getNearestOffset(request.getOffset());
        return new BooleanCommand(HttpStatus.Success, String.valueOf(offset), request.getOpaque());

    }
View Full Code Here

    public void recieveMessages(final Message message) throws InterruptedException {

        this.statsManager.statsSlavePut(message.getTopic(), message.getPartition().toString(), 1);
        this.statsManager.statsMessageSize(message.getTopic(), message.getData().length);
        final int partition = message.getPartition().getPartition();
        MessageStore store;
        try {
            store = this.storeManager.getOrCreateMessageStore(message.getTopic(), partition);
            final long messageId = message.getId();
            this.brokerZooKeeper.registerTopicInZk(message.getTopic(), false);

            final AppendOp cb = new AppendOp();
            store.append(messageId, new PutCommand(message.getTopic(), partition, MessageUtils.encodePayload(message),
                null, MessageAccessor.getFlag(message), 0), cb);
            cb.latch.await(APPEND_TIMEOUT, TimeUnit.MILLISECONDS);
            if (cb.offset < 0) {
                log.error("offset wasless then 0 when append meta slave message");
                throw new AppendMessageErrorException("Append message failed,topic=" + message.getTopic());
View Full Code Here

        final byte[] data = new byte[1024];
        final int flag = MessageFlagUtils.getFlag(null);
        final long msgId = 100000L;
        final SyncCommand request = new SyncCommand(this.topic, partition, data, flag, msgId, -1, opaque);

        final MessageStore store = this.mocksControl.createMock(MessageStore.class);
        EasyMock.expect(this.storeManager.getOrCreateMessageStore(this.topic, partition)).andReturn(store);
        final AtomicBoolean invoked = new AtomicBoolean(false);
        final BooleanCommand expectResp =
                new BooleanCommand(
                    HttpStatus.InternalServerError,
                    "Put message to [broker 'meta://localhost:8123'] [partition 'GregorCommandProcessorUnitTest-1'] failed.",
                    request.getOpaque());
        final PutCallback cb = new PutCallback() {

            @Override
            public void putComplete(final ResponseCommand resp) {
                invoked.set(true);
                System.out.println(((BooleanCommand) resp).getErrorMsg());
                if (!expectResp.equals(resp)) {
                    throw new RuntimeException();
                }
            }
        };
        store.append(msgId, request,
            this.commandProcessor.new StoreAppendCallback(partition, this.metaConfig.getBrokerId() + "-" + partition,
                request, msgId, cb));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
View Full Code Here

        final byte[] data = new byte[1024];
        final int flag = MessageFlagUtils.getFlag(null);
        final long msgId = 100000L;
        final SyncCommand request = new SyncCommand(this.topic, partition, data, flag, msgId, -1, opaque);

        final MessageStore store = this.mocksControl.createMock(MessageStore.class);
        EasyMock.expect(this.storeManager.getOrCreateMessageStore(this.topic, partition)).andReturn(store);
        final AtomicBoolean invoked = new AtomicBoolean(false);
        final BooleanCommand expectResp =
                new BooleanCommand(
                    HttpStatus.InternalServerError,
                    "Put message to [broker 'meta://localhost:8123'] [partition 'GregorCommandProcessorUnitTest-1'] failed.Detail:Mock exception",
                    request.getOpaque());
        final PutCallback cb = new PutCallback() {

            @Override
            public void putComplete(final ResponseCommand resp) {
                invoked.set(true);
                System.out.println(((BooleanCommand) resp).getErrorMsg());
                if (!expectResp.equals(resp)) {
                    throw new RuntimeException();
                }
            }
        };
        store.append(msgId, request,
            this.commandProcessor.new StoreAppendCallback(partition, this.metaConfig.getBrokerId() + "-" + partition,
                request, msgId, cb));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
View Full Code Here

TOP

Related Classes of com.taobao.metamorphosis.server.store.MessageStore$SegmentList

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.