Package org.apache.hedwig.protocol.PubSubProtocol

Examples of org.apache.hedwig.protocol.PubSubProtocol.MessageSeqId

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
        //
        // No topics acquired.
        //
        SubscribeRequest subRequest = SubscribeRequest.newBuilder().setSubscriberId(sub1).build();
        MessageSeqId msgId = MessageSeqId.newBuilder().setLocalComponent(100).build();

        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);

        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ServerNotResponsibleForTopicException.class);

        sm.unsubscribe(topic1, sub1, voidCallback, null);

        Assert.assertEquals(ConcurrencyUtils.take(BooleanCallbackQueue).right().getClass(),
                            PubSubException.ServerNotResponsibleForTopicException.class);

        //
        // Acquire topic.
        //

        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        Assert.assertTrue(sm.top2sub2seq.containsKey(topic1));
        Assert.assertEquals(0, sm.top2sub2seq.get(topic1).size());

        sm.unsubscribe(topic1, sub1, voidCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(BooleanCallbackQueue).right().getClass(),
                            PubSubException.ClientNotSubscribedException.class);

        //
        // Try to attach to a subscription.
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.ATTACH).setSubscriberId(sub1)
                     .build();

        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ClientNotSubscribedException.class);

        // now create
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.CREATE).setSubscriberId(sub1)
                     .build();
        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(msgId.getLocalComponent(), ConcurrencyUtils.take(msgIdCallbackQueue).left().getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        // try to create again
        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ClientAlreadySubscribedException.class);
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        sm.lostTopic(topic1);
        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        // try to attach
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.ATTACH).setSubscriberId(sub1)
                     .build();
        MessageSeqId msgId1 = MessageSeqId.newBuilder().setLocalComponent(msgId.getLocalComponent() + 10).build();
        sm.serveSubscribeRequest(topic1, subRequest, msgId1, msgIdCallback, null);
        Assert.assertEquals(msgId.getLocalComponent(), msgIdCallbackQueue.take().left().getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        // now manipulate the consume ptrs
        // dont give it enough to have it persist to ZK
        MessageSeqId msgId2 = MessageSeqId.newBuilder().setLocalComponent(
                                  msgId.getLocalComponent() + cfg.getConsumeInterval() - 1).build();
        sm.setConsumeSeqIdForSubscriber(topic1, sub1, msgId2, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());
        Assert.assertEquals(msgId2.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getSubscriptionState().getMsgId()
                            .getLocalComponent());

        // give it more so that it will write to ZK
        MessageSeqId msgId3 = MessageSeqId.newBuilder().setLocalComponent(
                                  msgId.getLocalComponent() + cfg.getConsumeInterval() + 1).build();
        sm.setConsumeSeqIdForSubscriber(topic1, sub1, msgId3, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        sm.lostTopic(topic1);
        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        Assert.assertEquals(msgId3.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());
        Assert.assertEquals(msgId3.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getSubscriptionState().getMsgId()
                            .getLocalComponent());

        // finally unsubscribe
        sm.unsubscribe(topic1, sub1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
     *
     * @param topic
     * @return
     */
    private MessageSeqId ensureSeqIdExistsForTopic(ByteString topic) {
        MessageSeqId presentSeqIdInMap = currTopicSeqIds.get(topic);

        if (presentSeqIdInMap != null) {
            return presentSeqIdInMap;
        }

        presentSeqIdInMap = MessageSeqId.newBuilder().setLocalComponent(0).build();
        MessageSeqId oldSeqIdInMap = currTopicSeqIds.putIfAbsent(topic, presentSeqIdInMap);

        if (oldSeqIdInMap != null) {
            return oldSeqIdInMap;
        }
        return presentSeqIdInMap;
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
     * @throws UnexpectedConditionException
     */
    private long adjustTopicSeqIdForPublish(ByteString topic, Message messageToPublish)
            throws UnexpectedConditionException {
        long retValue = 0;
        MessageSeqId oldId;
        MessageSeqId.Builder newIdBuilder = MessageSeqId.newBuilder();

        do {
            oldId = ensureSeqIdExistsForTopic(topic);

            // Increment our own component by 1
            retValue = oldId.getLocalComponent() + 1;
            newIdBuilder.setLocalComponent(retValue);

            if (messageToPublish.hasMsgId()) {
                // take a region-wise max
                MessageIdUtils.takeRegionMaximum(newIdBuilder, messageToPublish.getMsgId(), oldId);

            } else {
                newIdBuilder.addAllRemoteComponents(oldId.getRemoteComponentsList());
            }
        } while (!currTopicSeqIds.replace(topic, oldId, newIdBuilder.build()));

        return retValue;

View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
            return;
        }

        final ByteString topic = request.getTopic();

        MessageSeqId seqId;
        try {
            seqId = persistenceMgr.getCurrentSeqIdForTopic(topic);
        } catch (ServerNotResponsibleForTopicException e) {
            channel.write(PubSubResponseUtils.getResponseForException(e, request.getTxnId())).addListener(
                ChannelFutureListener.CLOSE);
            return;
        }

        final SubscribeRequest subRequest = request.getSubscribeRequest();
        final ByteString subscriberId = subRequest.getSubscriberId();

        MessageSeqId lastSeqIdPublished = MessageSeqId.newBuilder(seqId).setLocalComponent(seqId.getLocalComponent()).build();

        subMgr.serveSubscribeRequest(topic, subRequest, lastSeqIdPublished, new Callback<MessageSeqId>() {

            @Override
            public void operationFailed(Object ctx, PubSubException exception) {
                channel.write(PubSubResponseUtils.getResponseForException(exception, request.getTxnId())).addListener(
                    ChannelFutureListener.CLOSE);
            }

            @Override
            public void operationFinished(Object ctx, MessageSeqId resultOfOperation) {

                TopicSubscriber topicSub = new TopicSubscriber(topic, subscriberId);

                // race with channel getting disconnected while we are adding it
                // to the 2 maps
                synchronized (channel) {
                    if (!channel.isConnected()) {
                        // channel got disconnected while we were processing the
                        // subscribe request,
                        // nothing much we can do in this case
                        return;
                    }

                    if (null != sub2Channel.putIfAbsent(topicSub, channel)) {
                        // there was another channel mapped to this sub
                        PubSubException pse = new PubSubException.TopicBusyException(
                            "subscription for this topic, subscriberId is already being served on a different channel");
                        channel.write(PubSubResponseUtils.getResponseForException(pse, request.getTxnId()))
                        .addListener(ChannelFutureListener.CLOSE);
                        return;
                    } else {
                        // channel2sub is just a cache, so we can add to it
                        // without synchronization
                        channel2sub.put(channel, topicSub);
                    }
                }
                // First write success and then tell the delivery manager,
                // otherwise the first message might go out before the response
                // to the subscribe
                channel.write(PubSubResponseUtils.getSuccessResponse(request.getTxnId()));

                // want to start 1 ahead of the consume ptr
                MessageSeqId seqIdToStartFrom = MessageSeqId.newBuilder(resultOfOperation).setLocalComponent(
                                                    resultOfOperation.getLocalComponent() + 1).build();
                deliveryMgr.startServingSubscription(topic, subscriberId, seqIdToStartFrom,
                                                     new ChannelEndPoint(channel), TrueFilter.instance(), SubscriptionStateUtils
                                                     .isHubSubscriber(subRequest.getSubscriberId()));
            }
View Full Code Here
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
        // print msg id
        String msgId;
        if (!message.hasMsgId()) {
            msgId = "N/A";
        } else {
            MessageSeqId seqId = message.getMsgId();
            StringBuilder idBuilder = new StringBuilder();
            if (seqId.hasLocalComponent()) {
                idBuilder.append("LOCAL(").append(seqId.getLocalComponent()).append(")");
            } else {
                List<RegionSpecificSeqId> remoteIds = seqId.getRemoteComponentsList();
                int i = 0, numRegions = remoteIds.size();
                idBuilder.append("REMOTE(");
                for (RegionSpecificSeqId rssid : remoteIds) {
                    idBuilder.append(rssid.getRegion().toStringUtf8());
                    idBuilder.append("[");
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
        //
        // No topics acquired.
        //
        SubscribeRequest subRequest = SubscribeRequest.newBuilder().setSubscriberId(sub1).build();
        MessageSeqId msgId = MessageSeqId.newBuilder().setLocalComponent(100).build();

        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);

        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ServerNotResponsibleForTopicException.class);

        sm.unsubscribe(topic1, sub1, voidCallback, null);

        Assert.assertEquals(ConcurrencyUtils.take(BooleanCallbackQueue).right().getClass(),
                            PubSubException.ServerNotResponsibleForTopicException.class);

        //
        // Acquire topic.
        //

        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        Assert.assertTrue(sm.top2sub2seq.containsKey(topic1));
        Assert.assertEquals(0, sm.top2sub2seq.get(topic1).size());

        sm.unsubscribe(topic1, sub1, voidCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(BooleanCallbackQueue).right().getClass(),
                            PubSubException.ClientNotSubscribedException.class);

        //
        // Try to attach to a subscription.
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.ATTACH).setSubscriberId(sub1)
                     .build();

        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ClientNotSubscribedException.class);

        // now create
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.CREATE).setSubscriberId(sub1)
                     .build();
        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(msgId.getLocalComponent(), ConcurrencyUtils.take(msgIdCallbackQueue).left().getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        // try to create again
        sm.serveSubscribeRequest(topic1, subRequest, msgId, msgIdCallback, null);
        Assert.assertEquals(ConcurrencyUtils.take(msgIdCallbackQueue).right().getClass(),
                            PubSubException.ClientAlreadySubscribedException.class);
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        sm.lostTopic(topic1);
        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        // try to attach
        subRequest = SubscribeRequest.newBuilder().setCreateOrAttach(CreateOrAttach.ATTACH).setSubscriberId(sub1)
                     .build();
        MessageSeqId msgId1 = MessageSeqId.newBuilder().setLocalComponent(msgId.getLocalComponent() + 10).build();
        sm.serveSubscribeRequest(topic1, subRequest, msgId1, msgIdCallback, null);
        Assert.assertEquals(msgId.getLocalComponent(), msgIdCallbackQueue.take().left().getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());

        // now manipulate the consume ptrs
        // dont give it enough to have it persist to ZK
        MessageSeqId msgId2 = MessageSeqId.newBuilder().setLocalComponent(
                                  msgId.getLocalComponent() + cfg.getConsumeInterval() - 1).build();
        sm.setConsumeSeqIdForSubscriber(topic1, sub1, msgId2, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());
        Assert.assertEquals(msgId2.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());
        Assert.assertEquals(msgId.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getSubscriptionState().getMsgId()
                            .getLocalComponent());

        // give it more so that it will write to ZK
        MessageSeqId msgId3 = MessageSeqId.newBuilder().setLocalComponent(
                                  msgId.getLocalComponent() + cfg.getConsumeInterval() + 1).build();
        sm.setConsumeSeqIdForSubscriber(topic1, sub1, msgId3, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        sm.lostTopic(topic1);
        sm.acquiredTopic(topic1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());

        Assert.assertEquals(msgId3.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getLastConsumeSeqId()
                            .getLocalComponent());
        Assert.assertEquals(msgId3.getLocalComponent(), sm.top2sub2seq.get(topic1).get(sub1).getSubscriptionState().getMsgId()
                            .getLocalComponent());

        // finally unsubscribe
        sm.unsubscribe(topic1, sub1, voidCallback, null);
        Assert.assertTrue(BooleanCallbackQueue.take().left());
View Full Code Here
371
372
373
374
375
376
377
378
379
380
381
                return false;
            }
            ByteString topic = ByteString.copyFromUtf8(args[1]);
            ByteString subId = ByteString.copyFromUtf8(args[2]);
            long msgId = Long.parseLong(args[3]);
            MessageSeqId consumeId = MessageSeqId.newBuilder().setLocalComponent(msgId).build();
            try {
                subscriber.consume(topic, subId, consumeId);
            } catch (Exception e) {
                System.err.println("CONSUMETO FAILED");
            }
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
            }
            long numMessagesToConsume = Long.parseLong(args[3]);
            long idToConsumed = lastConsumedId + numMessagesToConsume;
            System.out.println("Try to move subscriber(" + args[2] + ") consume ptr of topic(" + args[1]
                             + ") from " + lastConsumedId + " to " + idToConsumed);
            MessageSeqId consumeId = MessageSeqId.newBuilder().setLocalComponent(idToConsumed).build();
            ByteString topic = ByteString.copyFromUtf8(args[1]);
            ByteString subId = ByteString.copyFromUtf8(args[2]);
            try {
                subscriber.consume(topic, subId, consumeId);
            } catch (Exception e) {
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
        assertEquals("Should be expected " + numMessages + " messages in map.",
                     numMessages, receivedMsgs.size());

        for (int i=0; i<numMessages; i++) {
            final String str = prefix + i;
            MessageSeqId pubId = publishedMsgs.get(str);
            MessageSeqId revId = receivedMsgs.get(str);
            assertTrue("Doesn't receive same message seq id for " + str,
                       pubId.equals(revId));
        }
    }
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
        assertEquals("Should be expected " + numMessages + " messages in map.",
                     numMessages, receivedMsgs.size());

        for (int i=0; i<numMessages; i++) {
            final String str = prefix + i;
            MessageSeqId pubId = publishedMsgs.get(str);
            MessageSeqId revId = receivedMsgs.get(str);
            assertTrue("Doesn't receive same message seq id for " + str,
                       pubId.equals(revId));
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.hedwig.protocol.PubSubProtocol.MessageSeqId

  • org.apache.hedwig.admin.console.HedwigConsole$ConsumeCmd
  • org.apache.hedwig.admin.console.HedwigConsole$ConsumeToCmd
  • org.apache.hedwig.admin.console.ReadTopic
  • org.apache.hedwig.admin.HedwigAdmin
  • org.apache.hedwig.client.netty.TestMultiplexing
  • org.apache.hedwig.client.TestPubSubClient
  • org.apache.hedwig.data.MessageFormatter
  • org.apache.hedwig.server.delivery.TestFIFODeliveryManager
  • org.apache.hedwig.server.delivery.TestFIFODeliveryManager$ExecutorDeliveryEndPoint
  • org.apache.hedwig.server.handlers.SubscribeHandler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.