Coverage for tests/brokers/redis/test_consume.py: 99%
607 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import asyncio
2from typing import Any
3from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
5import anyio
6import pytest
7from dirty_equals import IsPartialDict
8from pydantic import BaseModel
9from redis.asyncio import Redis
10from redis.exceptions import ResponseError
12from faststream import AckPolicy, BaseMiddleware
13from faststream.message import StreamMessage
14from faststream.redis import (
15 ListSub,
16 PubSub,
17 RedisMessage,
18 RedisStreamMessage,
19 StreamSub,
20)
21from tests.brokers.base.consume import BrokerRealConsumeTestcase
22from tests.tools import spy_decorator
24from .basic import RedisTestcaseConfig
27@pytest.mark.connected()
28@pytest.mark.redis()
29@pytest.mark.asyncio()
30class TestConsume(RedisTestcaseConfig, BrokerRealConsumeTestcase):
31 async def test_consume_native(
32 self, mock: MagicMock, queue: str, event: asyncio.Event
33 ) -> None:
34 consume_broker = self.get_broker()
36 @consume_broker.subscriber(queue)
37 async def handler(msg: Any) -> None:
38 mock(msg)
39 event.set()
41 async with self.patch_broker(consume_broker) as br:
42 await br.start()
44 client = await br.connect()
45 result = await client.publish(queue, "hello")
46 await asyncio.wait(
47 (asyncio.create_task(event.wait()),),
48 timeout=3,
49 )
50 assert result == 1, result
52 mock.assert_called_once_with(b"hello")
54 async def test_pattern_with_path(self, mock: MagicMock, event: asyncio.Event) -> None:
55 consume_broker = self.get_broker()
57 @consume_broker.subscriber("test.{name}")
58 async def handler(msg: Any) -> None:
59 mock(msg)
60 event.set()
62 async with self.patch_broker(consume_broker) as br:
63 await br.start()
65 await asyncio.wait(
66 (
67 asyncio.create_task(br.publish("hello", "test.name")),
68 asyncio.create_task(event.wait()),
69 ),
70 timeout=3,
71 )
73 mock.assert_called_once_with("hello")
75 async def test_pattern_without_path(
76 self, mock: MagicMock, event: asyncio.Event
77 ) -> None:
78 consume_broker = self.get_broker()
80 @consume_broker.subscriber(PubSub("test.*", pattern=True))
81 async def handler(msg: Any) -> None:
82 mock(msg)
83 event.set()
85 async with self.patch_broker(consume_broker) as br:
86 await br.start()
88 await asyncio.wait(
89 (
90 asyncio.create_task(br.publish("hello", "test.name")),
91 asyncio.create_task(event.wait()),
92 ),
93 timeout=3,
94 )
96 mock.assert_called_once_with("hello")
98 @pytest.mark.flaky(reruns=3, reruns_delay=1)
99 async def test_concurrent_consume_channel(
100 self, queue: str, mock: MagicMock, event: asyncio.Event, event2: asyncio.Event
101 ) -> None:
102 consume_broker = self.get_broker()
104 @consume_broker.subscriber(channel=PubSub(queue), max_workers=2)
105 async def handler(msg: Any) -> None:
106 mock()
107 if event.is_set():
108 event2.set()
109 else:
110 event.set()
111 await asyncio.sleep(0.1)
113 async with self.patch_broker(consume_broker) as br:
114 await br.start()
116 for i in range(5):
117 await br.publish(i, queue)
119 await asyncio.wait(
120 (
121 asyncio.create_task(event.wait()),
122 asyncio.create_task(event2.wait()),
123 ),
124 timeout=3,
125 )
127 assert event.is_set()
128 assert event2.is_set()
129 assert mock.call_count == 2, mock.call_count
132@pytest.mark.connected()
133@pytest.mark.redis()
134@pytest.mark.asyncio()
135class TestConsumeList(RedisTestcaseConfig):
136 async def test_consume_list(
137 self, queue: str, mock: MagicMock, event: asyncio.Event
138 ) -> None:
139 consume_broker = self.get_broker()
141 @consume_broker.subscriber(list=queue)
142 async def handler(msg: Any) -> None:
143 mock(msg)
144 event.set()
146 async with self.patch_broker(consume_broker) as br:
147 await br.start()
149 await asyncio.wait(
150 (
151 asyncio.create_task(br.publish("hello", list=queue)),
152 asyncio.create_task(event.wait()),
153 ),
154 timeout=3,
155 )
157 mock.assert_called_once_with("hello")
159 async def test_consume_list_native(
160 self, queue: str, mock: MagicMock, event: asyncio.Event
161 ) -> None:
162 consume_broker = self.get_broker()
164 @consume_broker.subscriber(list=queue)
165 async def handler(msg: Any) -> None:
166 mock(msg)
167 event.set()
169 async with self.patch_broker(consume_broker) as br:
170 await br.start()
172 client = await br.connect()
173 await asyncio.wait(
174 (
175 asyncio.create_task(client.rpush(queue, "hello")),
176 asyncio.create_task(event.wait()),
177 ),
178 timeout=3,
179 )
181 mock.assert_called_once_with(b"hello")
183 @pytest.mark.slow()
184 async def test_consume_list_batch_with_one(
185 self, queue: str, mock: MagicMock, event: asyncio.Event
186 ) -> None:
187 consume_broker = self.get_broker()
189 @consume_broker.subscriber(
190 list=ListSub(queue, batch=True, polling_interval=0.01),
191 )
192 async def handler(msg: Any) -> None:
193 mock(msg)
194 event.set()
196 async with self.patch_broker(consume_broker) as br:
197 await br.start()
198 await asyncio.wait(
199 (
200 asyncio.create_task(br.publish("hi", list=queue)),
201 asyncio.create_task(event.wait()),
202 ),
203 timeout=3,
204 )
206 assert event.is_set()
207 mock.assert_called_once_with(["hi"])
209 @pytest.mark.slow()
210 async def test_consume_list_batch_headers(
211 self, queue: str, mock: MagicMock, event: asyncio.Event
212 ) -> None:
213 consume_broker = self.get_broker(apply_types=True)
215 @consume_broker.subscriber(
216 list=ListSub(queue, batch=True, polling_interval=0.01),
217 )
218 def subscriber(m: Any, msg: RedisMessage) -> None:
219 check = all(
220 (
221 msg.headers,
222 msg.headers["correlation_id"]
223 == msg.batch_headers[0]["correlation_id"],
224 msg.headers.get("custom") == "1",
225 ),
226 )
227 mock(check)
228 event.set()
230 async with self.patch_broker(consume_broker) as br:
231 await br.start()
232 await asyncio.wait(
233 (
234 asyncio.create_task(
235 br.publish("", list=queue, headers={"custom": "1"}),
236 ),
237 asyncio.create_task(event.wait()),
238 ),
239 timeout=3,
240 )
242 assert event.is_set()
243 mock.assert_called_once_with(True)
245 @pytest.mark.slow()
246 async def test_consume_list_batch(
247 self,
248 queue: str,
249 ) -> None:
250 consume_broker = self.get_broker(apply_types=True)
252 msgs_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
254 @consume_broker.subscriber(
255 list=ListSub(queue, batch=True, polling_interval=0.01),
256 )
257 async def handler(msg: Any) -> None:
258 await msgs_queue.put(msg)
260 async with self.patch_broker(consume_broker) as br:
261 await br.start()
263 await br.publish_batch(1, "hi", list=queue)
265 result, _ = await asyncio.wait(
266 (asyncio.create_task(msgs_queue.get()),),
267 timeout=3,
268 )
270 assert [{1, "hi"}] == [set(r.result()) for r in result]
272 @pytest.mark.slow()
273 async def test_consume_list_batch_complex(
274 self,
275 queue: str,
276 ) -> None:
277 consume_broker = self.get_broker(apply_types=True)
279 class Data(BaseModel):
280 m: str
282 def __hash__(self) -> Any:
283 return hash(self.m)
285 msgs_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
287 @consume_broker.subscriber(
288 list=ListSub(queue, batch=True, polling_interval=0.01),
289 )
290 async def handler(msg: list[Data]) -> None:
291 await msgs_queue.put(msg)
293 async with self.patch_broker(consume_broker) as br:
294 await br.start()
296 await br.publish_batch(Data(m="hi"), Data(m="again"), list=queue)
298 result, _ = await asyncio.wait(
299 (asyncio.create_task(msgs_queue.get()),),
300 timeout=3,
301 )
303 assert [{Data(m="hi"), Data(m="again")}] == [set(r.result()) for r in result]
305 @pytest.mark.slow()
306 async def test_consume_list_batch_native(
307 self,
308 queue: str,
309 ) -> None:
310 consume_broker = self.get_broker()
312 msgs_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
314 @consume_broker.subscriber(
315 list=ListSub(queue, batch=True, polling_interval=0.01),
316 )
317 async def handler(msg: Any) -> None:
318 await msgs_queue.put(msg)
320 async with self.patch_broker(consume_broker) as br:
321 await br.start()
323 client = await br.connect()
324 await client.rpush(queue, 1, "hi")
326 result, _ = await asyncio.wait(
327 (asyncio.create_task(msgs_queue.get()),),
328 timeout=3,
329 )
331 assert [{1, "hi"}] == [set(r.result()) for r in result]
333 async def test_get_one(
334 self,
335 queue: str,
336 ) -> None:
337 broker = self.get_broker(apply_types=True)
338 subscriber = broker.subscriber(list=queue)
340 async with self.patch_broker(broker) as br:
341 await br.start()
343 message = None
345 async def consume() -> None:
346 nonlocal message
347 message = await subscriber.get_one(timeout=5)
349 async def publish() -> None:
350 await br.publish("test_message", list=queue)
352 await asyncio.wait(
353 (
354 asyncio.create_task(consume()),
355 asyncio.create_task(publish()),
356 ),
357 timeout=10,
358 )
360 assert message is not None
361 assert await message.decode() == "test_message"
363 async def test_get_one_timeout(
364 self,
365 queue: str,
366 mock: MagicMock,
367 ) -> None:
368 broker = self.get_broker(apply_types=True)
369 subscriber = broker.subscriber(list=queue)
371 async with self.patch_broker(broker) as br:
372 await br.start()
374 mock(await subscriber.get_one(timeout=1e-24))
375 mock.assert_called_once_with(None)
377 async def test_get_one_preserves_empty_message(
378 self,
379 queue: str,
380 ) -> None:
381 broker = self.get_broker()
382 subscriber = broker.subscriber(list=queue)
384 async with self.patch_broker(broker) as br:
385 await br.start()
387 client = await br.connect()
388 await client.rpush(queue, b"", b"payload")
390 received: list[bytes] = []
391 for _ in range(2):
392 message = await subscriber.get_one(timeout=1)
393 assert message is not None
394 received.append(message.body)
396 assert received == [b"", b"payload"]
397 assert await client.llen(queue) == 0
399 async def test_concurrent_consume_list(
400 self,
401 queue: str,
402 mock: MagicMock,
403 event: asyncio.Event,
404 event2: asyncio.Event,
405 ) -> None:
406 consume_broker = self.get_broker()
408 @consume_broker.subscriber(list=ListSub(queue), max_workers=2)
409 async def handler(msg: Any) -> None:
410 mock()
411 if event.is_set():
412 event2.set()
413 else:
414 event.set()
415 await asyncio.sleep(0.1)
417 async with self.patch_broker(consume_broker) as br:
418 await br.start()
420 for i in range(5):
421 await br.publish(i, list=queue)
423 await asyncio.wait(
424 (
425 asyncio.create_task(event.wait()),
426 asyncio.create_task(event2.wait()),
427 ),
428 timeout=3,
429 )
431 assert event.is_set()
432 assert event2.is_set()
433 assert mock.call_count == 2, mock.call_count
435 async def test_iterator(
436 self,
437 queue: str,
438 ) -> None:
439 expected_messages = ("test_message_1", "test_message_2")
441 broker = self.get_broker(apply_types=True)
442 subscriber = broker.subscriber(list=queue)
444 async with self.patch_broker(broker) as br:
445 await br.start()
447 async def publish_test_message() -> None:
448 for msg in expected_messages:
449 await br.publish(msg, list=queue)
451 _ = await asyncio.create_task(publish_test_message())
453 index_message = 0
454 async for msg in subscriber: 454 ↛ exitline 454 didn't jump to the function exit
455 result_message = await msg.decode()
457 assert result_message == expected_messages[index_message]
459 index_message += 1
460 if index_message >= len(expected_messages):
461 break
463 async def test_iterator_preserves_empty_message(
464 self,
465 queue: str,
466 ) -> None:
467 broker = self.get_broker()
468 subscriber = broker.subscriber(list=queue)
470 async with self.patch_broker(broker) as br:
471 await br.start()
473 client = await br.connect()
474 await client.rpush(queue, b"", b"payload")
476 received: list[bytes] = []
477 with anyio.fail_after(3):
478 async for message in subscriber: 478 ↛ 483line 478 didn't jump to line 483
479 received.append(message.body)
480 if len(received) == 2:
481 break
483 assert received == [b"", b"payload"]
484 assert await client.llen(queue) == 0
487@pytest.mark.connected()
488@pytest.mark.redis()
489@pytest.mark.asyncio()
490class TestConsumeStream(RedisTestcaseConfig):
491 async def test_consume_group_creates_stream_by_default(self, queue: str) -> None:
492 consume_broker = self.get_broker()
494 @consume_broker.subscriber(
495 stream=StreamSub(queue, group="group", consumer=queue),
496 )
497 async def handler(msg: RedisMessage) -> None: ...
499 async with self.patch_broker(consume_broker) as br:
500 await br.start()
501 client = await br.connect()
502 assert await client.exists(queue)
504 async def test_consume_group_with_no_ack_skips_pel(
505 self,
506 queue: str,
507 event: asyncio.Event,
508 mock: MagicMock,
509 ) -> None:
510 consume_broker = self.get_broker()
512 @consume_broker.subscriber(
513 stream=StreamSub(queue, group="group", consumer=queue, no_ack=True),
514 )
515 async def handler(msg: Any) -> None:
516 mock(msg)
517 event.set()
519 async with self.patch_broker(consume_broker) as br:
520 await br.start()
521 await br.publish({"data": "hello"}, stream=queue)
522 await asyncio.wait_for(event.wait(), timeout=self.timeout)
524 client = await br.connect()
525 # XREADGROUP NOACK delivers the entry without ever putting it in the PEL
526 assert await client.xpending(queue, "group") == IsPartialDict( # type: ignore[no-untyped-call]
527 pending=0,
528 )
530 mock.assert_called_once_with({"data": "hello"})
532 async def test_consume_group_without_declare_requires_stream(
533 self, queue: str
534 ) -> None:
535 consume_broker = self.get_broker()
537 @consume_broker.subscriber(
538 stream=StreamSub(
539 queue,
540 group="group",
541 consumer=queue,
542 declare=False,
543 ),
544 )
545 async def handler(msg: RedisMessage) -> None: ...
547 async with self.patch_broker(consume_broker) as br:
548 with pytest.raises(ResponseError, match="key to exist"):
549 await br.start()
550 client = await br.connect()
551 assert not await client.exists(queue)
553 @pytest.mark.slow()
554 async def test_consume_stream(
555 self, mock: MagicMock, queue: str, event: asyncio.Event
556 ) -> None:
557 consume_broker = self.get_broker()
559 @consume_broker.subscriber(stream=StreamSub(queue, polling_interval=10))
560 async def handler(msg: Any) -> None:
561 mock(msg)
562 event.set()
564 async with self.patch_broker(consume_broker) as br:
565 await br.start()
567 await asyncio.wait(
568 (
569 asyncio.create_task(br.publish("hello", stream=queue)),
570 asyncio.create_task(event.wait()),
571 ),
572 timeout=3,
573 )
575 mock.assert_called_once_with("hello")
577 @pytest.mark.slow()
578 async def test_consume_stream_with_big_interval(
579 self,
580 event: asyncio.Event,
581 mock: MagicMock,
582 queue: str,
583 ) -> None:
584 consume_broker = self.get_broker()
586 @consume_broker.subscriber(stream=StreamSub(queue, polling_interval=100000))
587 async def handler(msg: Any) -> None:
588 mock(msg)
589 event.set()
591 async with self.patch_broker(consume_broker) as br:
592 await br.start()
593 await asyncio.wait(
594 (
595 asyncio.create_task(br.publish("hello", stream=queue)),
596 asyncio.create_task(event.wait()),
597 ),
598 timeout=3,
599 )
601 mock.assert_called_once_with("hello")
603 @pytest.mark.slow()
604 async def test_consume_stream_native(
605 self, mock: MagicMock, queue: str, event: asyncio.Event
606 ) -> None:
607 consume_broker = self.get_broker()
609 @consume_broker.subscriber(stream=StreamSub(queue, polling_interval=10))
610 async def handler(msg: Any) -> None:
611 mock(msg)
612 event.set()
614 async with self.patch_broker(consume_broker) as br:
615 await br.start()
617 client = await br.connect()
618 await asyncio.wait(
619 (
620 asyncio.create_task(
621 client.xadd(queue, {"message": "hello"}),
622 ),
623 asyncio.create_task(event.wait()),
624 ),
625 timeout=3,
626 )
628 mock.assert_called_once_with({"message": "hello"})
630 @pytest.mark.slow()
631 @pytest.mark.flaky(reruns=3, reruns_delay=1)
632 async def test_consume_stream_batch(
633 self, mock: MagicMock, queue: str, event: asyncio.Event
634 ) -> None:
635 consume_broker = self.get_broker()
637 @consume_broker.subscriber(
638 stream=StreamSub(queue, polling_interval=10, batch=True),
639 )
640 async def handler(msg: Any) -> None:
641 mock(msg)
642 event.set()
644 async with self.patch_broker(consume_broker) as br:
645 await br.start()
647 await asyncio.wait(
648 (
649 asyncio.create_task(br.publish("hello", stream=queue)),
650 asyncio.create_task(event.wait()),
651 ),
652 timeout=3,
653 )
655 mock.assert_called_once_with(["hello"])
657 @pytest.mark.slow()
658 async def test_consume_stream_batch_headers(
659 self, queue: str, mock: MagicMock, event: asyncio.Event
660 ) -> None:
661 consume_broker = self.get_broker(apply_types=True)
663 @consume_broker.subscriber(
664 stream=StreamSub(queue, polling_interval=10, batch=True),
665 )
666 def subscriber(m: Any, msg: RedisMessage) -> None:
667 check = all(
668 (
669 msg.headers,
670 msg.headers["correlation_id"]
671 == msg.batch_headers[0]["correlation_id"],
672 msg.headers.get("custom") == "1",
673 ),
674 )
675 mock(check)
676 event.set()
678 async with self.patch_broker(consume_broker) as br:
679 await br.start()
680 await asyncio.wait(
681 (
682 asyncio.create_task(
683 br.publish("", stream=queue, headers={"custom": "1"}),
684 ),
685 asyncio.create_task(event.wait()),
686 ),
687 timeout=3,
688 )
690 assert event.is_set()
691 mock.assert_called_once_with(True)
693 @pytest.mark.slow()
694 async def test_consume_stream_batch_complex(
695 self,
696 queue: str,
697 ) -> None:
698 consume_broker = self.get_broker(apply_types=True)
700 class Data(BaseModel):
701 m: str
703 msgs_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
705 @consume_broker.subscriber(
706 stream=StreamSub(queue, polling_interval=10, batch=True),
707 )
708 async def handler(msg: list[Data]) -> None:
709 await msgs_queue.put(msg)
711 async with self.patch_broker(consume_broker) as br:
712 await br.start()
714 await br.publish(Data(m="hi"), stream=queue)
716 result, _ = await asyncio.wait(
717 (asyncio.create_task(msgs_queue.get()),),
718 timeout=3,
719 )
721 assert next(iter(result)).result() == [Data(m="hi")]
723 @pytest.mark.slow()
724 async def test_consume_stream_batch_native(
725 self, mock: MagicMock, queue: str, event: asyncio.Event
726 ) -> None:
727 consume_broker = self.get_broker()
729 @consume_broker.subscriber(
730 stream=StreamSub(queue, polling_interval=10, batch=True),
731 )
732 async def handler(msg: Any) -> None:
733 mock(msg)
734 event.set()
736 async with self.patch_broker(consume_broker) as br:
737 await br.start()
739 client = await br.connect()
740 await asyncio.wait(
741 (
742 asyncio.create_task(
743 client.xadd(queue, {"message": "hello"}),
744 ),
745 asyncio.create_task(event.wait()),
746 ),
747 timeout=3,
748 )
750 mock.assert_called_once_with([{"message": "hello"}])
752 async def test_consume_group(
753 self,
754 queue: str,
755 ) -> None:
756 consume_broker = self.get_broker()
758 subscriber = consume_broker.subscriber(
759 stream=StreamSub(queue, group="group", consumer=queue),
760 )
762 @subscriber
763 async def handler(msg: RedisMessage) -> None: ...
765 assert subscriber.last_id == ">"
767 async def test_consume_group_with_last_id(
768 self,
769 queue: str,
770 ) -> None:
771 consume_broker = self.get_broker()
773 subscriber = consume_broker.subscriber(
774 stream=StreamSub(queue, group="group", consumer=queue, last_id="0"),
775 )
777 @subscriber
778 async def handler(msg: RedisMessage) -> None: ...
780 assert subscriber.last_id == "0"
782 async def test_consume_group_from_beginning(
783 self,
784 queue: str,
785 event: asyncio.Event,
786 mock: MagicMock,
787 ) -> None:
788 consume_broker = self.get_broker()
790 @consume_broker.subscriber(
791 stream=StreamSub(queue, group="group", consumer=queue, last_id="0"),
792 )
793 async def handler(msg: Any) -> None:
794 mock(msg)
795 event.set()
797 async with self.patch_broker(consume_broker) as br:
798 await br.publish({"data": "before-group"}, stream=queue)
799 await br.start()
800 await asyncio.wait_for(event.wait(), timeout=self.timeout)
802 mock.assert_called_once_with({"data": "before-group"})
804 async def test_consume_existing_group_from_last_id(
805 self,
806 queue: str,
807 event: asyncio.Event,
808 mock: MagicMock,
809 ) -> None:
810 consume_broker = self.get_broker()
812 @consume_broker.subscriber(
813 stream=StreamSub(queue, group="group", consumer=queue, last_id="0"),
814 )
815 async def handler(msg: Any) -> None:
816 mock(msg)
817 event.set()
819 async with self.patch_broker(consume_broker) as br:
820 await br.publish({"data": "pending"}, stream=queue)
821 client = br._connection
822 assert client is not None
823 await client.xgroup_create(queue, "group", id="0")
824 await client.xreadgroup("group", queue, {queue: ">"})
826 await br.start()
827 await asyncio.wait_for(event.wait(), timeout=self.timeout)
829 mock.assert_called_once_with({"data": "pending"})
831 async def test_consume_nack(self, queue: str, event: asyncio.Event) -> None:
832 consume_broker = self.get_broker(apply_types=True)
834 @consume_broker.subscriber(
835 stream=StreamSub(queue, group="group", consumer=queue),
836 )
837 async def handler(msg: RedisMessage) -> None:
838 event.set()
839 await msg.nack()
841 async with self.patch_broker(consume_broker) as br:
842 await br.start()
844 with patch.object(Redis, "xack", spy_decorator(Redis.xack)) as m:
845 await asyncio.wait(
846 (
847 asyncio.create_task(br.publish("hello", stream=queue)),
848 asyncio.create_task(event.wait()),
849 ),
850 timeout=3,
851 )
853 assert not m.mock.called
855 assert event.is_set()
857 async def test_consume_ack(self, queue: str, event: asyncio.Event) -> None:
858 consume_broker = self.get_broker(apply_types=True)
860 @consume_broker.subscriber(
861 stream=StreamSub(queue, group="group", consumer=queue),
862 )
863 async def handler(msg: RedisMessage) -> None:
864 event.set()
866 async with self.patch_broker(consume_broker) as br:
867 await br.start()
869 with patch.object(Redis, "xack", spy_decorator(Redis.xack)) as m:
870 await asyncio.wait(
871 (
872 asyncio.create_task(br.publish("hello", stream=queue)),
873 asyncio.create_task(event.wait()),
874 ),
875 timeout=3,
876 )
878 m.mock.assert_called_once()
880 assert event.is_set()
882 async def test_consume_ack_first_on_middleware_error(
883 self, queue: str, event: asyncio.Event
884 ) -> None:
885 class FailingMiddleware(BaseMiddleware):
886 async def on_receive(self) -> None:
887 raise ValueError
889 consume_broker = self.get_broker()
890 consume_broker.add_middleware(FailingMiddleware)
892 @consume_broker.subscriber(
893 stream=StreamSub(queue, group="group", consumer=queue),
894 ack_policy=AckPolicy.ACK_FIRST,
895 )
896 async def handler(msg: RedisMessage) -> None: ...
898 with patch.object(StreamMessage, "ack", spy_decorator(StreamMessage.ack)) as ack:
899 ack.mock.side_effect = lambda *_, **__: event.set()
901 async with self.patch_broker(consume_broker) as br:
902 await br.start()
903 await br.publish("hello", stream=queue)
904 await asyncio.wait_for(event.wait(), timeout=self.timeout)
906 ack.mock.assert_awaited_once()
908 async def test_consume_ack_with_deprecated_args(
909 self,
910 queue: str,
911 event: asyncio.Event,
912 mock: MagicMock,
913 ) -> None:
914 consume_broker = self.get_broker(apply_types=True)
915 legacy_client = MagicMock(xack=AsyncMock())
917 @consume_broker.subscriber(
918 stream=StreamSub(queue, group="group", consumer=queue),
919 ack_policy=AckPolicy.MANUAL,
920 )
921 async def handler(msg: RedisStreamMessage) -> None:
922 with pytest.warns(
923 DeprecationWarning, match=r"will be removed in 1\.0\.0"
924 ) as warnings:
925 await msg.ack(legacy_client, "legacy-group")
926 mock(len(warnings))
927 event.set()
929 async with self.patch_broker(consume_broker) as br:
930 await br.start()
931 await br.publish("hello", stream=queue)
932 await asyncio.wait_for(event.wait(), timeout=self.timeout)
934 mock.assert_called_once_with(1)
935 legacy_client.xack.assert_awaited_once_with(queue, "legacy-group", ANY)
937 @pytest.mark.flaky(reruns=3, reruns_delay=1)
938 async def test_consume_and_delete_acked(
939 self,
940 queue: str,
941 event: asyncio.Event,
942 ) -> None:
943 consume_broker = self.get_broker(apply_types=True)
945 @consume_broker.subscriber(
946 stream=StreamSub(queue, group="group", consumer=queue),
947 )
948 async def handler(msg: RedisStreamMessage) -> None:
949 event.set()
950 await msg.delete()
952 async with self.patch_broker(consume_broker) as br:
953 await br.start()
955 with patch.object(Redis, "xdel", spy_decorator(Redis.xdel)) as m:
956 await asyncio.wait(
957 (
958 asyncio.create_task(br.publish("hello", stream=queue)),
959 asyncio.create_task(event.wait()),
960 ),
961 timeout=3,
962 )
964 m.mock.assert_called_once()
966 client = await br.connect()
967 queue_len = await client.xlen(queue)
968 assert queue_len == 0, (
969 f"Redis stream must be empty here, found {queue_len} messages"
970 )
972 async def test_consume_and_delete_nacked(
973 self,
974 queue: str,
975 mock: MagicMock,
976 event: asyncio.Event,
977 ) -> None:
978 consume_broker = self.get_broker(apply_types=True)
980 @consume_broker.subscriber(
981 stream=StreamSub(queue, group="group", consumer=queue),
982 ack_policy=AckPolicy.MANUAL,
983 )
984 async def handler(msg: RedisStreamMessage) -> None:
985 mock(committed=msg.committed)
986 await msg.delete()
987 event.set()
989 async with self.patch_broker(consume_broker) as br:
990 await br.start()
992 with patch.object(Redis, "xdel", spy_decorator(Redis.xdel)) as m:
993 await asyncio.wait(
994 (
995 asyncio.create_task(br.publish("hello", stream=queue)),
996 asyncio.create_task(event.wait()),
997 ),
998 timeout=3,
999 )
1001 m.mock.assert_called_once()
1003 mock.assert_called_once_with(committed=None)
1004 client = await br.connect()
1005 queue_len = await client.xlen(queue)
1006 assert queue_len == 0, (
1007 f"Redis stream must be empty here, found {queue_len} messages"
1008 )
1010 async def test_get_one(
1011 self,
1012 queue: str,
1013 ) -> None:
1014 broker = self.get_broker(apply_types=True)
1015 subscriber = broker.subscriber(stream=queue)
1017 async with self.patch_broker(broker) as br:
1018 await br.start()
1020 message = None
1022 async def consume() -> None:
1023 nonlocal message
1024 message = await subscriber.get_one(timeout=3)
1026 async def publish() -> None:
1027 await asyncio.sleep(0.1)
1028 await br.publish("test_message", stream=queue)
1030 await asyncio.wait(
1031 (
1032 asyncio.create_task(consume()),
1033 asyncio.create_task(publish()),
1034 ),
1035 timeout=10,
1036 )
1038 assert message is not None
1039 assert await message.decode() == "test_message"
1041 async def test_get_one_timeout(
1042 self,
1043 queue: str,
1044 mock: MagicMock,
1045 ) -> None:
1046 broker = self.get_broker(apply_types=True)
1047 subscriber = broker.subscriber(stream=queue)
1049 async with self.patch_broker(broker) as br:
1050 await br.start()
1052 mock(await subscriber.get_one(timeout=1e-24))
1053 mock.assert_called_once_with(None)
1055 @pytest.mark.parametrize("no_ack", (False, True))
1056 async def test_get_one_respects_no_ack(
1057 self,
1058 queue: str,
1059 no_ack: bool,
1060 ) -> None:
1061 broker = self.get_broker()
1062 sub = broker.subscriber(
1063 stream=StreamSub(queue, group=queue, consumer="reader", no_ack=no_ack),
1064 )
1066 async with self.patch_broker(broker) as br:
1067 await br.start()
1068 await br.publish("hello", stream=queue)
1070 message = await sub.get_one(timeout=3)
1072 assert message is not None
1073 assert await message.decode() == "hello"
1075 client = await br.connect()
1076 pending = await client.xpending(queue, queue) # type: ignore[no-untyped-call]
1077 assert pending["pending"] == (0 if no_ack else 1)
1079 async def test_concurrent_consume_stream(
1080 self,
1081 queue: str,
1082 ) -> None:
1083 consume_broker = self.get_broker()
1084 workers = 2
1085 message_count = 5
1086 active = peak = 0
1087 received: list[int] = []
1088 workers_started = asyncio.Event()
1089 release_workers = asyncio.Event()
1090 all_processed = asyncio.Event()
1092 @consume_broker.subscriber(stream=StreamSub(queue), max_workers=workers)
1093 async def handler(msg: int) -> None:
1094 nonlocal active, peak
1095 active += 1
1096 peak = max(peak, active)
1097 if active == workers:
1098 workers_started.set()
1100 await release_workers.wait()
1101 active -= 1
1102 received.append(msg)
1103 if len(received) == message_count:
1104 all_processed.set()
1106 async with self.patch_broker(consume_broker) as br:
1107 await br.start()
1109 try:
1110 for i in range(message_count):
1111 await br.publish(i, stream=queue)
1112 await asyncio.wait_for(workers_started.wait(), self.timeout)
1113 finally:
1114 release_workers.set()
1116 await asyncio.wait_for(all_processed.wait(), self.timeout)
1118 assert (peak, sorted(received)) == (workers, list(range(message_count)))
1120 async def test_iterator(
1121 self,
1122 queue: str,
1123 mock: MagicMock,
1124 ) -> None:
1125 expected_messages = ("test_message_1", "test_message_2")
1127 broker = self.get_broker(apply_types=True)
1128 subscriber = broker.subscriber(stream=queue)
1130 async with self.patch_broker(broker) as br:
1131 await br.start()
1133 async def publish_test_message() -> None:
1134 await asyncio.sleep(0.1)
1135 for msg in expected_messages:
1136 await br.publish(msg, stream=queue)
1138 async def consume() -> None:
1139 index_message = 0
1140 async for msg in subscriber: 1140 ↛ exitline 1140 didn't return from function 'consume' because the loop on line 1140 didn't complete
1141 result_message = await msg.decode()
1143 mock(result_message)
1145 index_message += 1
1146 if index_message >= len(expected_messages):
1147 break
1149 await asyncio.wait(
1150 (
1151 asyncio.create_task(consume()),
1152 asyncio.create_task(publish_test_message()),
1153 ),
1154 timeout=self.timeout,
1155 )
1157 calls = [call(msg) for msg in expected_messages]
1158 mock.assert_has_calls(calls=calls)
1160 @pytest.mark.parametrize("no_ack", (False, True))
1161 async def test_iterator_respects_no_ack(
1162 self,
1163 queue: str,
1164 no_ack: bool,
1165 ) -> None:
1166 broker = self.get_broker()
1167 sub = broker.subscriber(
1168 stream=StreamSub(queue, group=queue, consumer="reader", no_ack=no_ack),
1169 )
1171 async with self.patch_broker(broker) as br:
1172 await br.start()
1173 await br.publish("hello", stream=queue)
1175 with anyio.fail_after(3):
1176 message = await anext(aiter(sub))
1178 assert await message.decode() == "hello"
1180 client = await br.connect()
1181 pending = await client.xpending(queue, queue) # type: ignore[no-untyped-call]
1182 assert pending["pending"] == (0 if no_ack else 1)
1184 async def test_get_one_group_cursor_advances(
1185 self,
1186 queue: str,
1187 ) -> None:
1188 broker = self.get_broker()
1189 subscriber = broker.subscriber(
1190 stream=StreamSub(queue, group="test_group", consumer="test_consumer")
1191 )
1193 async with self.patch_broker(broker) as br:
1194 await br.start()
1196 await br.publish("first", stream=queue)
1197 await br.publish("second", stream=queue)
1199 msg1 = await subscriber.get_one(timeout=3)
1200 msg2 = await subscriber.get_one(timeout=3)
1202 assert msg1 is not None
1203 assert msg2 is not None
1204 assert await msg1.decode() == "first"
1205 assert await msg2.decode() == "second"
1207 async def test_iterator_group_cursor_advances(
1208 self,
1209 queue: str,
1210 ) -> None:
1211 broker = self.get_broker()
1212 subscriber = broker.subscriber(
1213 stream=StreamSub(queue, group="test_group", consumer="test_consumer")
1214 )
1216 async with self.patch_broker(broker) as br:
1217 await br.start()
1219 await br.publish("first", stream=queue)
1220 await br.publish("second", stream=queue)
1222 results = []
1223 async for msg in subscriber: 1223 ↛ 1228line 1223 didn't jump to line 1228 because the loop on line 1223 didn't complete
1224 results.append(await msg.decode())
1225 if len(results) >= 2:
1226 break
1228 assert results == ["first", "second"]
1230 @pytest.mark.slow()
1231 async def test_consume_stream_group_deleted(
1232 self,
1233 queue: str,
1234 mock: MagicMock,
1235 event: asyncio.Event,
1236 monkeypatch: pytest.MonkeyPatch,
1237 ) -> None:
1238 """Subscriber stops the app when the consumer group is deleted (NOGROUP)."""
1239 # The global conftest disables the task supervisor; this test is about
1240 # not entering its restart loop, so turn it back on.
1241 monkeypatch.setenv("FASTSTREAM_SUPERVISOR_DISABLED", "0")
1243 consume_broker = self.get_broker(apply_types=True)
1245 @consume_broker.subscriber(
1246 stream=StreamSub(queue, group="test_group", consumer="consumer1"),
1247 )
1248 async def handler(msg: RedisMessage) -> None:
1249 mock(msg)
1250 event.set()
1252 async with self.patch_broker(consume_broker) as br:
1253 fake_app = MagicMock()
1254 br.context.set_global("app", fake_app)
1256 await br.start()
1258 # Publish a message so the subscriber reads and starts consuming
1259 await br.publish("hello", stream=queue)
1260 await asyncio.wait_for(event.wait(), timeout=3)
1261 assert mock.call_count >= 1
1263 client = await br.connect()
1264 # Delete the stream — this removes the consumer group too
1265 await client.delete(queue)
1267 # Give the subscriber time to try reading and hit NOGROUP
1268 await asyncio.sleep(0.5)
1270 # Publish another message — subscriber should NOT receive it
1271 # because it already stopped due to NOGROUP
1272 event.clear()
1273 await br.publish("world", stream=queue)
1274 with pytest.raises(asyncio.TimeoutError):
1275 await asyncio.wait_for(event.wait(), timeout=1)
1277 # Stopped instead of restarting in a hot loop, and asked the app to exit
1278 assert not br.subscribers[0].running
1279 fake_app.exit.assert_called_once()