Coverage for faststream/redis/testing.py: 99%
226 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import re
2import uuid
3from collections.abc import AsyncGenerator, Generator, Iterable, Iterator, Sequence
4from concurrent.futures import ThreadPoolExecutor
5from contextlib import ExitStack, asynccontextmanager, contextmanager
6from dataclasses import dataclass
7from functools import partial
8from typing import (
9 TYPE_CHECKING,
10 Any,
11 Optional,
12 Protocol,
13 Union,
14 cast,
15 overload,
16)
17from unittest import mock
18from unittest.mock import AsyncMock, MagicMock
20import anyio
21from typing_extensions import TypedDict, override
23from faststream._internal.endpoint.utils import ParserComposition
24from faststream._internal.parser import DefaultCodec
25from faststream._internal.testing.broker import (
26 EnterType,
27 TestBroker,
28 change_producer,
29)
30from faststream.exceptions import SetupError, SubscriberNotFound
31from faststream.redis.broker.broker import RedisBroker
32from faststream.redis.configs.state import RedisClusterConnectionState
33from faststream.redis.message import (
34 BatchListMessage,
35 BatchStreamMessage,
36 DefaultListMessage,
37 DefaultStreamMessage,
38 PubSubMessage,
39 bDATA_KEY,
40)
41from faststream.redis.parser import MessageFormat, ParserConfig, RedisPubSubParser
42from faststream.redis.publisher.producer import RedisFastProducer
43from faststream.redis.response import DestinationType, RedisPublishCommand
44from faststream.redis.schemas import INCORRECT_SETUP_MSG
45from faststream.redis.subscriber.usecases.channel_subscriber import ChannelSubscriber
46from faststream.redis.subscriber.usecases.list_subscriber import _ListHandlerMixin
47from faststream.redis.subscriber.usecases.stream_subscriber import _StreamHandlerMixin
49if TYPE_CHECKING:
50 from fast_depends.library.serializer import SerializerProto
52 from faststream._internal.basic_types import SendableMessage
53 from faststream._internal.parser import CodecProto
54 from faststream.redis.publisher.usecase import LogicPublisher
55 from faststream.redis.subscriber.usecases.basic import LogicSubscriber
56 from faststream.response import Response
58__all__ = (
59 "PEL",
60 "TestRedisBroker",
61)
64@dataclass(kw_only=True)
65class Entry:
66 handler: "LogicSubscriber"
67 msg: Any
70PELKey = tuple[str | None, uuid.UUID, str | None]
73class PEL:
74 def __init__(self) -> None:
75 self.entries: dict[PELKey, Entry] = {}
76 self.put = MagicMock(wraps=self._put)
77 self.remove = MagicMock(wraps=self._remove)
79 def _remove(self, correlation_id: PELKey) -> None:
80 self.entries.pop(correlation_id)
82 def _put(
83 self,
84 msg: Any,
85 handler: "LogicSubscriber",
86 correlation_id: PELKey,
87 ) -> None:
88 self.entries.update({correlation_id: Entry(msg=msg, handler=handler)})
90 def get_entry(self, correlation_id: PELKey) -> Entry | None:
91 return self.entries.get(correlation_id)
94class TestRedisBroker(
95 TestBroker[RedisBroker, EnterType],
96 broker=RedisBroker,
97):
98 """A class to test Redis brokers."""
100 @overload
101 def __init__(
102 self: "TestRedisBroker[RedisBroker]",
103 broker: RedisBroker,
104 /,
105 *,
106 with_real: bool = False,
107 connect_only: bool | None = None,
108 pel: PEL | None = None,
109 ) -> None: ...
111 @overload
112 def __init__(
113 self: "TestRedisBroker[tuple[RedisBroker, ...]]",
114 *brokers: RedisBroker,
115 with_real: bool = False,
116 connect_only: bool | None = None,
117 pel: PEL | None = None,
118 ) -> None: ...
120 def __init__(
121 self,
122 *brokers: RedisBroker,
123 with_real: bool = False,
124 connect_only: bool | None = None,
125 pel: PEL | None = None,
126 ) -> None:
128 self.pel = pel or PEL()
130 super().__init__(
131 *brokers,
132 with_real=with_real,
133 connect_only=connect_only,
134 )
136 @asynccontextmanager
137 async def _create_ctx(self) -> AsyncGenerator[list[RedisBroker], None]:
138 with ExitStack() as cluster_stack:
139 for broker in self.brokers:
140 is_cluster = isinstance(
141 broker.config.broker_config.connection,
142 RedisClusterConnectionState,
143 )
144 if self.with_real and is_cluster:
145 with mock.patch.object(
146 broker,
147 "_connect",
148 wraps=partial(self._fake_connect, broker),
149 ):
150 _ = await broker.connect()
151 cluster_stack.enter_context(self._patch_producer(broker))
152 async with super()._create_ctx() as brokers:
153 yield brokers
155 @contextmanager
156 def _patch_producer(self, broker: RedisBroker) -> Generator[None, None, None]:
157 with ExitStack() as es:
158 es.enter_context(
159 change_producer(
160 broker.config.broker_config,
161 FakeProducer(broker, self.brokers, broker.config, self.pel),
162 ),
163 )
165 for publisher in cast("list[LogicPublisher]", broker.publishers):
166 es.enter_context(
167 change_producer(
168 publisher,
169 FakeProducer(broker, self.brokers, publisher.config, self.pel),
170 ),
171 )
173 yield
175 def create_publisher_fake_subscriber(
176 self,
177 broker: RedisBroker,
178 publisher: "LogicPublisher",
179 ) -> tuple["LogicSubscriber", bool]:
180 sub: LogicSubscriber | None = None
182 named_property = publisher.subscriber_property(name_only=True)
183 visitors = (ChannelVisitor(), ListVisitor(), StreamVisitor())
185 for handler in (
186 s for b in self.brokers for s in b.subscribers
187 ): # pragma: no branch
188 handler = cast("LogicSubscriber", handler)
189 for visitor in visitors:
190 if visitor.visit(**named_property, sub=handler):
191 sub = handler
192 break
194 if sub is None:
195 is_real = False
196 sub_options = publisher.subscriber_property(name_only=False)
197 sub = broker.subscriber(**sub_options, persistent=False)
198 else:
199 is_real = True
201 return sub, is_real
203 @staticmethod
204 async def _fake_connect( # type: ignore[override]
205 broker: RedisBroker,
206 *args: Any,
207 **kwargs: Any,
208 ) -> AsyncMock:
209 connection = MagicMock()
211 pub_sub = AsyncMock()
213 async def get_msg(*args: Any, timeout: float, **kwargs: Any) -> None:
214 await anyio.sleep(timeout)
216 pub_sub.get_message = get_msg
218 connection.pubsub.side_effect = lambda: pub_sub
219 connection.aclose = AsyncMock()
221 connection.xack = AsyncMock()
222 connection.xdel = AsyncMock()
224 connection_state = broker.config.broker_config.connection
225 connection_state._client = connection
226 connection_state._sync_cluster = MagicMock()
227 connection_state._thread_pool = ThreadPoolExecutor(max_workers=1)
228 connection_state._connected = True
229 return connection
232class FakeProducer(RedisFastProducer):
233 def __init__(
234 self,
235 broker: RedisBroker,
236 brokers: Sequence[RedisBroker],
237 config: ParserConfig,
238 pel: PEL | None = None,
239 ) -> None:
240 self.broker = broker
241 self.brokers = brokers
242 self._fake_config = config
244 default = RedisPubSubParser(config)
246 self._parser = ParserComposition(
247 broker._parser,
248 default.parse_message,
249 )
250 self._decoder = ParserComposition(
251 broker._decoder,
252 default.decode_message,
253 )
254 self.codec = broker.config.broker_codec or DefaultCodec()
255 self.pel = pel or PEL()
257 @property
258 def subscribers(self) -> "Iterable[LogicSubscriber]":
259 return (cast("LogicSubscriber", s) for b in self.brokers for s in b.subscribers)
261 @override
262 def _build_child(self, **kwargs: Any) -> "FakeProducer":
263 return FakeProducer(
264 broker=self.broker,
265 brokers=self.brokers,
266 config=self._fake_config,
267 )
269 @override
270 async def publish(self, cmd: "RedisPublishCommand") -> int | bytes:
271 body = await build_message(
272 message=cmd.body,
273 reply_to=cmd.reply_to,
274 correlation_id=cmd.correlation_id or self.broker.config.id_generator(),
275 headers=cmd.headers,
276 message_format=cmd.message_format,
277 serializer=self.broker.config.fd_config._serializer,
278 codec=self.codec,
279 )
280 destination = _make_destination_kwargs(cmd)
281 visitors = (ChannelVisitor(), ListVisitor(), StreamVisitor())
282 session_id = uuid.uuid4()
283 for visitor, visited_ch, handler in self._find_handlers(
284 destination=destination,
285 visitors=visitors,
286 cmd=cmd,
287 session_id=session_id,
288 ):
289 msg = visitor.get_message(
290 visited_ch,
291 body,
292 handler,
293 )
294 self._put_pel(msg=msg, cmd=cmd, handler=handler, session_id=session_id)
295 await self._execute_handler(msg, handler, session_id=session_id)
297 return 0
299 @override
300 async def request(self, cmd: "RedisPublishCommand") -> "PubSubMessage":
301 body = await build_message(
302 message=cmd.body,
303 correlation_id=cmd.correlation_id or self.broker.config.id_generator(),
304 headers=cmd.headers,
305 message_format=cmd.message_format,
306 serializer=self.broker.config.fd_config._serializer,
307 codec=self.codec,
308 )
310 destination = _make_destination_kwargs(cmd)
311 visitors = (ChannelVisitor(), ListVisitor(), StreamVisitor())
312 session_id = uuid.uuid4()
314 for visitor, visited_ch, handler in self._find_handlers(
315 destination=destination,
316 visitors=visitors,
317 cmd=cmd,
318 session_id=session_id,
319 ):
320 msg = visitor.get_message(
321 visited_ch,
322 body,
323 handler,
324 )
325 self._put_pel(msg=msg, cmd=cmd, handler=handler, session_id=session_id)
326 with anyio.fail_after(cmd.timeout):
327 return await self._execute_handler(msg, handler, session_id=session_id)
329 raise SubscriberNotFound
331 @override
332 async def publish_batch(self, cmd: "RedisPublishCommand") -> int:
333 data_to_send = [
334 await build_message(
335 m,
336 correlation_id=cmd.correlation_id or self.broker.config.id_generator(),
337 headers=cmd.headers,
338 message_format=cmd.message_format,
339 serializer=self.broker.config.fd_config._serializer,
340 codec=self.codec,
341 )
342 for m in cmd.batch_bodies
343 ]
344 session_id = uuid.uuid4()
346 for visitor, visited_ch, handler in self._find_handlers(
347 {"list": cmd.destination},
348 (ListVisitor(),),
349 cmd=cmd,
350 session_id=session_id,
351 ):
352 casted_handler = cast("_ListHandlerMixin", handler)
354 messages = [data_to_send] if casted_handler.list_sub.batch else data_to_send
355 for body in messages:
356 msg = visitor.get_message(
357 visited_ch,
358 body,
359 casted_handler,
360 )
361 self._put_pel(msg=msg, cmd=cmd, handler=handler, session_id=session_id)
362 await self._execute_handler(msg, handler, session_id=session_id)
364 return 0
366 async def _execute_handler(
367 self,
368 msg: Any,
369 handler: "LogicSubscriber",
370 session_id: uuid.UUID,
371 ) -> "PubSubMessage":
372 result = await handler.process_message(msg)
373 self._remove_pel(handler=handler, result=result, session_id=session_id)
374 return PubSubMessage(
375 type="message",
376 data=await build_message(
377 message=result.body,
378 headers=result.headers,
379 correlation_id=result.correlation_id or "",
380 message_format=handler.config.message_format,
381 serializer=self.broker.config.fd_config._serializer,
382 codec=self.codec,
383 ),
384 channel="",
385 pattern=None,
386 )
388 def _find_handlers(
389 self,
390 destination: "_DestinationKwargs",
391 visitors: "Sequence[Visitor]",
392 cmd: "RedisPublishCommand",
393 session_id: uuid.UUID,
394 ) -> "Iterator[tuple[Visitor, str, LogicSubscriber]]":
395 published_groups: set[tuple[str, str]] = set()
397 for handler in self.subscribers: # pragma: no branch
398 for visitor in visitors:
399 visited_ch = visitor.visit(**destination, sub=handler)
400 if visited_ch is None:
401 continue
402 if not self._return_handlers(
403 handler=handler,
404 visited_ch=visited_ch,
405 published_groups=published_groups,
406 cmd=cmd,
407 session_id=session_id,
408 ):
409 break
410 yield visitor, visited_ch, handler
411 break
413 def _return_handlers(
414 self,
415 handler: "LogicSubscriber",
416 visited_ch: str,
417 published_groups: set[tuple[str, str]],
418 cmd: "RedisPublishCommand",
419 session_id: uuid.UUID,
420 ) -> bool:
421 if isinstance(handler, _StreamHandlerMixin) and handler.stream_sub.group:
422 group_key = (visited_ch, handler.stream_sub.group)
424 if _handler_min_idle_time(handler) and self._check_pel(
425 handler=handler,
426 cmd=cmd,
427 session_id=session_id,
428 ):
429 return True
430 if group_key in published_groups:
431 return False
432 published_groups.add(group_key)
433 return True
434 return True
436 def _check_pel(
437 self,
438 handler: "LogicSubscriber",
439 cmd: "RedisPublishCommand",
440 session_id: uuid.UUID,
441 ) -> Optional["Entry"]:
442 return self.pel.get_entry(
443 correlation_id=(
444 cmd.correlation_id,
445 session_id,
446 _handler_group(handler),
447 )
448 )
450 def _put_pel(
451 self,
452 handler: "LogicSubscriber",
453 msg: Any,
454 cmd: "RedisPublishCommand",
455 session_id: uuid.UUID,
456 ) -> None:
457 if not _handler_no_ack(handler):
458 self.pel.put(
459 msg=msg,
460 handler=handler,
461 correlation_id=(
462 cmd.correlation_id,
463 session_id,
464 _handler_group(handler),
465 ),
466 )
468 def _remove_pel(
469 self,
470 result: "Response",
471 handler: "LogicSubscriber",
472 session_id: uuid.UUID,
473 ) -> None:
474 if result.correlation_id and not _handler_no_ack(handler):
475 self.pel.remove(
476 correlation_id=(
477 result.correlation_id,
478 session_id,
479 _handler_group(handler),
480 )
481 )
484async def build_message(
485 message: Union[Sequence["SendableMessage"], "SendableMessage"],
486 *,
487 correlation_id: str,
488 message_format: type["MessageFormat"],
489 reply_to: str = "",
490 headers: dict[str, Any] | None = None,
491 serializer: Optional["SerializerProto"] = None,
492 codec: Optional["CodecProto"] = None,
493) -> bytes:
494 return await message_format.encode(
495 message=message,
496 reply_to=reply_to,
497 headers=headers,
498 correlation_id=correlation_id,
499 serializer=serializer,
500 codec=codec,
501 )
504class Visitor(Protocol):
505 def visit(
506 self,
507 *,
508 channel: str | None,
509 list: str | None,
510 stream: str | None,
511 sub: "LogicSubscriber",
512 ) -> str | None: ...
514 def get_message(self, channel: str, body: Any, sub: "LogicSubscriber") -> Any: ...
517class ChannelVisitor(Visitor):
518 @override
519 def visit(
520 self,
521 *,
522 sub: "LogicSubscriber",
523 channel: str | None = None,
524 list: str | None = None,
525 stream: str | None = None,
526 ) -> str | None:
527 if channel is None or not isinstance(sub, ChannelSubscriber):
528 return None
530 sub_channel = sub.channel
532 if (
533 sub_channel.pattern
534 and bool(
535 re.fullmatch(
536 # Real Redis globs match the whole channel name, not a prefix.
537 # Escaped wholesale and then opened back up at the wildcard,
538 # the way `REDIS_ADDRESS_SYNTAX` does it: a Broker address may
539 # hold a literal brace, and `{2}` left bare is a quantifier.
540 re.escape(sub_channel.name).replace(r"\*", ".*"),
541 channel or "",
542 ),
543 )
544 ) or channel == sub_channel.name:
545 return channel
547 return None
549 @override
550 def get_message( # type: ignore[override]
551 self,
552 channel: str,
553 body: Any,
554 sub: "ChannelSubscriber",
555 ) -> Any:
556 return PubSubMessage(
557 type="message",
558 data=body,
559 channel=channel,
560 # Real Redis reports the pattern it was psubscribed with; this keeps
561 # reporting the template, as it did before the two were split apart.
562 # Aligning it belongs with #2450, which gives the template a meaning.
563 pattern=sub.channel.address.template.encode()
564 if sub.channel.pattern
565 else None,
566 )
569class ListVisitor(Visitor):
570 @override
571 def visit(
572 self,
573 *,
574 sub: "LogicSubscriber",
575 channel: str | None = None,
576 list: str | None = None,
577 stream: str | None = None,
578 ) -> str | None:
579 if list is None or not isinstance(sub, _ListHandlerMixin):
580 return None
582 if list == sub.list_sub.name:
583 return list
585 return None
587 @override
588 def get_message( # type: ignore[override]
589 self,
590 channel: str,
591 body: Any,
592 sub: "_ListHandlerMixin",
593 ) -> Any:
594 if sub.list_sub.batch:
595 return BatchListMessage(
596 type="blist",
597 channel=channel,
598 data=body if isinstance(body, list) else [body],
599 )
601 return DefaultListMessage(
602 type="list",
603 channel=channel,
604 data=body,
605 )
608class StreamVisitor(Visitor):
609 @override
610 def visit(
611 self,
612 *,
613 sub: "LogicSubscriber",
614 channel: str | None = None,
615 list: str | None = None,
616 stream: str | None = None,
617 ) -> str | None:
618 if stream is None or not isinstance(sub, _StreamHandlerMixin):
619 return None
621 if stream == sub.stream_sub.name:
622 return stream
624 return None
626 @override
627 def get_message( # type: ignore[override]
628 self,
629 channel: str,
630 body: Any,
631 sub: "_StreamHandlerMixin",
632 ) -> Any:
633 message: BatchStreamMessage | DefaultStreamMessage
634 if sub.stream_sub.batch:
635 message = BatchStreamMessage(
636 type="bstream",
637 channel=channel,
638 data=[{bDATA_KEY: body}],
639 message_ids=[],
640 )
641 else:
642 message = DefaultStreamMessage(
643 type="stream",
644 channel=channel,
645 data={bDATA_KEY: body},
646 message_ids=[],
647 )
649 if sub.stream_sub.claim_min_idle_time is not None:
650 # The in-memory broker has no claiming: every delivery is a new
651 # message, so expose the new-message metadata values
652 message["idle_times"] = [0]
653 message["delivery_counts"] = [0]
655 return message
658class _DestinationKwargs(TypedDict, total=False):
659 channel: str
660 list: str
661 stream: str
664def _make_destination_kwargs(cmd: RedisPublishCommand) -> _DestinationKwargs:
665 destination: _DestinationKwargs = {}
666 if cmd.destination_type is DestinationType.Channel:
667 destination["channel"] = cmd.destination
668 if cmd.destination_type is DestinationType.List:
669 destination["list"] = cmd.destination
670 if cmd.destination_type is DestinationType.Stream:
671 destination["stream"] = cmd.destination
673 if len(destination) != 1: 673 ↛ 674line 673 didn't jump to line 674 because the condition on line 673 was never true
674 raise SetupError(INCORRECT_SETUP_MSG)
676 return destination
679def _handler_group(handler: "LogicSubscriber") -> str | None:
680 if isinstance(handler, _StreamHandlerMixin):
681 return handler.stream_sub.group
682 return None
685def _handler_no_ack(handler: "LogicSubscriber") -> bool:
686 return isinstance(handler, _StreamHandlerMixin) and handler.stream_sub.no_ack
689def _handler_min_idle_time(handler: "LogicSubscriber") -> int | None:
690 if isinstance(handler, _StreamHandlerMixin): 690 ↛ 692line 690 didn't jump to line 692 because the condition on line 690 was always true
691 return handler.stream_sub.min_idle_time
692 return None