Coverage for faststream/kafka/testing.py: 89%
111 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import re
2from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
3from contextlib import ExitStack, contextmanager
4from datetime import datetime, timezone
5from typing import TYPE_CHECKING, Any, Optional, cast, overload
6from unittest.mock import AsyncMock, MagicMock
8import anyio
9from aiokafka import ConsumerRecord
10from typing_extensions import override
12from faststream._internal.endpoint.utils import ParserComposition
13from faststream._internal.kafka import encode_headers
14from faststream._internal.parser import BatchCodecProto, DefaultCodec
15from faststream._internal.testing.broker import (
16 EnterType,
17 TestBroker,
18 change_producer,
19)
20from faststream._internal.types import IdGenerator
21from faststream.exceptions import SubscriberNotFound
22from faststream.kafka import TopicPartition
23from faststream.kafka.broker import KafkaBroker
24from faststream.kafka.message import KafkaMessage
25from faststream.kafka.parser import AioKafkaParser
26from faststream.kafka.publisher.producer import AioKafkaFastProducer
27from faststream.kafka.publisher.usecase import BatchPublisher
28from faststream.kafka.subscriber.usecase import BatchSubscriber
29from faststream.message import gen_cor_id
31if TYPE_CHECKING:
32 from fast_depends.library.serializer import SerializerProto
34 from faststream._internal.basic_types import SendableMessage
35 from faststream._internal.parser import CodecProto
36 from faststream.kafka.publisher.usecase import LogicPublisher
37 from faststream.kafka.response import KafkaPublishCommand
38 from faststream.kafka.subscriber.usecase import LogicSubscriber
40__all__ = ("TestKafkaBroker",)
43class TestKafkaBroker(
44 TestBroker[KafkaBroker, EnterType],
45 broker=KafkaBroker,
46):
47 """A class to test Kafka brokers."""
49 @overload
50 def __init__(
51 self: "TestKafkaBroker[KafkaBroker]",
52 broker: KafkaBroker,
53 /,
54 *,
55 with_real: bool = False,
56 connect_only: bool | None = None,
57 ) -> None: ...
59 @overload
60 def __init__(
61 self: "TestKafkaBroker[tuple[KafkaBroker, ...]]",
62 *brokers: KafkaBroker,
63 with_real: bool = False,
64 connect_only: bool | None = None,
65 ) -> None: ...
67 def __init__(
68 self,
69 *brokers: KafkaBroker,
70 with_real: bool = False,
71 connect_only: bool | None = None,
72 ) -> None:
73 super().__init__(
74 *brokers,
75 with_real=with_real,
76 connect_only=connect_only,
77 )
79 @contextmanager
80 def _patch_producer(self, broker: KafkaBroker) -> Generator[None, None, None]:
81 fake_producer = FakeProducer(broker, self.brokers)
83 with ExitStack() as es:
84 es.enter_context(
85 change_producer(broker.config.broker_config, fake_producer),
86 )
87 yield
89 @staticmethod
90 async def _fake_connect( # type: ignore[override]
91 broker: KafkaBroker,
92 *args: Any,
93 **kwargs: Any,
94 ) -> Callable[..., AsyncMock]:
95 builder = MagicMock(return_value=FakeConsumer())
96 broker.config.broker_config.builder = builder
98 return _fake_connection
100 def create_publisher_fake_subscriber(
101 self,
102 broker: KafkaBroker,
103 publisher: "LogicPublisher",
104 ) -> tuple["LogicSubscriber[Any]", bool]:
105 sub: LogicSubscriber[Any] | None = None
106 for handler in (s for b in self.brokers for s in b.subscribers):
107 handler = cast("LogicSubscriber[Any]", handler)
108 if _is_handler_matches(handler, publisher.topic, publisher.partition):
109 sub = handler
110 break
112 if sub is None:
113 is_real = False
115 topic_name = publisher.topic
117 if publisher.partition: 117 ↛ 118line 117 didn't jump to line 118 because the condition on line 117 was never true
118 tp = TopicPartition(
119 topic=topic_name,
120 partition=publisher.partition,
121 )
122 sub = broker.subscriber(
123 partitions=[tp],
124 batch=isinstance(publisher, BatchPublisher),
125 persistent=False,
126 )
127 else:
128 sub = broker.subscriber(
129 topic_name,
130 batch=isinstance(publisher, BatchPublisher),
131 persistent=False,
132 )
133 else:
134 is_real = True
136 return sub, is_real
139class FakeConsumer:
140 async def start(self) -> None:
141 pass
143 async def stop(self) -> None:
144 pass
146 def subscribe(self, *args: Any, **kwargs: Any) -> None:
147 pass
150class FakeProducer(AioKafkaFastProducer):
151 """A fake Kafka producer for testing purposes.
153 This class extends AioKafkaFastProducer and is used to simulate Kafka message publishing during tests.
154 """
156 def __init__(
157 self,
158 broker: KafkaBroker,
159 brokers: Sequence[KafkaBroker],
160 ) -> None:
161 self.broker = broker
162 self.brokers = brokers
164 default = AioKafkaParser(
165 msg_class=KafkaMessage,
166 regex=None,
167 )
169 self._parser = ParserComposition(broker._parser, default.parse_message)
170 self._decoder = ParserComposition(broker._decoder, default.decode_message)
171 self.codec = broker.config.broker_codec or DefaultCodec()
173 @property
174 def subscribers(self) -> Iterable["LogicSubscriber[Any]"]:
175 return (
176 cast("LogicSubscriber[Any]", s) for b in self.brokers for s in b.subscribers
177 )
179 def __bool__(self) -> bool:
180 return True
182 @property
183 def closed(self) -> bool:
184 return False
186 @override
187 async def publish(self, cmd: "KafkaPublishCommand") -> None:
188 """Publish a message to the Kafka broker."""
189 incoming = await build_message(
190 message=cmd.body,
191 topic=cmd.destination,
192 key=cmd.key,
193 partition=cmd.partition,
194 timestamp_ms=cmd.timestamp_ms,
195 headers=cmd.headers,
196 correlation_id=cmd.correlation_id,
197 reply_to=cmd.reply_to,
198 serializer=self.broker.config.fd_config._serializer,
199 codec=self.codec,
200 id_generator=self.broker.config.id_generator,
201 )
203 for handler in _find_handler(
204 self.subscribers,
205 cmd.destination,
206 cmd.partition,
207 ):
208 msg_to_send = (
209 (incoming,) if isinstance(handler, BatchSubscriber) else incoming
210 )
212 await self._execute_handler(msg_to_send, cmd.destination, handler)
214 @override
215 async def request(self, cmd: "KafkaPublishCommand") -> "ConsumerRecord":
216 incoming = await build_message(
217 message=cmd.body,
218 topic=cmd.destination,
219 key=cmd.key,
220 partition=cmd.partition,
221 timestamp_ms=cmd.timestamp_ms,
222 headers=cmd.headers,
223 correlation_id=cmd.correlation_id,
224 serializer=self.broker.config.fd_config._serializer,
225 codec=self.codec,
226 id_generator=self.broker.config.id_generator,
227 )
229 for handler in _find_handler(
230 self.subscribers,
231 cmd.destination,
232 cmd.partition,
233 ):
234 msg_to_send = (
235 (incoming,) if isinstance(handler, BatchSubscriber) else incoming
236 )
238 with anyio.fail_after(cmd.timeout):
239 return await self._execute_handler(
240 msg_to_send,
241 cmd.destination,
242 handler,
243 )
245 raise SubscriberNotFound
247 @override
248 async def publish_batch(
249 self,
250 cmd: "KafkaPublishCommand",
251 ) -> None:
252 """Publish a batch of messages to the Kafka broker."""
253 serializer = self.broker.config.fd_config._serializer
255 if isinstance(self.codec, BatchCodecProto):
256 encoded = await self.codec.encode_batch(cmd.batch_bodies, serializer)
257 else:
258 encoded = [
259 await self.codec.encode(body, serializer) for body in cmd.batch_bodies
260 ]
262 for handler in _find_handler(
263 self.subscribers,
264 cmd.destination,
265 cmd.partition,
266 ):
267 messages = [
268 _build_record(
269 body=body,
270 content_type=content_type,
271 topic=cmd.destination,
272 partition=cmd.partition,
273 timestamp_ms=cmd.timestamp_ms,
274 key=cmd.key_for(message_position),
275 headers=cmd.headers,
276 correlation_id=cmd.correlation_id,
277 reply_to=cmd.reply_to,
278 id_generator=self.broker.config.id_generator,
279 )
280 for message_position, (body, content_type) in enumerate(encoded)
281 ]
283 if isinstance(handler, BatchSubscriber): 283 ↛ 287line 283 didn't jump to line 287 because the condition on line 283 was always true
284 await self._execute_handler(tuple(messages), cmd.destination, handler)
286 else:
287 for m in messages:
288 await self._execute_handler(m, cmd.destination, handler)
290 async def _execute_handler(
291 self,
292 msg: Any,
293 topic: str,
294 handler: "LogicSubscriber[Any]",
295 ) -> "ConsumerRecord":
296 result = await handler.process_message(msg)
298 return await build_message(
299 topic=topic,
300 message=result.body,
301 headers=result.headers,
302 correlation_id=result.correlation_id,
303 serializer=self.broker.config.fd_config._serializer,
304 codec=self.codec,
305 id_generator=self.broker.config.id_generator,
306 )
309async def build_message(
310 message: "SendableMessage",
311 topic: str,
312 partition: int | None = None,
313 timestamp_ms: int | None = None,
314 key: bytes | None = None,
315 headers: Mapping[str, str | bytes | None] | None = None,
316 correlation_id: str | None = None,
317 *,
318 reply_to: str = "",
319 serializer: Optional["SerializerProto"],
320 codec: Optional["CodecProto"] = None,
321 id_generator: IdGenerator = gen_cor_id,
322) -> "ConsumerRecord":
323 """Build a Kafka ConsumerRecord for a sendable message."""
324 if message is None and key is not None:
325 # keyed None is a real tombstone, matching publish()'s own rule
326 # (aiokafka needs a key or value, a keyless None still goes b"")
327 msg, content_type = None, None
328 else:
329 msg, content_type = await (codec or DefaultCodec()).encode(message, serializer)
331 k = key or b""
333 headers = {
334 "content-type": content_type or "",
335 "correlation_id": correlation_id or id_generator(),
336 **(headers or {}),
337 }
339 if reply_to:
340 headers["reply_to"] = headers.get("reply_to", reply_to)
342 return ConsumerRecord(
343 value=msg,
344 topic=topic,
345 partition=partition or 0,
346 key=k,
347 serialized_key_size=len(k),
348 serialized_value_size=0 if msg is None else len(msg),
349 checksum=0 if msg is None else sum(msg),
350 offset=0,
351 headers=encode_headers(headers),
352 timestamp_type=1,
353 timestamp=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000),
354 )
357def _build_record(
358 body: bytes,
359 content_type: str | None,
360 topic: str,
361 partition: int | None = None,
362 timestamp_ms: int | None = None,
363 key: bytes | None = None,
364 headers: Mapping[str, str | bytes | None] | None = None,
365 correlation_id: str | None = None,
366 reply_to: str = "",
367 id_generator: IdGenerator = gen_cor_id,
368) -> "ConsumerRecord":
369 k = key or b""
370 h = {
371 "content-type": content_type or "",
372 "correlation_id": correlation_id or id_generator(),
373 **(headers or {}),
374 }
375 if reply_to: 375 ↛ 376line 375 didn't jump to line 376 because the condition on line 375 was never true
376 h["reply_to"] = h.get("reply_to", reply_to)
377 return ConsumerRecord(
378 value=body,
379 topic=topic,
380 partition=partition or 0,
381 key=k,
382 serialized_key_size=len(k),
383 serialized_value_size=len(body),
384 checksum=sum(body),
385 offset=0,
386 headers=encode_headers(h),
387 timestamp_type=1,
388 timestamp=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000),
389 )
392def _fake_connection(*args: Any, **kwargs: Any) -> AsyncMock:
393 mock = AsyncMock()
394 mock.subscribe = MagicMock
395 mock.assign = MagicMock
396 return mock
399def _find_handler(
400 subscribers: Iterable["LogicSubscriber[Any]"],
401 topic: str,
402 partition: int | None,
403) -> Generator["LogicSubscriber[Any]", None, None]:
404 published_groups = set()
405 for handler in subscribers: # pragma: no branch
406 if _is_handler_matches(handler, topic, partition):
407 if handler.group_id:
408 if handler.group_id in published_groups:
409 continue
410 else:
411 published_groups.add(handler.group_id)
412 yield handler
415def _is_handler_matches(
416 handler: "LogicSubscriber[Any]",
417 topic: str,
418 partition: int | None,
419) -> bool:
420 if any(
421 p.topic == topic and (partition is None or p.partition == partition)
422 for p in handler.partitions
423 ):
424 return True
426 if any(t.name == topic for t in handler.topics):
427 return True
429 pattern = handler.pattern
430 return bool(pattern and re.match(pattern.broker_address, topic))