Coverage for faststream/confluent/testing.py: 90%
131 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
2from contextlib import ExitStack, contextmanager
3from datetime import datetime, timezone
4from typing import TYPE_CHECKING, Any, Optional, cast, overload
5from unittest.mock import AsyncMock, MagicMock
7import anyio
8from confluent_kafka import KafkaError, Message
9from typing_extensions import override
11from faststream._internal.endpoint.utils import ParserComposition
12from faststream._internal.kafka import encode_headers
13from faststream._internal.parser import BatchCodecProto, DefaultCodec
14from faststream._internal.testing.broker import (
15 EnterType,
16 TestBroker,
17 change_producer,
18)
19from faststream._internal.types import IdGenerator
20from faststream.confluent.broker import KafkaBroker
21from faststream.confluent.parser import AsyncConfluentParser
22from faststream.confluent.publisher.producer import AsyncConfluentFastProducer
23from faststream.confluent.publisher.usecase import BatchPublisher
24from faststream.confluent.schemas import TopicPartition
25from faststream.confluent.subscriber.usecase import BatchSubscriber
26from faststream.exceptions import SubscriberNotFound
27from faststream.message import gen_cor_id
29if TYPE_CHECKING:
30 from fast_depends.library.serializer import SerializerProto
32 from faststream._internal.basic_types import SendableMessage
33 from faststream._internal.parser import CodecProto
34 from faststream.confluent.publisher.usecase import LogicPublisher
35 from faststream.confluent.response import KafkaPublishCommand
36 from faststream.confluent.subscriber.usecase import LogicSubscriber
39__all__ = ("TestKafkaBroker",)
42class TestKafkaBroker(
43 TestBroker[KafkaBroker, EnterType],
44 broker=KafkaBroker,
45):
46 """A class to test Kafka brokers."""
48 @overload
49 def __init__(
50 self: "TestKafkaBroker[KafkaBroker]",
51 broker: KafkaBroker,
52 /,
53 *,
54 with_real: bool = False,
55 connect_only: bool | None = None,
56 ) -> None: ...
58 @overload
59 def __init__(
60 self: "TestKafkaBroker[tuple[KafkaBroker, ...]]",
61 *brokers: KafkaBroker,
62 with_real: bool = False,
63 connect_only: bool | None = None,
64 ) -> None: ...
66 def __init__(
67 self,
68 *brokers: KafkaBroker,
69 with_real: bool = False,
70 connect_only: bool | None = None,
71 ) -> None:
72 super().__init__(
73 *brokers,
74 with_real=with_real,
75 connect_only=connect_only,
76 )
78 @contextmanager
79 def _patch_producer(self, broker: KafkaBroker) -> Generator[None, None, None]:
80 fake_producer = FakeProducer(broker, self.brokers)
82 with ExitStack() as es:
83 es.enter_context(
84 change_producer(broker.config.broker_config, fake_producer),
85 )
86 yield
88 @staticmethod
89 async def _fake_connect( # type: ignore[override]
90 broker: KafkaBroker,
91 *args: Any,
92 **kwargs: Any,
93 ) -> Callable[..., AsyncMock]:
94 broker.config.broker_config.admin.admin_client = MagicMock()
95 return _fake_connection
97 def create_publisher_fake_subscriber(
98 self,
99 broker: KafkaBroker,
100 publisher: "LogicPublisher",
101 ) -> tuple["LogicSubscriber[Any]", bool]:
102 sub: LogicSubscriber[Any] | None = None
103 for handler in (s for b in self.brokers for s in b.subscribers):
104 handler = cast("LogicSubscriber[Any]", handler)
105 if _is_handler_matches(
106 handler,
107 topic=publisher.topic,
108 partition=publisher.partition,
109 ):
110 sub = handler
111 break
113 if sub is None:
114 is_real = False
116 topic_name = publisher.topic
118 if publisher.partition: 118 ↛ 119line 118 didn't jump to line 119 because the condition on line 118 was never true
119 tp = TopicPartition(
120 topic=topic_name,
121 partition=publisher.partition,
122 )
123 sub = broker.subscriber(
124 partitions=[tp],
125 batch=isinstance(publisher, BatchPublisher),
126 auto_offset_reset="earliest",
127 persistent=False,
128 )
129 else:
130 sub = broker.subscriber(
131 topic_name,
132 batch=isinstance(publisher, BatchPublisher),
133 auto_offset_reset="earliest",
134 persistent=False,
135 )
136 else:
137 is_real = True
139 return sub, is_real
142class FakeProducer(AsyncConfluentFastProducer):
143 """A fake Kafka producer for testing purposes.
145 This class extends AsyncConfluentFastProducer and is used to simulate Kafka message publishing during tests.
146 """
148 def __init__(
149 self,
150 broker: KafkaBroker,
151 brokers: Sequence[KafkaBroker],
152 ) -> None:
153 self.broker = broker
154 self.brokers = brokers
156 default = AsyncConfluentParser()
157 self._parser = ParserComposition(broker._parser, default.parse_message)
158 self._decoder = ParserComposition(broker._decoder, default.decode_message)
159 self.codec = broker.config.broker_codec or DefaultCodec()
161 @property
162 def subscribers(self) -> Iterable["LogicSubscriber[Any]"]:
163 return (
164 cast("LogicSubscriber[Any]", s) for b in self.brokers for s in b.subscribers
165 )
167 def __bool__(self) -> bool:
168 return True
170 @override
171 async def ping(self, timeout: float) -> bool:
172 return True
174 @override
175 async def publish(self, cmd: "KafkaPublishCommand") -> None:
176 """Publish a message to the Kafka broker."""
177 incoming = await build_message(
178 message=cmd.body,
179 topic=cmd.destination,
180 key=cmd.key,
181 partition=cmd.partition,
182 timestamp_ms=cmd.timestamp_ms,
183 headers=cmd.headers,
184 correlation_id=cmd.correlation_id,
185 reply_to=cmd.reply_to,
186 serializer=self.broker.config.fd_config._serializer,
187 codec=self.codec,
188 id_generator=self.broker.config.id_generator,
189 )
191 for handler in _find_handler(
192 self.subscribers,
193 cmd.destination,
194 cmd.partition,
195 ):
196 msg_to_send = (
197 (incoming,) if isinstance(handler, BatchSubscriber) else incoming
198 )
200 await self._execute_handler(msg_to_send, cmd.destination, handler)
202 @override
203 async def publish_batch(self, cmd: "KafkaPublishCommand") -> None:
204 """Publish a batch of messages to the Kafka broker."""
205 serializer = self.broker.config.fd_config._serializer
207 if isinstance(self.codec, BatchCodecProto):
208 encoded = await self.codec.encode_batch(cmd.batch_bodies, serializer)
209 else:
210 encoded = [
211 await self.codec.encode(body, serializer) for body in cmd.batch_bodies
212 ]
214 for handler in _find_handler(
215 self.subscribers,
216 cmd.destination,
217 cmd.partition,
218 ):
219 messages = [
220 _build_mock_message(
221 body=body,
222 content_type=content_type,
223 topic=cmd.destination,
224 partition=cmd.partition,
225 timestamp_ms=cmd.timestamp_ms,
226 key=cmd.key_for(message_position),
227 headers=cmd.headers,
228 correlation_id=cmd.correlation_id,
229 reply_to=cmd.reply_to,
230 id_generator=self.broker.config.id_generator,
231 )
232 for message_position, (body, content_type) in enumerate(encoded)
233 ]
235 if isinstance(handler, BatchSubscriber): 235 ↛ 239line 235 didn't jump to line 239 because the condition on line 235 was always true
236 await self._execute_handler(tuple(messages), cmd.destination, handler)
238 else:
239 for m in messages:
240 await self._execute_handler(m, cmd.destination, handler)
242 @override
243 async def request(self, cmd: "KafkaPublishCommand") -> "MockConfluentMessage":
244 incoming = await build_message(
245 message=cmd.body,
246 topic=cmd.destination,
247 key=cmd.key,
248 partition=cmd.partition,
249 timestamp_ms=cmd.timestamp_ms,
250 headers=cmd.headers,
251 correlation_id=cmd.correlation_id,
252 serializer=self.broker.config.fd_config._serializer,
253 codec=self.codec,
254 id_generator=self.broker.config.id_generator,
255 )
257 for handler in _find_handler(
258 self.subscribers,
259 cmd.destination,
260 cmd.partition,
261 ):
262 msg_to_send = (
263 (incoming,) if isinstance(handler, BatchSubscriber) else incoming
264 )
266 with anyio.fail_after(cmd.timeout):
267 return await self._execute_handler(
268 msg_to_send,
269 cmd.destination,
270 handler,
271 )
273 raise SubscriberNotFound
275 async def _execute_handler(
276 self,
277 msg: Any,
278 topic: str,
279 handler: "LogicSubscriber[Any]",
280 ) -> "MockConfluentMessage":
281 result = await handler.process_message(msg)
283 return await build_message(
284 topic=topic,
285 message=result.body,
286 headers=result.headers,
287 correlation_id=result.correlation_id,
288 serializer=self.broker.config.fd_config._serializer,
289 codec=self.codec,
290 id_generator=self.broker.config.id_generator,
291 )
294# A `Message` so that the reader of the Kafka fields takes it for the client's own
295class MockConfluentMessage(Message):
296 """The message the in-memory broker delivers, answering as the client's does."""
298 def __init__(
299 self,
300 raw_msg: bytes | None,
301 topic: str,
302 key: bytes | str,
303 headers: Sequence[tuple[str, str | bytes | None]],
304 offset: int,
305 partition: int,
306 timestamp_type: int,
307 timestamp_ms: int,
308 error: KafkaError | None = None,
309 ) -> None:
310 self._raw_msg = raw_msg
311 self._topic = topic
313 if isinstance(key, str): 313 ↛ 314line 313 didn't jump to line 314 because the condition on line 313 was never true
314 self._key = key.encode()
315 else:
316 self._key = key
318 self._headers = list(headers)
319 self._error = error
320 self._offset = offset
321 self._partition = partition
322 self._timestamp = (timestamp_type, timestamp_ms)
324 def len(self) -> int:
325 return 0 if self._raw_msg is None else len(self._raw_msg)
327 # `Message.__len__` reads the C-level value this mock never sets
328 def __len__(self) -> int:
329 return self.len()
331 def error(self) -> KafkaError | None:
332 return self._error
334 def headers(self) -> list[tuple[str, str | bytes | None]]:
335 return self._headers
337 def key(self) -> bytes:
338 return self._key
340 def offset(self) -> int:
341 return self._offset
343 def partition(self) -> int:
344 return self._partition
346 def timestamp(self) -> tuple[int, int]:
347 return self._timestamp
349 def topic(self) -> str:
350 return self._topic
352 def value(self) -> bytes | None:
353 return self._raw_msg
356async def build_message(
357 message: "SendableMessage",
358 topic: str,
359 *,
360 correlation_id: str | None = None,
361 partition: int | None = None,
362 timestamp_ms: int | None = None,
363 key: bytes | str | None = None,
364 headers: Mapping[str, str | bytes | None] | None = None,
365 reply_to: str = "",
366 serializer: Optional["SerializerProto"] = None,
367 codec: Optional["CodecProto"] = None,
368 id_generator: IdGenerator = gen_cor_id,
369) -> MockConfluentMessage:
370 """Build a mock confluent_kafka.Message for a sendable message."""
371 if message is None:
372 # keep a real tombstone (message.value() is None) distinct from b""
373 msg, content_type = None, None
374 else:
375 codec_instance = codec or DefaultCodec()
376 msg, content_type = await codec_instance.encode(message, serializer)
377 k = key or b""
378 headers = {
379 "content-type": content_type or "",
380 "correlation_id": correlation_id or id_generator(),
381 "reply_to": reply_to,
382 **(headers or {}),
383 }
385 # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#confluent_kafka.Message.timestamp
386 return MockConfluentMessage(
387 raw_msg=msg,
388 topic=topic,
389 key=k,
390 headers=encode_headers(headers),
391 offset=0,
392 partition=partition or 0,
393 timestamp_type=1,
394 timestamp_ms=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000),
395 )
398def _build_mock_message(
399 body: bytes,
400 content_type: str | None,
401 topic: str,
402 partition: int | None = None,
403 timestamp_ms: int | None = None,
404 key: bytes | str | None = None,
405 headers: Mapping[str, str | bytes | None] | None = None,
406 correlation_id: str | None = None,
407 reply_to: str = "",
408 id_generator: IdGenerator = gen_cor_id,
409) -> MockConfluentMessage:
410 k = key or b""
411 h = {
412 "content-type": content_type or "",
413 "correlation_id": correlation_id or id_generator(),
414 "reply_to": reply_to,
415 **(headers or {}),
416 }
417 return MockConfluentMessage(
418 raw_msg=body,
419 topic=topic,
420 key=k,
421 headers=encode_headers(h),
422 offset=0,
423 partition=partition or 0,
424 timestamp_type=1,
425 timestamp_ms=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000),
426 )
429def _fake_connection(*args: Any, **kwargs: Any) -> AsyncMock:
430 mock = AsyncMock()
431 mock.getone.return_value = MagicMock()
432 mock.getmany.return_value = [MagicMock()]
433 return mock
436def _find_handler(
437 subscribers: Iterable["LogicSubscriber[Any]"],
438 topic: str,
439 partition: int | None,
440) -> Generator["LogicSubscriber[Any]", None, None]:
441 published_groups = set()
442 for handler in subscribers: # pragma: no branch
443 if _is_handler_matches(handler, topic, partition):
444 if handler.group_id:
445 if handler.group_id in published_groups:
446 continue
447 else:
448 published_groups.add(handler.group_id)
449 yield handler
452def _is_handler_matches(
453 handler: "LogicSubscriber[Any]",
454 topic: str,
455 partition: int | None,
456) -> bool:
457 return bool(
458 any(
459 p.topic == topic and (partition is None or p.partition == partition)
460 for p in handler.partitions
461 )
462 or any(t.name == topic for t in handler.topics),
463 )