Coverage for faststream/confluent/testing.py: 90%

131 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1from collections.abc import Callable, Generator, Iterable, Mapping, Sequence 

2from contextlib import ExitStack, contextmanager 

3from datetime import datetime, timezone 

4from typing import TYPE_CHECKING, Any, Optional, cast, overload 

5from unittest.mock import AsyncMock, MagicMock 

6 

7import anyio 

8from confluent_kafka import KafkaError, Message 

9from typing_extensions import override 

10 

11from faststream._internal.endpoint.utils import ParserComposition 

12from faststream._internal.kafka import encode_headers 

13from faststream._internal.parser import BatchCodecProto, DefaultCodec 

14from faststream._internal.testing.broker import ( 

15 EnterType, 

16 TestBroker, 

17 change_producer, 

18) 

19from faststream._internal.types import IdGenerator 

20from faststream.confluent.broker import KafkaBroker 

21from faststream.confluent.parser import AsyncConfluentParser 

22from faststream.confluent.publisher.producer import AsyncConfluentFastProducer 

23from faststream.confluent.publisher.usecase import BatchPublisher 

24from faststream.confluent.schemas import TopicPartition 

25from faststream.confluent.subscriber.usecase import BatchSubscriber 

26from faststream.exceptions import SubscriberNotFound 

27from faststream.message import gen_cor_id 

28 

29if TYPE_CHECKING: 

30 from fast_depends.library.serializer import SerializerProto 

31 

32 from faststream._internal.basic_types import SendableMessage 

33 from faststream._internal.parser import CodecProto 

34 from faststream.confluent.publisher.usecase import LogicPublisher 

35 from faststream.confluent.response import KafkaPublishCommand 

36 from faststream.confluent.subscriber.usecase import LogicSubscriber 

37 

38 

39__all__ = ("TestKafkaBroker",) 

40 

41 

42class TestKafkaBroker( 

43 TestBroker[KafkaBroker, EnterType], 

44 broker=KafkaBroker, 

45): 

46 """A class to test Kafka brokers.""" 

47 

48 @overload 

49 def __init__( 

50 self: "TestKafkaBroker[KafkaBroker]", 

51 broker: KafkaBroker, 

52 /, 

53 *, 

54 with_real: bool = False, 

55 connect_only: bool | None = None, 

56 ) -> None: ... 

57 

58 @overload 

59 def __init__( 

60 self: "TestKafkaBroker[tuple[KafkaBroker, ...]]", 

61 *brokers: KafkaBroker, 

62 with_real: bool = False, 

63 connect_only: bool | None = None, 

64 ) -> None: ... 

65 

66 def __init__( 

67 self, 

68 *brokers: KafkaBroker, 

69 with_real: bool = False, 

70 connect_only: bool | None = None, 

71 ) -> None: 

72 super().__init__( 

73 *brokers, 

74 with_real=with_real, 

75 connect_only=connect_only, 

76 ) 

77 

78 @contextmanager 

79 def _patch_producer(self, broker: KafkaBroker) -> Generator[None, None, None]: 

80 fake_producer = FakeProducer(broker, self.brokers) 

81 

82 with ExitStack() as es: 

83 es.enter_context( 

84 change_producer(broker.config.broker_config, fake_producer), 

85 ) 

86 yield 

87 

88 @staticmethod 

89 async def _fake_connect( # type: ignore[override] 

90 broker: KafkaBroker, 

91 *args: Any, 

92 **kwargs: Any, 

93 ) -> Callable[..., AsyncMock]: 

94 broker.config.broker_config.admin.admin_client = MagicMock() 

95 return _fake_connection 

96 

97 def create_publisher_fake_subscriber( 

98 self, 

99 broker: KafkaBroker, 

100 publisher: "LogicPublisher", 

101 ) -> tuple["LogicSubscriber[Any]", bool]: 

102 sub: LogicSubscriber[Any] | None = None 

103 for handler in (s for b in self.brokers for s in b.subscribers): 

104 handler = cast("LogicSubscriber[Any]", handler) 

105 if _is_handler_matches( 

106 handler, 

107 topic=publisher.topic, 

108 partition=publisher.partition, 

109 ): 

110 sub = handler 

111 break 

112 

113 if sub is None: 

114 is_real = False 

115 

116 topic_name = publisher.topic 

117 

118 if publisher.partition: 118 ↛ 119line 118 didn't jump to line 119 because the condition on line 118 was never true

119 tp = TopicPartition( 

120 topic=topic_name, 

121 partition=publisher.partition, 

122 ) 

123 sub = broker.subscriber( 

124 partitions=[tp], 

125 batch=isinstance(publisher, BatchPublisher), 

126 auto_offset_reset="earliest", 

127 persistent=False, 

128 ) 

129 else: 

130 sub = broker.subscriber( 

131 topic_name, 

132 batch=isinstance(publisher, BatchPublisher), 

133 auto_offset_reset="earliest", 

134 persistent=False, 

135 ) 

136 else: 

137 is_real = True 

138 

139 return sub, is_real 

140 

141 

142class FakeProducer(AsyncConfluentFastProducer): 

143 """A fake Kafka producer for testing purposes. 

144 

145 This class extends AsyncConfluentFastProducer and is used to simulate Kafka message publishing during tests. 

146 """ 

147 

148 def __init__( 

149 self, 

150 broker: KafkaBroker, 

151 brokers: Sequence[KafkaBroker], 

152 ) -> None: 

153 self.broker = broker 

154 self.brokers = brokers 

155 

156 default = AsyncConfluentParser() 

157 self._parser = ParserComposition(broker._parser, default.parse_message) 

158 self._decoder = ParserComposition(broker._decoder, default.decode_message) 

159 self.codec = broker.config.broker_codec or DefaultCodec() 

160 

161 @property 

162 def subscribers(self) -> Iterable["LogicSubscriber[Any]"]: 

163 return ( 

164 cast("LogicSubscriber[Any]", s) for b in self.brokers for s in b.subscribers 

165 ) 

166 

167 def __bool__(self) -> bool: 

168 return True 

169 

170 @override 

171 async def ping(self, timeout: float) -> bool: 

172 return True 

173 

174 @override 

175 async def publish(self, cmd: "KafkaPublishCommand") -> None: 

176 """Publish a message to the Kafka broker.""" 

177 incoming = await build_message( 

178 message=cmd.body, 

179 topic=cmd.destination, 

180 key=cmd.key, 

181 partition=cmd.partition, 

182 timestamp_ms=cmd.timestamp_ms, 

183 headers=cmd.headers, 

184 correlation_id=cmd.correlation_id, 

185 reply_to=cmd.reply_to, 

186 serializer=self.broker.config.fd_config._serializer, 

187 codec=self.codec, 

188 id_generator=self.broker.config.id_generator, 

189 ) 

190 

191 for handler in _find_handler( 

192 self.subscribers, 

193 cmd.destination, 

194 cmd.partition, 

195 ): 

196 msg_to_send = ( 

197 (incoming,) if isinstance(handler, BatchSubscriber) else incoming 

198 ) 

199 

200 await self._execute_handler(msg_to_send, cmd.destination, handler) 

201 

202 @override 

203 async def publish_batch(self, cmd: "KafkaPublishCommand") -> None: 

204 """Publish a batch of messages to the Kafka broker.""" 

205 serializer = self.broker.config.fd_config._serializer 

206 

207 if isinstance(self.codec, BatchCodecProto): 

208 encoded = await self.codec.encode_batch(cmd.batch_bodies, serializer) 

209 else: 

210 encoded = [ 

211 await self.codec.encode(body, serializer) for body in cmd.batch_bodies 

212 ] 

213 

214 for handler in _find_handler( 

215 self.subscribers, 

216 cmd.destination, 

217 cmd.partition, 

218 ): 

219 messages = [ 

220 _build_mock_message( 

221 body=body, 

222 content_type=content_type, 

223 topic=cmd.destination, 

224 partition=cmd.partition, 

225 timestamp_ms=cmd.timestamp_ms, 

226 key=cmd.key_for(message_position), 

227 headers=cmd.headers, 

228 correlation_id=cmd.correlation_id, 

229 reply_to=cmd.reply_to, 

230 id_generator=self.broker.config.id_generator, 

231 ) 

232 for message_position, (body, content_type) in enumerate(encoded) 

233 ] 

234 

235 if isinstance(handler, BatchSubscriber): 235 ↛ 239line 235 didn't jump to line 239 because the condition on line 235 was always true

236 await self._execute_handler(tuple(messages), cmd.destination, handler) 

237 

238 else: 

239 for m in messages: 

240 await self._execute_handler(m, cmd.destination, handler) 

241 

242 @override 

243 async def request(self, cmd: "KafkaPublishCommand") -> "MockConfluentMessage": 

244 incoming = await build_message( 

245 message=cmd.body, 

246 topic=cmd.destination, 

247 key=cmd.key, 

248 partition=cmd.partition, 

249 timestamp_ms=cmd.timestamp_ms, 

250 headers=cmd.headers, 

251 correlation_id=cmd.correlation_id, 

252 serializer=self.broker.config.fd_config._serializer, 

253 codec=self.codec, 

254 id_generator=self.broker.config.id_generator, 

255 ) 

256 

257 for handler in _find_handler( 

258 self.subscribers, 

259 cmd.destination, 

260 cmd.partition, 

261 ): 

262 msg_to_send = ( 

263 (incoming,) if isinstance(handler, BatchSubscriber) else incoming 

264 ) 

265 

266 with anyio.fail_after(cmd.timeout): 

267 return await self._execute_handler( 

268 msg_to_send, 

269 cmd.destination, 

270 handler, 

271 ) 

272 

273 raise SubscriberNotFound 

274 

275 async def _execute_handler( 

276 self, 

277 msg: Any, 

278 topic: str, 

279 handler: "LogicSubscriber[Any]", 

280 ) -> "MockConfluentMessage": 

281 result = await handler.process_message(msg) 

282 

283 return await build_message( 

284 topic=topic, 

285 message=result.body, 

286 headers=result.headers, 

287 correlation_id=result.correlation_id, 

288 serializer=self.broker.config.fd_config._serializer, 

289 codec=self.codec, 

290 id_generator=self.broker.config.id_generator, 

291 ) 

292 

293 

294# A `Message` so that the reader of the Kafka fields takes it for the client's own 

295class MockConfluentMessage(Message): 

296 """The message the in-memory broker delivers, answering as the client's does.""" 

297 

298 def __init__( 

299 self, 

300 raw_msg: bytes | None, 

301 topic: str, 

302 key: bytes | str, 

303 headers: Sequence[tuple[str, str | bytes | None]], 

304 offset: int, 

305 partition: int, 

306 timestamp_type: int, 

307 timestamp_ms: int, 

308 error: KafkaError | None = None, 

309 ) -> None: 

310 self._raw_msg = raw_msg 

311 self._topic = topic 

312 

313 if isinstance(key, str): 313 ↛ 314line 313 didn't jump to line 314 because the condition on line 313 was never true

314 self._key = key.encode() 

315 else: 

316 self._key = key 

317 

318 self._headers = list(headers) 

319 self._error = error 

320 self._offset = offset 

321 self._partition = partition 

322 self._timestamp = (timestamp_type, timestamp_ms) 

323 

324 def len(self) -> int: 

325 return 0 if self._raw_msg is None else len(self._raw_msg) 

326 

327 # `Message.__len__` reads the C-level value this mock never sets 

328 def __len__(self) -> int: 

329 return self.len() 

330 

331 def error(self) -> KafkaError | None: 

332 return self._error 

333 

334 def headers(self) -> list[tuple[str, str | bytes | None]]: 

335 return self._headers 

336 

337 def key(self) -> bytes: 

338 return self._key 

339 

340 def offset(self) -> int: 

341 return self._offset 

342 

343 def partition(self) -> int: 

344 return self._partition 

345 

346 def timestamp(self) -> tuple[int, int]: 

347 return self._timestamp 

348 

349 def topic(self) -> str: 

350 return self._topic 

351 

352 def value(self) -> bytes | None: 

353 return self._raw_msg 

354 

355 

356async def build_message( 

357 message: "SendableMessage", 

358 topic: str, 

359 *, 

360 correlation_id: str | None = None, 

361 partition: int | None = None, 

362 timestamp_ms: int | None = None, 

363 key: bytes | str | None = None, 

364 headers: Mapping[str, str | bytes | None] | None = None, 

365 reply_to: str = "", 

366 serializer: Optional["SerializerProto"] = None, 

367 codec: Optional["CodecProto"] = None, 

368 id_generator: IdGenerator = gen_cor_id, 

369) -> MockConfluentMessage: 

370 """Build a mock confluent_kafka.Message for a sendable message.""" 

371 if message is None: 

372 # keep a real tombstone (message.value() is None) distinct from b"" 

373 msg, content_type = None, None 

374 else: 

375 codec_instance = codec or DefaultCodec() 

376 msg, content_type = await codec_instance.encode(message, serializer) 

377 k = key or b"" 

378 headers = { 

379 "content-type": content_type or "", 

380 "correlation_id": correlation_id or id_generator(), 

381 "reply_to": reply_to, 

382 **(headers or {}), 

383 } 

384 

385 # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#confluent_kafka.Message.timestamp 

386 return MockConfluentMessage( 

387 raw_msg=msg, 

388 topic=topic, 

389 key=k, 

390 headers=encode_headers(headers), 

391 offset=0, 

392 partition=partition or 0, 

393 timestamp_type=1, 

394 timestamp_ms=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000), 

395 ) 

396 

397 

398def _build_mock_message( 

399 body: bytes, 

400 content_type: str | None, 

401 topic: str, 

402 partition: int | None = None, 

403 timestamp_ms: int | None = None, 

404 key: bytes | str | None = None, 

405 headers: Mapping[str, str | bytes | None] | None = None, 

406 correlation_id: str | None = None, 

407 reply_to: str = "", 

408 id_generator: IdGenerator = gen_cor_id, 

409) -> MockConfluentMessage: 

410 k = key or b"" 

411 h = { 

412 "content-type": content_type or "", 

413 "correlation_id": correlation_id or id_generator(), 

414 "reply_to": reply_to, 

415 **(headers or {}), 

416 } 

417 return MockConfluentMessage( 

418 raw_msg=body, 

419 topic=topic, 

420 key=k, 

421 headers=encode_headers(h), 

422 offset=0, 

423 partition=partition or 0, 

424 timestamp_type=1, 

425 timestamp_ms=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000), 

426 ) 

427 

428 

429def _fake_connection(*args: Any, **kwargs: Any) -> AsyncMock: 

430 mock = AsyncMock() 

431 mock.getone.return_value = MagicMock() 

432 mock.getmany.return_value = [MagicMock()] 

433 return mock 

434 

435 

436def _find_handler( 

437 subscribers: Iterable["LogicSubscriber[Any]"], 

438 topic: str, 

439 partition: int | None, 

440) -> Generator["LogicSubscriber[Any]", None, None]: 

441 published_groups = set() 

442 for handler in subscribers: # pragma: no branch 

443 if _is_handler_matches(handler, topic, partition): 

444 if handler.group_id: 

445 if handler.group_id in published_groups: 

446 continue 

447 else: 

448 published_groups.add(handler.group_id) 

449 yield handler 

450 

451 

452def _is_handler_matches( 

453 handler: "LogicSubscriber[Any]", 

454 topic: str, 

455 partition: int | None, 

456) -> bool: 

457 return bool( 

458 any( 

459 p.topic == topic and (partition is None or p.partition == partition) 

460 for p in handler.partitions 

461 ) 

462 or any(t.name == topic for t in handler.topics), 

463 )