Coverage for faststream/kafka/testing.py: 89%

111 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1import re 

2from collections.abc import Callable, Generator, Iterable, Mapping, Sequence 

3from contextlib import ExitStack, contextmanager 

4from datetime import datetime, timezone 

5from typing import TYPE_CHECKING, Any, Optional, cast, overload 

6from unittest.mock import AsyncMock, MagicMock 

7 

8import anyio 

9from aiokafka import ConsumerRecord 

10from typing_extensions import override 

11 

12from faststream._internal.endpoint.utils import ParserComposition 

13from faststream._internal.kafka import encode_headers 

14from faststream._internal.parser import BatchCodecProto, DefaultCodec 

15from faststream._internal.testing.broker import ( 

16 EnterType, 

17 TestBroker, 

18 change_producer, 

19) 

20from faststream._internal.types import IdGenerator 

21from faststream.exceptions import SubscriberNotFound 

22from faststream.kafka import TopicPartition 

23from faststream.kafka.broker import KafkaBroker 

24from faststream.kafka.message import KafkaMessage 

25from faststream.kafka.parser import AioKafkaParser 

26from faststream.kafka.publisher.producer import AioKafkaFastProducer 

27from faststream.kafka.publisher.usecase import BatchPublisher 

28from faststream.kafka.subscriber.usecase import BatchSubscriber 

29from faststream.message import gen_cor_id 

30 

31if TYPE_CHECKING: 

32 from fast_depends.library.serializer import SerializerProto 

33 

34 from faststream._internal.basic_types import SendableMessage 

35 from faststream._internal.parser import CodecProto 

36 from faststream.kafka.publisher.usecase import LogicPublisher 

37 from faststream.kafka.response import KafkaPublishCommand 

38 from faststream.kafka.subscriber.usecase import LogicSubscriber 

39 

40__all__ = ("TestKafkaBroker",) 

41 

42 

43class TestKafkaBroker( 

44 TestBroker[KafkaBroker, EnterType], 

45 broker=KafkaBroker, 

46): 

47 """A class to test Kafka brokers.""" 

48 

49 @overload 

50 def __init__( 

51 self: "TestKafkaBroker[KafkaBroker]", 

52 broker: KafkaBroker, 

53 /, 

54 *, 

55 with_real: bool = False, 

56 connect_only: bool | None = None, 

57 ) -> None: ... 

58 

59 @overload 

60 def __init__( 

61 self: "TestKafkaBroker[tuple[KafkaBroker, ...]]", 

62 *brokers: KafkaBroker, 

63 with_real: bool = False, 

64 connect_only: bool | None = None, 

65 ) -> None: ... 

66 

67 def __init__( 

68 self, 

69 *brokers: KafkaBroker, 

70 with_real: bool = False, 

71 connect_only: bool | None = None, 

72 ) -> None: 

73 super().__init__( 

74 *brokers, 

75 with_real=with_real, 

76 connect_only=connect_only, 

77 ) 

78 

79 @contextmanager 

80 def _patch_producer(self, broker: KafkaBroker) -> Generator[None, None, None]: 

81 fake_producer = FakeProducer(broker, self.brokers) 

82 

83 with ExitStack() as es: 

84 es.enter_context( 

85 change_producer(broker.config.broker_config, fake_producer), 

86 ) 

87 yield 

88 

89 @staticmethod 

90 async def _fake_connect( # type: ignore[override] 

91 broker: KafkaBroker, 

92 *args: Any, 

93 **kwargs: Any, 

94 ) -> Callable[..., AsyncMock]: 

95 builder = MagicMock(return_value=FakeConsumer()) 

96 broker.config.broker_config.builder = builder 

97 

98 return _fake_connection 

99 

100 def create_publisher_fake_subscriber( 

101 self, 

102 broker: KafkaBroker, 

103 publisher: "LogicPublisher", 

104 ) -> tuple["LogicSubscriber[Any]", bool]: 

105 sub: LogicSubscriber[Any] | None = None 

106 for handler in (s for b in self.brokers for s in b.subscribers): 

107 handler = cast("LogicSubscriber[Any]", handler) 

108 if _is_handler_matches(handler, publisher.topic, publisher.partition): 

109 sub = handler 

110 break 

111 

112 if sub is None: 

113 is_real = False 

114 

115 topic_name = publisher.topic 

116 

117 if publisher.partition: 117 ↛ 118line 117 didn't jump to line 118 because the condition on line 117 was never true

118 tp = TopicPartition( 

119 topic=topic_name, 

120 partition=publisher.partition, 

121 ) 

122 sub = broker.subscriber( 

123 partitions=[tp], 

124 batch=isinstance(publisher, BatchPublisher), 

125 persistent=False, 

126 ) 

127 else: 

128 sub = broker.subscriber( 

129 topic_name, 

130 batch=isinstance(publisher, BatchPublisher), 

131 persistent=False, 

132 ) 

133 else: 

134 is_real = True 

135 

136 return sub, is_real 

137 

138 

139class FakeConsumer: 

140 async def start(self) -> None: 

141 pass 

142 

143 async def stop(self) -> None: 

144 pass 

145 

146 def subscribe(self, *args: Any, **kwargs: Any) -> None: 

147 pass 

148 

149 

150class FakeProducer(AioKafkaFastProducer): 

151 """A fake Kafka producer for testing purposes. 

152 

153 This class extends AioKafkaFastProducer and is used to simulate Kafka message publishing during tests. 

154 """ 

155 

156 def __init__( 

157 self, 

158 broker: KafkaBroker, 

159 brokers: Sequence[KafkaBroker], 

160 ) -> None: 

161 self.broker = broker 

162 self.brokers = brokers 

163 

164 default = AioKafkaParser( 

165 msg_class=KafkaMessage, 

166 regex=None, 

167 ) 

168 

169 self._parser = ParserComposition(broker._parser, default.parse_message) 

170 self._decoder = ParserComposition(broker._decoder, default.decode_message) 

171 self.codec = broker.config.broker_codec or DefaultCodec() 

172 

173 @property 

174 def subscribers(self) -> Iterable["LogicSubscriber[Any]"]: 

175 return ( 

176 cast("LogicSubscriber[Any]", s) for b in self.brokers for s in b.subscribers 

177 ) 

178 

179 def __bool__(self) -> bool: 

180 return True 

181 

182 @property 

183 def closed(self) -> bool: 

184 return False 

185 

186 @override 

187 async def publish(self, cmd: "KafkaPublishCommand") -> None: 

188 """Publish a message to the Kafka broker.""" 

189 incoming = await build_message( 

190 message=cmd.body, 

191 topic=cmd.destination, 

192 key=cmd.key, 

193 partition=cmd.partition, 

194 timestamp_ms=cmd.timestamp_ms, 

195 headers=cmd.headers, 

196 correlation_id=cmd.correlation_id, 

197 reply_to=cmd.reply_to, 

198 serializer=self.broker.config.fd_config._serializer, 

199 codec=self.codec, 

200 id_generator=self.broker.config.id_generator, 

201 ) 

202 

203 for handler in _find_handler( 

204 self.subscribers, 

205 cmd.destination, 

206 cmd.partition, 

207 ): 

208 msg_to_send = ( 

209 (incoming,) if isinstance(handler, BatchSubscriber) else incoming 

210 ) 

211 

212 await self._execute_handler(msg_to_send, cmd.destination, handler) 

213 

214 @override 

215 async def request(self, cmd: "KafkaPublishCommand") -> "ConsumerRecord": 

216 incoming = await build_message( 

217 message=cmd.body, 

218 topic=cmd.destination, 

219 key=cmd.key, 

220 partition=cmd.partition, 

221 timestamp_ms=cmd.timestamp_ms, 

222 headers=cmd.headers, 

223 correlation_id=cmd.correlation_id, 

224 serializer=self.broker.config.fd_config._serializer, 

225 codec=self.codec, 

226 id_generator=self.broker.config.id_generator, 

227 ) 

228 

229 for handler in _find_handler( 

230 self.subscribers, 

231 cmd.destination, 

232 cmd.partition, 

233 ): 

234 msg_to_send = ( 

235 (incoming,) if isinstance(handler, BatchSubscriber) else incoming 

236 ) 

237 

238 with anyio.fail_after(cmd.timeout): 

239 return await self._execute_handler( 

240 msg_to_send, 

241 cmd.destination, 

242 handler, 

243 ) 

244 

245 raise SubscriberNotFound 

246 

247 @override 

248 async def publish_batch( 

249 self, 

250 cmd: "KafkaPublishCommand", 

251 ) -> None: 

252 """Publish a batch of messages to the Kafka broker.""" 

253 serializer = self.broker.config.fd_config._serializer 

254 

255 if isinstance(self.codec, BatchCodecProto): 

256 encoded = await self.codec.encode_batch(cmd.batch_bodies, serializer) 

257 else: 

258 encoded = [ 

259 await self.codec.encode(body, serializer) for body in cmd.batch_bodies 

260 ] 

261 

262 for handler in _find_handler( 

263 self.subscribers, 

264 cmd.destination, 

265 cmd.partition, 

266 ): 

267 messages = [ 

268 _build_record( 

269 body=body, 

270 content_type=content_type, 

271 topic=cmd.destination, 

272 partition=cmd.partition, 

273 timestamp_ms=cmd.timestamp_ms, 

274 key=cmd.key_for(message_position), 

275 headers=cmd.headers, 

276 correlation_id=cmd.correlation_id, 

277 reply_to=cmd.reply_to, 

278 id_generator=self.broker.config.id_generator, 

279 ) 

280 for message_position, (body, content_type) in enumerate(encoded) 

281 ] 

282 

283 if isinstance(handler, BatchSubscriber): 283 ↛ 287line 283 didn't jump to line 287 because the condition on line 283 was always true

284 await self._execute_handler(tuple(messages), cmd.destination, handler) 

285 

286 else: 

287 for m in messages: 

288 await self._execute_handler(m, cmd.destination, handler) 

289 

290 async def _execute_handler( 

291 self, 

292 msg: Any, 

293 topic: str, 

294 handler: "LogicSubscriber[Any]", 

295 ) -> "ConsumerRecord": 

296 result = await handler.process_message(msg) 

297 

298 return await build_message( 

299 topic=topic, 

300 message=result.body, 

301 headers=result.headers, 

302 correlation_id=result.correlation_id, 

303 serializer=self.broker.config.fd_config._serializer, 

304 codec=self.codec, 

305 id_generator=self.broker.config.id_generator, 

306 ) 

307 

308 

309async def build_message( 

310 message: "SendableMessage", 

311 topic: str, 

312 partition: int | None = None, 

313 timestamp_ms: int | None = None, 

314 key: bytes | None = None, 

315 headers: Mapping[str, str | bytes | None] | None = None, 

316 correlation_id: str | None = None, 

317 *, 

318 reply_to: str = "", 

319 serializer: Optional["SerializerProto"], 

320 codec: Optional["CodecProto"] = None, 

321 id_generator: IdGenerator = gen_cor_id, 

322) -> "ConsumerRecord": 

323 """Build a Kafka ConsumerRecord for a sendable message.""" 

324 if message is None and key is not None: 

325 # keyed None is a real tombstone, matching publish()'s own rule 

326 # (aiokafka needs a key or value, a keyless None still goes b"") 

327 msg, content_type = None, None 

328 else: 

329 msg, content_type = await (codec or DefaultCodec()).encode(message, serializer) 

330 

331 k = key or b"" 

332 

333 headers = { 

334 "content-type": content_type or "", 

335 "correlation_id": correlation_id or id_generator(), 

336 **(headers or {}), 

337 } 

338 

339 if reply_to: 

340 headers["reply_to"] = headers.get("reply_to", reply_to) 

341 

342 return ConsumerRecord( 

343 value=msg, 

344 topic=topic, 

345 partition=partition or 0, 

346 key=k, 

347 serialized_key_size=len(k), 

348 serialized_value_size=0 if msg is None else len(msg), 

349 checksum=0 if msg is None else sum(msg), 

350 offset=0, 

351 headers=encode_headers(headers), 

352 timestamp_type=1, 

353 timestamp=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000), 

354 ) 

355 

356 

357def _build_record( 

358 body: bytes, 

359 content_type: str | None, 

360 topic: str, 

361 partition: int | None = None, 

362 timestamp_ms: int | None = None, 

363 key: bytes | None = None, 

364 headers: Mapping[str, str | bytes | None] | None = None, 

365 correlation_id: str | None = None, 

366 reply_to: str = "", 

367 id_generator: IdGenerator = gen_cor_id, 

368) -> "ConsumerRecord": 

369 k = key or b"" 

370 h = { 

371 "content-type": content_type or "", 

372 "correlation_id": correlation_id or id_generator(), 

373 **(headers or {}), 

374 } 

375 if reply_to: 375 ↛ 376line 375 didn't jump to line 376 because the condition on line 375 was never true

376 h["reply_to"] = h.get("reply_to", reply_to) 

377 return ConsumerRecord( 

378 value=body, 

379 topic=topic, 

380 partition=partition or 0, 

381 key=k, 

382 serialized_key_size=len(k), 

383 serialized_value_size=len(body), 

384 checksum=sum(body), 

385 offset=0, 

386 headers=encode_headers(h), 

387 timestamp_type=1, 

388 timestamp=timestamp_ms or int(datetime.now(timezone.utc).timestamp() * 1000), 

389 ) 

390 

391 

392def _fake_connection(*args: Any, **kwargs: Any) -> AsyncMock: 

393 mock = AsyncMock() 

394 mock.subscribe = MagicMock 

395 mock.assign = MagicMock 

396 return mock 

397 

398 

399def _find_handler( 

400 subscribers: Iterable["LogicSubscriber[Any]"], 

401 topic: str, 

402 partition: int | None, 

403) -> Generator["LogicSubscriber[Any]", None, None]: 

404 published_groups = set() 

405 for handler in subscribers: # pragma: no branch 

406 if _is_handler_matches(handler, topic, partition): 

407 if handler.group_id: 

408 if handler.group_id in published_groups: 

409 continue 

410 else: 

411 published_groups.add(handler.group_id) 

412 yield handler 

413 

414 

415def _is_handler_matches( 

416 handler: "LogicSubscriber[Any]", 

417 topic: str, 

418 partition: int | None, 

419) -> bool: 

420 if any( 

421 p.topic == topic and (partition is None or p.partition == partition) 

422 for p in handler.partitions 

423 ): 

424 return True 

425 

426 if any(t.name == topic for t in handler.topics): 

427 return True 

428 

429 pattern = handler.pattern 

430 return bool(pattern and re.match(pattern.broker_address, topic))