Coverage for faststream/kafka/subscriber/usecase.py: 94%

189 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1import logging 

2from abc import abstractmethod 

3from collections.abc import AsyncIterator, Callable, Sequence 

4from itertools import chain 

5from typing import TYPE_CHECKING, Any, Optional, Union, cast 

6 

7import anyio 

8from aiokafka import ( 

9 ConsumerRecord, 

10 TopicPartition as AIOKafkaTopicPartition, 

11) 

12from aiokafka.errors import ConsumerStoppedError, KafkaError, UnsupportedCodecError 

13from typing_extensions import overload, override 

14 

15from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin, TasksMixin 

16from faststream._internal.endpoint.subscriber.usecase import SubscriberUsecase 

17from faststream._internal.endpoint.subscriber.utils import default_filter 

18from faststream._internal.endpoint.utils import process_msg 

19from faststream._internal.types import MsgType, P_HandlerParams, T_HandlerReturn 

20from faststream._internal.utils.path import Address, AddressSyntax 

21from faststream.kafka.call_wrapper import KafkaHandlerCallWrapper 

22from faststream.kafka.helpers import make_logging_listener 

23from faststream.kafka.message import KafkaAckableMessage, KafkaMessage, KafkaRawMessage 

24from faststream.kafka.parser import AioKafkaBatchParser, AioKafkaParser 

25from faststream.kafka.publisher.fake import KafkaFakePublisher 

26from faststream.kafka.schemas import Topic 

27 

28if TYPE_CHECKING: 

29 from aiokafka import AIOKafkaConsumer 

30 from fast_depends.dependencies import Dependant 

31 

32 from faststream._internal.endpoint.publisher import PublisherProto 

33 from faststream._internal.endpoint.subscriber import SubscriberSpecification 

34 from faststream._internal.endpoint.subscriber.call_item import CallsCollection 

35 from faststream._internal.types import CustomCallable, Filter 

36 from faststream.kafka.configs import KafkaBrokerConfig 

37 from faststream.message import StreamMessage 

38 

39 from .config import KafkaSubscriberConfig 

40 

41 

42KAFKA_ADDRESS_SYNTAX = AddressSyntax( 

43 replace_symbol=".*", 

44 patch_regex=lambda x: x.replace(r"\*", ".*"), 

45) 

46 

47 

48class LogicSubscriber(TasksMixin, SubscriberUsecase[MsgType]): 

49 """A class to handle logic for consuming messages from Kafka.""" 

50 

51 __slots__ = ( 

52 "_connection_args", 

53 "_listener", 

54 "_partitions", 

55 "_pattern", 

56 "_topics", 

57 "consumer", 

58 "group_id", 

59 ) 

60 

61 consumer: Optional["AIOKafkaConsumer"] 

62 

63 batch: bool 

64 parser: AioKafkaParser 

65 

66 _outer_config: "KafkaBrokerConfig" 

67 _call_wrapper_class = KafkaHandlerCallWrapper 

68 

69 def __init__( 

70 self, 

71 config: "KafkaSubscriberConfig", 

72 specification: "SubscriberSpecification[Any, Any]", 

73 calls: "CallsCollection[MsgType]", 

74 ) -> None: 

75 super().__init__(config, specification, calls) 

76 

77 self._topics = config.topics 

78 self._partitions = config.partitions 

79 self.group_id = config.group_id 

80 

81 self._pattern = config.pattern 

82 self._listener = config.listener 

83 self._connection_args = config.connection_args 

84 

85 self.consumer = None 

86 

87 @overload 

88 def __call__( 

89 self, 

90 func: Callable[P_HandlerParams, T_HandlerReturn], 

91 *, 

92 filter: "Filter[Any]" = default_filter, 

93 parser: Optional["CustomCallable"] = None, 

94 decoder: Optional["CustomCallable"] = None, 

95 dependencies: Sequence["Dependant"] = (), 

96 ) -> "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]": ... 

97 

98 @overload 

99 def __call__( 

100 self, 

101 func: None = None, 

102 *, 

103 filter: "Filter[Any]" = default_filter, 

104 parser: Optional["CustomCallable"] = None, 

105 decoder: Optional["CustomCallable"] = None, 

106 dependencies: Sequence["Dependant"] = (), 

107 ) -> Callable[ 

108 [Callable[P_HandlerParams, T_HandlerReturn]], 

109 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

110 ]: ... 

111 

112 @override 

113 def __call__( 

114 self, 

115 func: Callable[P_HandlerParams, T_HandlerReturn] | None = None, 

116 *, 

117 filter: "Filter[Any]" = default_filter, 

118 parser: Optional["CustomCallable"] = None, 

119 decoder: Optional["CustomCallable"] = None, 

120 dependencies: Sequence["Dependant"] = (), 

121 ) -> Union[ 

122 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

123 Callable[ 

124 [Callable[P_HandlerParams, T_HandlerReturn]], 

125 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

126 ], 

127 ]: 

128 # The base builds the wrapper from `_call_wrapper_class`; this only narrows the name 

129 return cast( 

130 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn] | Callable[" 

131 "[Callable[P_HandlerParams, T_HandlerReturn]], " 

132 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]]", 

133 super().__call__( 

134 func, 

135 filter=filter, 

136 parser=parser, 

137 decoder=decoder, 

138 dependencies=dependencies, 

139 ), 

140 ) 

141 

142 @property 

143 def pattern(self) -> Address | None: 

144 """The pattern this Subscriber was declared with, and its Broker address.""" 

145 if not self._pattern: 

146 return None 

147 

148 return Address(self._pattern, KAFKA_ADDRESS_SYNTAX).add_prefix( 

149 self._outer_config.prefix, 

150 ) 

151 

152 @property 

153 def topics(self) -> list[Topic]: 

154 return [t.add_prefix(self._outer_config.prefix) for t in self._topics] 

155 

156 @property 

157 def topic_names_for_subscribe(self) -> list[str]: 

158 return [t.name for t in self.topics] 

159 

160 @property 

161 def partitions(self) -> list[AIOKafkaTopicPartition]: 

162 """The assignment as the consumer receives it: the client library's tuples, prefixed. 

163 

164 Declared with `faststream.kafka.TopicPartition`; handed to aiokafka as its own. 

165 """ 

166 return [ 

167 AIOKafkaTopicPartition( 

168 topic=f"{self._outer_config.prefix}{p.topic}", 

169 partition=p.partition, 

170 ) 

171 for p in self._partitions 

172 ] 

173 

174 @property 

175 def builder(self) -> Callable[..., "AIOKafkaConsumer"]: 

176 return self._outer_config.builder 

177 

178 @property 

179 def client_id(self) -> str | None: 

180 return self._outer_config.client_id 

181 

182 @property 

183 def topics_to_create(self) -> list[Topic]: 

184 # Conflicting duplicates are reported by `create_subscriber`, the only 

185 # public way to get here, so collapsing to the last one is enough. 

186 topics: dict[str, Topic] = {t.name: t for t in self.topics} 

187 

188 prefix = self._outer_config.prefix 

189 for p in self._partitions: 

190 # Conflicting declare flags are reported by `create_subscriber`. 

191 partition = p.add_prefix(prefix) 

192 topics[partition.topic] = Topic( 

193 partition.topic, 

194 declare=partition.declare, 

195 ) 

196 

197 return [t for t in topics.values() if t.declare] 

198 

199 async def _ensure_topics(self) -> None: 

200 if not self._outer_config.allow_auto_create_topics: 

201 self._log( 

202 logging.WARNING, 

203 "Auto create topics is disabled. Make sure the topics exist.", 

204 ) 

205 return 

206 

207 if self._outer_config.consumer_only: 

208 self._log( 

209 logging.WARNING, 

210 "Topic creation is skipped in consumer-only mode. Make sure the topics exist.", 

211 ) 

212 return 

213 

214 topics = self.topics_to_create 

215 if not topics: 

216 return 

217 

218 results = await self._outer_config.admin.create_topics(topics) 

219 

220 for create_result in results: 

221 if create_result.error: 

222 self._log( 

223 logging.WARNING, 

224 f"Failed to create topic {create_result.topic}: {create_result.error}", 

225 ) 

226 

227 async def start(self) -> None: 

228 """Start the consumer.""" 

229 await super().start() 

230 await self._ensure_topics() 

231 

232 self.consumer = consumer = self.builder( 

233 group_id=self.group_id, 

234 client_id=self.client_id, 

235 **self._connection_args, 

236 ) 

237 

238 self.parser._setup(consumer) 

239 

240 pattern = self.pattern 

241 if self.topics or pattern: 

242 consumer.subscribe( 

243 topics=self.topic_names_for_subscribe, 

244 pattern=pattern.broker_address if pattern else None, 

245 listener=make_logging_listener( 

246 consumer=consumer, 

247 logger=self._outer_config.logger.logger.logger, 

248 log_extra=self.get_log_context(None), 

249 listener=self._listener, 

250 ), 

251 ) 

252 

253 elif self.partitions: 253 ↛ 256line 253 didn't jump to line 256 because the condition on line 253 was always true

254 consumer.assign(partitions=self.partitions) 

255 

256 await consumer.start() 

257 

258 self._post_start() 

259 

260 if self.calls: 

261 self.add_task(self._run_consume_loop, (self.consumer,)) 

262 

263 async def stop(self) -> None: 

264 await super().stop() 

265 

266 if self.consumer is not None: 

267 await self.consumer.stop() 

268 self.consumer = None 

269 

270 @override 

271 async def get_one( 

272 self, 

273 *, 

274 timeout: float = 5.0, 

275 ) -> "KafkaMessage | None": 

276 assert not self.calls, ( 

277 "You can't use `get_one` method if subscriber has registered handlers." 

278 ) 

279 

280 assert self.consumer, "You should start subscriber at first." 

281 

282 raw_messages = await self.consumer.getmany( 

283 timeout_ms=timeout * 1000, 

284 max_records=1, 

285 ) 

286 

287 if not raw_messages: 

288 return None 

289 

290 ((raw_message,),) = raw_messages.values() 

291 

292 context = self._outer_config.context 

293 

294 async_parser, async_decoder = self._get_parser_and_decoder() 

295 

296 msg: KafkaMessage | None = await process_msg( # type: ignore[assignment] 

297 msg=raw_message, 

298 middlewares=( 

299 m(raw_message, context=context) for m in self._broker_middlewares 

300 ), 

301 parser=async_parser, 

302 decoder=async_decoder, 

303 ) 

304 return msg 

305 

306 @override 

307 async def __aiter__(self) -> AsyncIterator["KafkaMessage"]: 

308 assert self.consumer, "You should start subscriber at first." 

309 assert not self.calls, ( 

310 "You can't use `get_one` method if subscriber has registered handlers." 

311 ) 

312 

313 context = self._outer_config.context 

314 async_parser, async_decoder = self._get_parser_and_decoder() 

315 

316 async for raw_message in self.consumer: 

317 msg: KafkaMessage = await process_msg( # type: ignore[assignment] 

318 msg=raw_message, 

319 middlewares=( 

320 m(raw_message, context=context) for m in self._broker_middlewares 

321 ), 

322 parser=async_parser, 

323 decoder=async_decoder, 

324 ) 

325 yield msg 

326 

327 def _make_response_publisher( 

328 self, 

329 message: "StreamMessage[Any]", 

330 ) -> Sequence["PublisherProto"]: 

331 return ( 

332 KafkaFakePublisher( 

333 self._outer_config.producer, 

334 topic=message.reply_to, 

335 ), 

336 ) 

337 

338 @abstractmethod 

339 async def get_msg(self, consumer: "AIOKafkaConsumer") -> MsgType: 

340 raise NotImplementedError 

341 

342 async def _run_consume_loop(self, consumer: "AIOKafkaConsumer") -> None: 

343 assert consumer, "You should start subscriber at first." 

344 

345 connected = True 

346 while self.running: 

347 try: 

348 msg = await self.get_msg(consumer) 

349 

350 except UnsupportedCodecError as e: # ruff: ignore[try-except-in-loop] 

351 self._log( 

352 logging.ERROR, 

353 "There is no suitable compression library available. Please refer to the Kafka " 

354 "documentation for more information - " 

355 "https://aiokafka.readthedocs.io/en/stable/#installation", 

356 exc_info=e, 

357 ) 

358 await anyio.sleep(15) 

359 

360 except KafkaError as e: 

361 self._log(logging.ERROR, "Kafka error occurred", exc_info=e) 

362 

363 if connected: 

364 connected = False 

365 

366 await anyio.sleep(5) 

367 

368 except ConsumerStoppedError: 

369 return 

370 

371 else: 

372 if not connected: # pragma: no cover 

373 connected = True 

374 

375 if msg: 375 ↛ 346line 375 didn't jump to line 346 because the condition on line 375 was always true

376 await self.consume_one(msg) 

377 

378 async def consume_one(self, msg: MsgType) -> None: 

379 await self.consume(msg) 

380 

381 @property 

382 def topic_names(self) -> list[str]: 

383 if pattern := self.pattern: 

384 topics = [pattern.broker_address] 

385 

386 elif self.topics: 

387 topics = self.topic_names_for_subscribe 

388 

389 else: 

390 topics = [f"{p.topic}-{p.partition}" for p in self.partitions] 

391 

392 return topics 

393 

394 @staticmethod 

395 def build_log_context( 

396 message: Optional["StreamMessage[Any]"], 

397 topic: str, 

398 group_id: str | None = None, 

399 ) -> dict[str, str]: 

400 return { 

401 "topic": topic, 

402 "group_id": group_id or "", 

403 "message_id": getattr(message, "message_id", ""), 

404 } 

405 

406 

407class DefaultSubscriber(LogicSubscriber["ConsumerRecord"]): 

408 __slots__ = ("parser",) 

409 

410 def __init__( 

411 self, 

412 config: "KafkaSubscriberConfig", 

413 specification: "SubscriberSpecification[Any, Any]", 

414 calls: "CallsCollection[ConsumerRecord]", 

415 ) -> None: 

416 reg = ( 

417 Address(config.pattern, KAFKA_ADDRESS_SYNTAX).regex 

418 if config.pattern 

419 else None 

420 ) 

421 

422 self.parser = AioKafkaParser( 

423 msg_class=KafkaMessage if config.ack_first else KafkaAckableMessage, 

424 regex=reg, 

425 ) 

426 config.parser = self.parser.parse_message 

427 config.decoder = self.parser.decode_message 

428 super().__init__(config, specification, calls) 

429 

430 @override 

431 async def get_msg(self, consumer: "AIOKafkaConsumer") -> "ConsumerRecord": 

432 assert consumer, "You should setup subscriber at first." 

433 return await consumer.getone() 

434 

435 def get_log_context( 

436 self, 

437 message: Optional["StreamMessage[ConsumerRecord]"], 

438 ) -> dict[str, str]: 

439 if message is None: 

440 topic = ",".join(self.topic_names) 

441 else: 

442 topic = message.raw_message.topic 

443 

444 return self.build_log_context( 

445 message=message, 

446 topic=topic, 

447 group_id=self.group_id, 

448 ) 

449 

450 

451class BatchSubscriber(LogicSubscriber[tuple["ConsumerRecord", ...]]): 

452 __slots__ = ( 

453 "batch_timeout_ms", 

454 "max_records", 

455 "parser", 

456 ) 

457 

458 def __init__( 

459 self, 

460 config: "KafkaSubscriberConfig", 

461 specification: "SubscriberSpecification[Any, Any]", 

462 calls: "CallsCollection[tuple[ConsumerRecord, ...]]", 

463 batch_timeout_ms: int, 

464 max_records: int | None, 

465 ) -> None: 

466 reg = ( 

467 Address(config.pattern, KAFKA_ADDRESS_SYNTAX).regex 

468 if config.pattern 

469 else None 

470 ) 

471 

472 self.parser = AioKafkaBatchParser( 

473 msg_class=KafkaMessage if config.ack_first else KafkaAckableMessage, 

474 regex=reg, 

475 ) 

476 config.decoder = self.parser.decode_batch 

477 config.parser = self.parser.parse_batch 

478 super().__init__(config, specification, calls) 

479 

480 self.batch_timeout_ms = batch_timeout_ms 

481 self.max_records = max_records 

482 

483 async def get_msg( 

484 self, 

485 consumer: "AIOKafkaConsumer", 

486 ) -> tuple["ConsumerRecord", ...]: 

487 assert consumer, "You should setup subscriber at first." 

488 

489 messages = await consumer.getmany( 

490 timeout_ms=self.batch_timeout_ms, 

491 max_records=self.max_records, 

492 ) 

493 

494 if not messages: # pragma: no cover 

495 await anyio.sleep(self.batch_timeout_ms / 1000) 

496 return () 

497 

498 return tuple(chain(*messages.values())) 

499 

500 def get_log_context( 

501 self, 

502 message: Optional["StreamMessage[tuple[ConsumerRecord, ...]]"], 

503 ) -> dict[str, str]: 

504 if message is None: 

505 topic = ",".join(self.topic_names) 

506 else: 

507 topic = message.raw_message[0].topic 

508 

509 return self.build_log_context( 

510 message=message, 

511 topic=topic, 

512 group_id=self.group_id, 

513 ) 

514 

515 

516class ConcurrentDefaultSubscriber(ConcurrentMixin["ConsumerRecord"], DefaultSubscriber): 

517 __slots__ = () 

518 

519 async def start(self) -> None: 

520 await super().start() 

521 self.start_consume_task() 

522 

523 async def consume_one(self, msg: "ConsumerRecord") -> None: 

524 await self._put_msg(msg) 

525 

526 

527class ConcurrentBetweenPartitionsSubscriber(DefaultSubscriber): 

528 __slots__ = ( 

529 "consumer_subgroup", 

530 "max_workers", 

531 ) 

532 

533 consumer_subgroup: list["AIOKafkaConsumer"] 

534 

535 def __init__( 

536 self, 

537 config: "KafkaSubscriberConfig", 

538 specification: "SubscriberSpecification[Any, Any]", 

539 calls: "CallsCollection[ConsumerRecord]", 

540 max_workers: int, 

541 ) -> None: 

542 super().__init__(config, specification, calls) 

543 

544 self.max_workers = max_workers 

545 self.consumer_subgroup = [] 

546 

547 async def start(self) -> None: 

548 """Start the consumer subgroup.""" 

549 await super(LogicSubscriber, self).start() 

550 await self._ensure_topics() 

551 

552 if self.calls: 552 ↛ 565line 552 didn't jump to line 565 because the condition on line 552 was always true

553 self.consumer_subgroup = [ 

554 self.builder( 

555 group_id=self.group_id, 

556 client_id=self.client_id, 

557 **self._connection_args, 

558 ) 

559 for _ in range(self.max_workers) 

560 ] 

561 

562 else: 

563 # We should create single consumer to support 

564 # `get_one()` and `__aiter__` methods 

565 self.consumer = self.builder( 

566 group_id=self.group_id, 

567 client_id=self.client_id, 

568 **self._connection_args, 

569 ) 

570 self.consumer_subgroup = [self.consumer] 

571 

572 # Subscribers starting should be called concurrently 

573 # to balance them correctly 

574 async with anyio.create_task_group() as tg: 

575 for c in self.consumer_subgroup: 

576 c.subscribe( 

577 topics=self.topic_names_for_subscribe, 

578 listener=make_logging_listener( 

579 consumer=c, 

580 logger=self._outer_config.logger.logger.logger, 

581 log_extra=self.get_log_context(None), 

582 listener=self._listener, 

583 ), 

584 ) 

585 

586 _ = tg.start_soon(c.start) 

587 

588 self._post_start() 

589 

590 if self.calls: 590 ↛ exitline 590 didn't return from function 'start' because the condition on line 590 was always true

591 for c in self.consumer_subgroup: 

592 self.add_task(self._run_consume_loop, (c,)) 

593 

594 async def stop(self) -> None: 

595 if self.consumer_subgroup: 595 ↛ 602line 595 didn't jump to line 602 because the condition on line 595 was always true

596 async with anyio.create_task_group() as tg: 

597 for consumer in self.consumer_subgroup: 

598 _ = tg.start_soon(consumer.stop) 

599 

600 self.consumer_subgroup = [] 

601 

602 await super().stop() 

603 

604 @override 

605 async def get_msg(self, consumer: "AIOKafkaConsumer") -> "KafkaRawMessage": 

606 assert consumer, "You should setup subscriber at first." 

607 message = await consumer.getone() 

608 message.consumer = consumer 

609 return cast("KafkaRawMessage", message)