Coverage for faststream/confluent/subscriber/usecase.py: 98%

103 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1import logging 

2from abc import abstractmethod 

3from collections.abc import AsyncIterator, Callable, Sequence 

4from typing import ( 

5 TYPE_CHECKING, 

6 Any, 

7 Optional, 

8 Union, 

9 cast, 

10) 

11 

12import anyio 

13from confluent_kafka import KafkaException, Message 

14from typing_extensions import overload, override 

15 

16from faststream._internal.endpoint.subscriber import SubscriberUsecase 

17from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin, TasksMixin 

18from faststream._internal.endpoint.subscriber.utils import default_filter 

19from faststream._internal.endpoint.utils import process_msg 

20from faststream._internal.types import MsgType, P_HandlerParams, T_HandlerReturn 

21from faststream.confluent.call_wrapper import KafkaHandlerCallWrapper 

22from faststream.confluent.parser import AsyncConfluentParser 

23from faststream.confluent.publisher.fake import KafkaFakePublisher 

24from faststream.confluent.schemas import Topic, TopicPartition 

25 

26if TYPE_CHECKING: 

27 from fast_depends.dependencies import Dependant 

28 

29 from faststream._internal.endpoint.publisher import PublisherProto 

30 from faststream._internal.endpoint.subscriber import SubscriberSpecification 

31 from faststream._internal.endpoint.subscriber.call_item import CallsCollection 

32 from faststream._internal.types import CustomCallable, Filter 

33 from faststream.confluent.configs import KafkaBrokerConfig 

34 from faststream.confluent.helpers.client import AsyncConfluentConsumer 

35 from faststream.confluent.message import KafkaMessage 

36 from faststream.message import StreamMessage 

37 

38 from .config import KafkaSubscriberConfig 

39 

40 

41class LogicSubscriber(TasksMixin, SubscriberUsecase[MsgType]): 

42 """A class to handle logic for consuming messages from Kafka.""" 

43 

44 __slots__ = ( 

45 "__connection_data", 

46 "_partitions", 

47 "_topics", 

48 "consumer", 

49 "group_id", 

50 "polling_interval", 

51 ) 

52 

53 _outer_config: "KafkaBrokerConfig" 

54 _call_wrapper_class = KafkaHandlerCallWrapper 

55 

56 group_id: str | None 

57 

58 consumer: Optional["AsyncConfluentConsumer"] 

59 parser: AsyncConfluentParser 

60 

61 def __init__( 

62 self, 

63 config: "KafkaSubscriberConfig", 

64 specification: "SubscriberSpecification[Any, Any]", 

65 calls: "CallsCollection[MsgType]", 

66 ) -> None: 

67 super().__init__(config, specification, calls) 

68 

69 self.__connection_data = config.connection_data 

70 

71 self.group_id = config.group_id 

72 

73 self._topics = config.topics 

74 self._partitions = config.partitions 

75 

76 self.consumer = None 

77 self.polling_interval = config.polling_interval 

78 

79 @overload 

80 def __call__( 

81 self, 

82 func: Callable[P_HandlerParams, T_HandlerReturn], 

83 *, 

84 filter: "Filter[Any]" = default_filter, 

85 parser: Optional["CustomCallable"] = None, 

86 decoder: Optional["CustomCallable"] = None, 

87 dependencies: Sequence["Dependant"] = (), 

88 ) -> "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]": ... 

89 

90 @overload 

91 def __call__( 

92 self, 

93 func: None = None, 

94 *, 

95 filter: "Filter[Any]" = default_filter, 

96 parser: Optional["CustomCallable"] = None, 

97 decoder: Optional["CustomCallable"] = None, 

98 dependencies: Sequence["Dependant"] = (), 

99 ) -> Callable[ 

100 [Callable[P_HandlerParams, T_HandlerReturn]], 

101 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

102 ]: ... 

103 

104 @override 

105 def __call__( 

106 self, 

107 func: Callable[P_HandlerParams, T_HandlerReturn] | None = None, 

108 *, 

109 filter: "Filter[Any]" = default_filter, 

110 parser: Optional["CustomCallable"] = None, 

111 decoder: Optional["CustomCallable"] = None, 

112 dependencies: Sequence["Dependant"] = (), 

113 ) -> Union[ 

114 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

115 Callable[ 

116 [Callable[P_HandlerParams, T_HandlerReturn]], 

117 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]", 

118 ], 

119 ]: 

120 # The base builds the wrapper from `_call_wrapper_class`; this only narrows the name 

121 return cast( 

122 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn] | Callable[" 

123 "[Callable[P_HandlerParams, T_HandlerReturn]], " 

124 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]]", 

125 super().__call__( 

126 func, 

127 filter=filter, 

128 parser=parser, 

129 decoder=decoder, 

130 dependencies=dependencies, 

131 ), 

132 ) 

133 

134 @property 

135 def client_id(self) -> str | None: 

136 return self._outer_config.client_id 

137 

138 @property 

139 def topics(self) -> list[Topic]: 

140 return [t.add_prefix(self._outer_config.prefix) for t in self._topics] 

141 

142 @property 

143 def partitions(self) -> list[TopicPartition]: 

144 return [p.add_prefix(self._outer_config.prefix) for p in self._partitions] 

145 

146 @override 

147 async def start(self) -> None: 

148 """Start the consumer.""" 

149 await super().start() 

150 self.consumer = consumer = self._outer_config.builder( 

151 *self.topics, 

152 partitions=self.partitions, 

153 group_id=self.group_id, 

154 client_id=self.client_id, 

155 **self.__connection_data, 

156 ) 

157 self.parser._setup(consumer) 

158 await consumer.start() 

159 

160 self._post_start() 

161 

162 if self.calls: 

163 self.add_task(self._consume) 

164 

165 async def stop(self) -> None: 

166 await super().stop() 

167 

168 if self.consumer is not None: 

169 await self.consumer.stop() 

170 self.consumer = None 

171 

172 @override 

173 async def get_one( 

174 self, 

175 *, 

176 timeout: float = 5.0, 

177 ) -> "KafkaMessage | None": 

178 assert self.consumer, "You should start subscriber at first." 

179 assert not self.calls, ( 

180 "You can't use `get_one` method if subscriber has registered handlers." 

181 ) 

182 

183 raw_message = await self.consumer.getone(timeout=timeout) 

184 

185 context = self._outer_config.context 

186 

187 async_parser, async_decoder = self._get_parser_and_decoder() 

188 

189 return await process_msg( # type: ignore[return-value] 

190 msg=raw_message, 

191 middlewares=( 

192 m(raw_message, context=context) for m in self._broker_middlewares 

193 ), 

194 parser=async_parser, 

195 decoder=async_decoder, 

196 ) 

197 

198 @override 

199 async def __aiter__(self) -> AsyncIterator["KafkaMessage"]: 

200 assert self.consumer, "You should start subscriber at first." 

201 assert not self.calls, ( 

202 "You can't use iterator if subscriber has registered handlers." 

203 ) 

204 

205 context = self._outer_config.context 

206 async_parser, async_decoder = self._get_parser_and_decoder() 

207 

208 timeout = 5.0 

209 while True: 

210 raw_message = await self.consumer.getone(timeout=timeout) 

211 

212 if raw_message is None: 212 ↛ 213line 212 didn't jump to line 213 because the condition on line 212 was never true

213 continue 

214 

215 yield cast( 

216 "KafkaMessage", 

217 await process_msg( 

218 msg=raw_message, 

219 middlewares=( 

220 m(raw_message, context=context) for m in self._broker_middlewares 

221 ), 

222 parser=async_parser, 

223 decoder=async_decoder, 

224 ), 

225 ) 

226 

227 def _make_response_publisher( 

228 self, 

229 message: "StreamMessage[Any]", 

230 ) -> Sequence["PublisherProto"]: 

231 return ( 

232 KafkaFakePublisher( 

233 self._outer_config.producer, 

234 topic=message.reply_to, 

235 ), 

236 ) 

237 

238 async def consume_one(self, msg: MsgType) -> None: 

239 await self.consume(msg) 

240 

241 @abstractmethod 

242 async def get_msg(self) -> MsgType | None: 

243 raise NotImplementedError 

244 

245 async def _consume(self) -> None: 

246 assert self.consumer, "You should start subscriber at first." 

247 

248 connected = True 

249 while self.running: 

250 try: 

251 msg = await self.get_msg() 

252 

253 except KafkaException as e: # pragma: no cover # ruff: ignore[try-except-in-loop] 

254 self._log( 

255 logging.ERROR, 

256 message="Message fetch error", 

257 exc_info=e, 

258 ) 

259 

260 if connected: 

261 connected = False 

262 

263 await anyio.sleep(5) 

264 

265 else: 

266 if not connected: # pragma: no cover 

267 connected = True 

268 

269 if msg is not None: 

270 await self.consume_one(msg) 

271 

272 @property 

273 def topic_names(self) -> list[str]: 

274 if topics := self.topics: 

275 return [t.name for t in topics] 

276 

277 return [f"{p.topic}-{p.partition}" for p in self.partitions] 

278 

279 @staticmethod 

280 def build_log_context( 

281 message: Optional["StreamMessage[Any]"], 

282 topic: str, 

283 group_id: str | None = None, 

284 ) -> dict[str, str]: 

285 return { 

286 "topic": topic, 

287 "group_id": group_id or "", 

288 "message_id": getattr(message, "message_id", ""), 

289 } 

290 

291 

292class DefaultSubscriber(LogicSubscriber[Message]): 

293 __slots__ = ("parser",) 

294 

295 def __init__( 

296 self, 

297 config: "KafkaSubscriberConfig", 

298 specification: "SubscriberSpecification[Any, Any]", 

299 calls: "CallsCollection[Message]", 

300 ) -> None: 

301 self.parser = AsyncConfluentParser(is_manual=not config.ack_first) 

302 config.decoder = self.parser.decode_message 

303 config.parser = self.parser.parse_message 

304 super().__init__(config, specification, calls) 

305 

306 async def get_msg(self) -> Optional["Message"]: 

307 assert self.consumer, "You should setup subscriber at first." 

308 return await self.consumer.getone(timeout=self.polling_interval) 

309 

310 def get_log_context( 

311 self, 

312 message: Optional["StreamMessage[Message]"], 

313 ) -> dict[str, str]: 

314 if message is None: 

315 topic = ",".join(self.topic_names) 

316 else: 

317 topic = message.raw_message.topic() or ",".join(self.topic_names) 

318 

319 return self.build_log_context( 

320 message=message, 

321 topic=topic, 

322 group_id=self.group_id, 

323 ) 

324 

325 

326class ConcurrentDefaultSubscriber(ConcurrentMixin["Message"], DefaultSubscriber): 

327 __slots__ = () 

328 

329 async def start(self) -> None: 

330 await super().start() 

331 self.start_consume_task() 

332 

333 async def consume_one(self, msg: "Message") -> None: 

334 await self._put_msg(msg) 

335 

336 

337class BatchSubscriber(LogicSubscriber[tuple[Message, ...]]): 

338 __slots__ = ( 

339 "max_records", 

340 "parser", 

341 ) 

342 

343 def __init__( 

344 self, 

345 config: "KafkaSubscriberConfig", 

346 specification: "SubscriberSpecification[Any, Any]", 

347 calls: "CallsCollection[tuple[Message, ...]]", 

348 max_records: int | None, 

349 ) -> None: 

350 self.parser = AsyncConfluentParser(is_manual=not config.ack_first) 

351 config.decoder = self.parser.decode_batch 

352 config.parser = self.parser.parse_batch 

353 super().__init__(config, specification, calls) 

354 

355 self.max_records = max_records 

356 

357 async def get_msg(self) -> tuple["Message", ...] | None: 

358 assert self.consumer, "You should setup subscriber at first." 

359 return ( 

360 await self.consumer.getmany( 

361 timeout=self.polling_interval, 

362 max_records=self.max_records, 

363 ) 

364 or None 

365 ) 

366 

367 def get_log_context( 

368 self, 

369 message: Optional["StreamMessage[tuple[Message, ...]]"], 

370 ) -> dict[str, str]: 

371 if message is None: 

372 topic = ",".join(self.topic_names) 

373 else: 

374 topic = message.raw_message[0].topic() or ",".join(self.topic_names) 

375 

376 return self.build_log_context( 

377 message=message, 

378 topic=topic, 

379 group_id=self.group_id, 

380 )