Coverage for faststream/confluent/subscriber/usecase.py: 98%
103 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import logging
2from abc import abstractmethod
3from collections.abc import AsyncIterator, Callable, Sequence
4from typing import (
5 TYPE_CHECKING,
6 Any,
7 Optional,
8 Union,
9 cast,
10)
12import anyio
13from confluent_kafka import KafkaException, Message
14from typing_extensions import overload, override
16from faststream._internal.endpoint.subscriber import SubscriberUsecase
17from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin, TasksMixin
18from faststream._internal.endpoint.subscriber.utils import default_filter
19from faststream._internal.endpoint.utils import process_msg
20from faststream._internal.types import MsgType, P_HandlerParams, T_HandlerReturn
21from faststream.confluent.call_wrapper import KafkaHandlerCallWrapper
22from faststream.confluent.parser import AsyncConfluentParser
23from faststream.confluent.publisher.fake import KafkaFakePublisher
24from faststream.confluent.schemas import Topic, TopicPartition
26if TYPE_CHECKING:
27 from fast_depends.dependencies import Dependant
29 from faststream._internal.endpoint.publisher import PublisherProto
30 from faststream._internal.endpoint.subscriber import SubscriberSpecification
31 from faststream._internal.endpoint.subscriber.call_item import CallsCollection
32 from faststream._internal.types import CustomCallable, Filter
33 from faststream.confluent.configs import KafkaBrokerConfig
34 from faststream.confluent.helpers.client import AsyncConfluentConsumer
35 from faststream.confluent.message import KafkaMessage
36 from faststream.message import StreamMessage
38 from .config import KafkaSubscriberConfig
41class LogicSubscriber(TasksMixin, SubscriberUsecase[MsgType]):
42 """A class to handle logic for consuming messages from Kafka."""
44 __slots__ = (
45 "__connection_data",
46 "_partitions",
47 "_topics",
48 "consumer",
49 "group_id",
50 "polling_interval",
51 )
53 _outer_config: "KafkaBrokerConfig"
54 _call_wrapper_class = KafkaHandlerCallWrapper
56 group_id: str | None
58 consumer: Optional["AsyncConfluentConsumer"]
59 parser: AsyncConfluentParser
61 def __init__(
62 self,
63 config: "KafkaSubscriberConfig",
64 specification: "SubscriberSpecification[Any, Any]",
65 calls: "CallsCollection[MsgType]",
66 ) -> None:
67 super().__init__(config, specification, calls)
69 self.__connection_data = config.connection_data
71 self.group_id = config.group_id
73 self._topics = config.topics
74 self._partitions = config.partitions
76 self.consumer = None
77 self.polling_interval = config.polling_interval
79 @overload
80 def __call__(
81 self,
82 func: Callable[P_HandlerParams, T_HandlerReturn],
83 *,
84 filter: "Filter[Any]" = default_filter,
85 parser: Optional["CustomCallable"] = None,
86 decoder: Optional["CustomCallable"] = None,
87 dependencies: Sequence["Dependant"] = (),
88 ) -> "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]": ...
90 @overload
91 def __call__(
92 self,
93 func: None = None,
94 *,
95 filter: "Filter[Any]" = default_filter,
96 parser: Optional["CustomCallable"] = None,
97 decoder: Optional["CustomCallable"] = None,
98 dependencies: Sequence["Dependant"] = (),
99 ) -> Callable[
100 [Callable[P_HandlerParams, T_HandlerReturn]],
101 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
102 ]: ...
104 @override
105 def __call__(
106 self,
107 func: Callable[P_HandlerParams, T_HandlerReturn] | None = None,
108 *,
109 filter: "Filter[Any]" = default_filter,
110 parser: Optional["CustomCallable"] = None,
111 decoder: Optional["CustomCallable"] = None,
112 dependencies: Sequence["Dependant"] = (),
113 ) -> Union[
114 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
115 Callable[
116 [Callable[P_HandlerParams, T_HandlerReturn]],
117 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
118 ],
119 ]:
120 # The base builds the wrapper from `_call_wrapper_class`; this only narrows the name
121 return cast(
122 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn] | Callable["
123 "[Callable[P_HandlerParams, T_HandlerReturn]], "
124 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]]",
125 super().__call__(
126 func,
127 filter=filter,
128 parser=parser,
129 decoder=decoder,
130 dependencies=dependencies,
131 ),
132 )
134 @property
135 def client_id(self) -> str | None:
136 return self._outer_config.client_id
138 @property
139 def topics(self) -> list[Topic]:
140 return [t.add_prefix(self._outer_config.prefix) for t in self._topics]
142 @property
143 def partitions(self) -> list[TopicPartition]:
144 return [p.add_prefix(self._outer_config.prefix) for p in self._partitions]
146 @override
147 async def start(self) -> None:
148 """Start the consumer."""
149 await super().start()
150 self.consumer = consumer = self._outer_config.builder(
151 *self.topics,
152 partitions=self.partitions,
153 group_id=self.group_id,
154 client_id=self.client_id,
155 **self.__connection_data,
156 )
157 self.parser._setup(consumer)
158 await consumer.start()
160 self._post_start()
162 if self.calls:
163 self.add_task(self._consume)
165 async def stop(self) -> None:
166 await super().stop()
168 if self.consumer is not None:
169 await self.consumer.stop()
170 self.consumer = None
172 @override
173 async def get_one(
174 self,
175 *,
176 timeout: float = 5.0,
177 ) -> "KafkaMessage | None":
178 assert self.consumer, "You should start subscriber at first."
179 assert not self.calls, (
180 "You can't use `get_one` method if subscriber has registered handlers."
181 )
183 raw_message = await self.consumer.getone(timeout=timeout)
185 context = self._outer_config.context
187 async_parser, async_decoder = self._get_parser_and_decoder()
189 return await process_msg( # type: ignore[return-value]
190 msg=raw_message,
191 middlewares=(
192 m(raw_message, context=context) for m in self._broker_middlewares
193 ),
194 parser=async_parser,
195 decoder=async_decoder,
196 )
198 @override
199 async def __aiter__(self) -> AsyncIterator["KafkaMessage"]:
200 assert self.consumer, "You should start subscriber at first."
201 assert not self.calls, (
202 "You can't use iterator if subscriber has registered handlers."
203 )
205 context = self._outer_config.context
206 async_parser, async_decoder = self._get_parser_and_decoder()
208 timeout = 5.0
209 while True:
210 raw_message = await self.consumer.getone(timeout=timeout)
212 if raw_message is None: 212 ↛ 213line 212 didn't jump to line 213 because the condition on line 212 was never true
213 continue
215 yield cast(
216 "KafkaMessage",
217 await process_msg(
218 msg=raw_message,
219 middlewares=(
220 m(raw_message, context=context) for m in self._broker_middlewares
221 ),
222 parser=async_parser,
223 decoder=async_decoder,
224 ),
225 )
227 def _make_response_publisher(
228 self,
229 message: "StreamMessage[Any]",
230 ) -> Sequence["PublisherProto"]:
231 return (
232 KafkaFakePublisher(
233 self._outer_config.producer,
234 topic=message.reply_to,
235 ),
236 )
238 async def consume_one(self, msg: MsgType) -> None:
239 await self.consume(msg)
241 @abstractmethod
242 async def get_msg(self) -> MsgType | None:
243 raise NotImplementedError
245 async def _consume(self) -> None:
246 assert self.consumer, "You should start subscriber at first."
248 connected = True
249 while self.running:
250 try:
251 msg = await self.get_msg()
253 except KafkaException as e: # pragma: no cover # ruff: ignore[try-except-in-loop]
254 self._log(
255 logging.ERROR,
256 message="Message fetch error",
257 exc_info=e,
258 )
260 if connected:
261 connected = False
263 await anyio.sleep(5)
265 else:
266 if not connected: # pragma: no cover
267 connected = True
269 if msg is not None:
270 await self.consume_one(msg)
272 @property
273 def topic_names(self) -> list[str]:
274 if topics := self.topics:
275 return [t.name for t in topics]
277 return [f"{p.topic}-{p.partition}" for p in self.partitions]
279 @staticmethod
280 def build_log_context(
281 message: Optional["StreamMessage[Any]"],
282 topic: str,
283 group_id: str | None = None,
284 ) -> dict[str, str]:
285 return {
286 "topic": topic,
287 "group_id": group_id or "",
288 "message_id": getattr(message, "message_id", ""),
289 }
292class DefaultSubscriber(LogicSubscriber[Message]):
293 __slots__ = ("parser",)
295 def __init__(
296 self,
297 config: "KafkaSubscriberConfig",
298 specification: "SubscriberSpecification[Any, Any]",
299 calls: "CallsCollection[Message]",
300 ) -> None:
301 self.parser = AsyncConfluentParser(is_manual=not config.ack_first)
302 config.decoder = self.parser.decode_message
303 config.parser = self.parser.parse_message
304 super().__init__(config, specification, calls)
306 async def get_msg(self) -> Optional["Message"]:
307 assert self.consumer, "You should setup subscriber at first."
308 return await self.consumer.getone(timeout=self.polling_interval)
310 def get_log_context(
311 self,
312 message: Optional["StreamMessage[Message]"],
313 ) -> dict[str, str]:
314 if message is None:
315 topic = ",".join(self.topic_names)
316 else:
317 topic = message.raw_message.topic() or ",".join(self.topic_names)
319 return self.build_log_context(
320 message=message,
321 topic=topic,
322 group_id=self.group_id,
323 )
326class ConcurrentDefaultSubscriber(ConcurrentMixin["Message"], DefaultSubscriber):
327 __slots__ = ()
329 async def start(self) -> None:
330 await super().start()
331 self.start_consume_task()
333 async def consume_one(self, msg: "Message") -> None:
334 await self._put_msg(msg)
337class BatchSubscriber(LogicSubscriber[tuple[Message, ...]]):
338 __slots__ = (
339 "max_records",
340 "parser",
341 )
343 def __init__(
344 self,
345 config: "KafkaSubscriberConfig",
346 specification: "SubscriberSpecification[Any, Any]",
347 calls: "CallsCollection[tuple[Message, ...]]",
348 max_records: int | None,
349 ) -> None:
350 self.parser = AsyncConfluentParser(is_manual=not config.ack_first)
351 config.decoder = self.parser.decode_batch
352 config.parser = self.parser.parse_batch
353 super().__init__(config, specification, calls)
355 self.max_records = max_records
357 async def get_msg(self) -> tuple["Message", ...] | None:
358 assert self.consumer, "You should setup subscriber at first."
359 return (
360 await self.consumer.getmany(
361 timeout=self.polling_interval,
362 max_records=self.max_records,
363 )
364 or None
365 )
367 def get_log_context(
368 self,
369 message: Optional["StreamMessage[tuple[Message, ...]]"],
370 ) -> dict[str, str]:
371 if message is None:
372 topic = ",".join(self.topic_names)
373 else:
374 topic = message.raw_message[0].topic() or ",".join(self.topic_names)
376 return self.build_log_context(
377 message=message,
378 topic=topic,
379 group_id=self.group_id,
380 )