Coverage for faststream/kafka/subscriber/usecase.py: 94%
189 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import logging
2from abc import abstractmethod
3from collections.abc import AsyncIterator, Callable, Sequence
4from itertools import chain
5from typing import TYPE_CHECKING, Any, Optional, Union, cast
7import anyio
8from aiokafka import (
9 ConsumerRecord,
10 TopicPartition as AIOKafkaTopicPartition,
11)
12from aiokafka.errors import ConsumerStoppedError, KafkaError, UnsupportedCodecError
13from typing_extensions import overload, override
15from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin, TasksMixin
16from faststream._internal.endpoint.subscriber.usecase import SubscriberUsecase
17from faststream._internal.endpoint.subscriber.utils import default_filter
18from faststream._internal.endpoint.utils import process_msg
19from faststream._internal.types import MsgType, P_HandlerParams, T_HandlerReturn
20from faststream._internal.utils.path import Address, AddressSyntax
21from faststream.kafka.call_wrapper import KafkaHandlerCallWrapper
22from faststream.kafka.helpers import make_logging_listener
23from faststream.kafka.message import KafkaAckableMessage, KafkaMessage, KafkaRawMessage
24from faststream.kafka.parser import AioKafkaBatchParser, AioKafkaParser
25from faststream.kafka.publisher.fake import KafkaFakePublisher
26from faststream.kafka.schemas import Topic
28if TYPE_CHECKING:
29 from aiokafka import AIOKafkaConsumer
30 from fast_depends.dependencies import Dependant
32 from faststream._internal.endpoint.publisher import PublisherProto
33 from faststream._internal.endpoint.subscriber import SubscriberSpecification
34 from faststream._internal.endpoint.subscriber.call_item import CallsCollection
35 from faststream._internal.types import CustomCallable, Filter
36 from faststream.kafka.configs import KafkaBrokerConfig
37 from faststream.message import StreamMessage
39 from .config import KafkaSubscriberConfig
42KAFKA_ADDRESS_SYNTAX = AddressSyntax(
43 replace_symbol=".*",
44 patch_regex=lambda x: x.replace(r"\*", ".*"),
45)
48class LogicSubscriber(TasksMixin, SubscriberUsecase[MsgType]):
49 """A class to handle logic for consuming messages from Kafka."""
51 __slots__ = (
52 "_connection_args",
53 "_listener",
54 "_partitions",
55 "_pattern",
56 "_topics",
57 "consumer",
58 "group_id",
59 )
61 consumer: Optional["AIOKafkaConsumer"]
63 batch: bool
64 parser: AioKafkaParser
66 _outer_config: "KafkaBrokerConfig"
67 _call_wrapper_class = KafkaHandlerCallWrapper
69 def __init__(
70 self,
71 config: "KafkaSubscriberConfig",
72 specification: "SubscriberSpecification[Any, Any]",
73 calls: "CallsCollection[MsgType]",
74 ) -> None:
75 super().__init__(config, specification, calls)
77 self._topics = config.topics
78 self._partitions = config.partitions
79 self.group_id = config.group_id
81 self._pattern = config.pattern
82 self._listener = config.listener
83 self._connection_args = config.connection_args
85 self.consumer = None
87 @overload
88 def __call__(
89 self,
90 func: Callable[P_HandlerParams, T_HandlerReturn],
91 *,
92 filter: "Filter[Any]" = default_filter,
93 parser: Optional["CustomCallable"] = None,
94 decoder: Optional["CustomCallable"] = None,
95 dependencies: Sequence["Dependant"] = (),
96 ) -> "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]": ...
98 @overload
99 def __call__(
100 self,
101 func: None = None,
102 *,
103 filter: "Filter[Any]" = default_filter,
104 parser: Optional["CustomCallable"] = None,
105 decoder: Optional["CustomCallable"] = None,
106 dependencies: Sequence["Dependant"] = (),
107 ) -> Callable[
108 [Callable[P_HandlerParams, T_HandlerReturn]],
109 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
110 ]: ...
112 @override
113 def __call__(
114 self,
115 func: Callable[P_HandlerParams, T_HandlerReturn] | None = None,
116 *,
117 filter: "Filter[Any]" = default_filter,
118 parser: Optional["CustomCallable"] = None,
119 decoder: Optional["CustomCallable"] = None,
120 dependencies: Sequence["Dependant"] = (),
121 ) -> Union[
122 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
123 Callable[
124 [Callable[P_HandlerParams, T_HandlerReturn]],
125 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]",
126 ],
127 ]:
128 # The base builds the wrapper from `_call_wrapper_class`; this only narrows the name
129 return cast(
130 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn] | Callable["
131 "[Callable[P_HandlerParams, T_HandlerReturn]], "
132 "KafkaHandlerCallWrapper[P_HandlerParams, T_HandlerReturn]]",
133 super().__call__(
134 func,
135 filter=filter,
136 parser=parser,
137 decoder=decoder,
138 dependencies=dependencies,
139 ),
140 )
142 @property
143 def pattern(self) -> Address | None:
144 """The pattern this Subscriber was declared with, and its Broker address."""
145 if not self._pattern:
146 return None
148 return Address(self._pattern, KAFKA_ADDRESS_SYNTAX).add_prefix(
149 self._outer_config.prefix,
150 )
152 @property
153 def topics(self) -> list[Topic]:
154 return [t.add_prefix(self._outer_config.prefix) for t in self._topics]
156 @property
157 def topic_names_for_subscribe(self) -> list[str]:
158 return [t.name for t in self.topics]
160 @property
161 def partitions(self) -> list[AIOKafkaTopicPartition]:
162 """The assignment as the consumer receives it: the client library's tuples, prefixed.
164 Declared with `faststream.kafka.TopicPartition`; handed to aiokafka as its own.
165 """
166 return [
167 AIOKafkaTopicPartition(
168 topic=f"{self._outer_config.prefix}{p.topic}",
169 partition=p.partition,
170 )
171 for p in self._partitions
172 ]
174 @property
175 def builder(self) -> Callable[..., "AIOKafkaConsumer"]:
176 return self._outer_config.builder
178 @property
179 def client_id(self) -> str | None:
180 return self._outer_config.client_id
182 @property
183 def topics_to_create(self) -> list[Topic]:
184 # Conflicting duplicates are reported by `create_subscriber`, the only
185 # public way to get here, so collapsing to the last one is enough.
186 topics: dict[str, Topic] = {t.name: t for t in self.topics}
188 prefix = self._outer_config.prefix
189 for p in self._partitions:
190 # Conflicting declare flags are reported by `create_subscriber`.
191 partition = p.add_prefix(prefix)
192 topics[partition.topic] = Topic(
193 partition.topic,
194 declare=partition.declare,
195 )
197 return [t for t in topics.values() if t.declare]
199 async def _ensure_topics(self) -> None:
200 if not self._outer_config.allow_auto_create_topics:
201 self._log(
202 logging.WARNING,
203 "Auto create topics is disabled. Make sure the topics exist.",
204 )
205 return
207 if self._outer_config.consumer_only:
208 self._log(
209 logging.WARNING,
210 "Topic creation is skipped in consumer-only mode. Make sure the topics exist.",
211 )
212 return
214 topics = self.topics_to_create
215 if not topics:
216 return
218 results = await self._outer_config.admin.create_topics(topics)
220 for create_result in results:
221 if create_result.error:
222 self._log(
223 logging.WARNING,
224 f"Failed to create topic {create_result.topic}: {create_result.error}",
225 )
227 async def start(self) -> None:
228 """Start the consumer."""
229 await super().start()
230 await self._ensure_topics()
232 self.consumer = consumer = self.builder(
233 group_id=self.group_id,
234 client_id=self.client_id,
235 **self._connection_args,
236 )
238 self.parser._setup(consumer)
240 pattern = self.pattern
241 if self.topics or pattern:
242 consumer.subscribe(
243 topics=self.topic_names_for_subscribe,
244 pattern=pattern.broker_address if pattern else None,
245 listener=make_logging_listener(
246 consumer=consumer,
247 logger=self._outer_config.logger.logger.logger,
248 log_extra=self.get_log_context(None),
249 listener=self._listener,
250 ),
251 )
253 elif self.partitions: 253 ↛ 256line 253 didn't jump to line 256 because the condition on line 253 was always true
254 consumer.assign(partitions=self.partitions)
256 await consumer.start()
258 self._post_start()
260 if self.calls:
261 self.add_task(self._run_consume_loop, (self.consumer,))
263 async def stop(self) -> None:
264 await super().stop()
266 if self.consumer is not None:
267 await self.consumer.stop()
268 self.consumer = None
270 @override
271 async def get_one(
272 self,
273 *,
274 timeout: float = 5.0,
275 ) -> "KafkaMessage | None":
276 assert not self.calls, (
277 "You can't use `get_one` method if subscriber has registered handlers."
278 )
280 assert self.consumer, "You should start subscriber at first."
282 raw_messages = await self.consumer.getmany(
283 timeout_ms=timeout * 1000,
284 max_records=1,
285 )
287 if not raw_messages:
288 return None
290 ((raw_message,),) = raw_messages.values()
292 context = self._outer_config.context
294 async_parser, async_decoder = self._get_parser_and_decoder()
296 msg: KafkaMessage | None = await process_msg( # type: ignore[assignment]
297 msg=raw_message,
298 middlewares=(
299 m(raw_message, context=context) for m in self._broker_middlewares
300 ),
301 parser=async_parser,
302 decoder=async_decoder,
303 )
304 return msg
306 @override
307 async def __aiter__(self) -> AsyncIterator["KafkaMessage"]:
308 assert self.consumer, "You should start subscriber at first."
309 assert not self.calls, (
310 "You can't use `get_one` method if subscriber has registered handlers."
311 )
313 context = self._outer_config.context
314 async_parser, async_decoder = self._get_parser_and_decoder()
316 async for raw_message in self.consumer:
317 msg: KafkaMessage = await process_msg( # type: ignore[assignment]
318 msg=raw_message,
319 middlewares=(
320 m(raw_message, context=context) for m in self._broker_middlewares
321 ),
322 parser=async_parser,
323 decoder=async_decoder,
324 )
325 yield msg
327 def _make_response_publisher(
328 self,
329 message: "StreamMessage[Any]",
330 ) -> Sequence["PublisherProto"]:
331 return (
332 KafkaFakePublisher(
333 self._outer_config.producer,
334 topic=message.reply_to,
335 ),
336 )
338 @abstractmethod
339 async def get_msg(self, consumer: "AIOKafkaConsumer") -> MsgType:
340 raise NotImplementedError
342 async def _run_consume_loop(self, consumer: "AIOKafkaConsumer") -> None:
343 assert consumer, "You should start subscriber at first."
345 connected = True
346 while self.running:
347 try:
348 msg = await self.get_msg(consumer)
350 except UnsupportedCodecError as e: # ruff: ignore[try-except-in-loop]
351 self._log(
352 logging.ERROR,
353 "There is no suitable compression library available. Please refer to the Kafka "
354 "documentation for more information - "
355 "https://aiokafka.readthedocs.io/en/stable/#installation",
356 exc_info=e,
357 )
358 await anyio.sleep(15)
360 except KafkaError as e:
361 self._log(logging.ERROR, "Kafka error occurred", exc_info=e)
363 if connected:
364 connected = False
366 await anyio.sleep(5)
368 except ConsumerStoppedError:
369 return
371 else:
372 if not connected: # pragma: no cover
373 connected = True
375 if msg: 375 ↛ 346line 375 didn't jump to line 346 because the condition on line 375 was always true
376 await self.consume_one(msg)
378 async def consume_one(self, msg: MsgType) -> None:
379 await self.consume(msg)
381 @property
382 def topic_names(self) -> list[str]:
383 if pattern := self.pattern:
384 topics = [pattern.broker_address]
386 elif self.topics:
387 topics = self.topic_names_for_subscribe
389 else:
390 topics = [f"{p.topic}-{p.partition}" for p in self.partitions]
392 return topics
394 @staticmethod
395 def build_log_context(
396 message: Optional["StreamMessage[Any]"],
397 topic: str,
398 group_id: str | None = None,
399 ) -> dict[str, str]:
400 return {
401 "topic": topic,
402 "group_id": group_id or "",
403 "message_id": getattr(message, "message_id", ""),
404 }
407class DefaultSubscriber(LogicSubscriber["ConsumerRecord"]):
408 __slots__ = ("parser",)
410 def __init__(
411 self,
412 config: "KafkaSubscriberConfig",
413 specification: "SubscriberSpecification[Any, Any]",
414 calls: "CallsCollection[ConsumerRecord]",
415 ) -> None:
416 reg = (
417 Address(config.pattern, KAFKA_ADDRESS_SYNTAX).regex
418 if config.pattern
419 else None
420 )
422 self.parser = AioKafkaParser(
423 msg_class=KafkaMessage if config.ack_first else KafkaAckableMessage,
424 regex=reg,
425 )
426 config.parser = self.parser.parse_message
427 config.decoder = self.parser.decode_message
428 super().__init__(config, specification, calls)
430 @override
431 async def get_msg(self, consumer: "AIOKafkaConsumer") -> "ConsumerRecord":
432 assert consumer, "You should setup subscriber at first."
433 return await consumer.getone()
435 def get_log_context(
436 self,
437 message: Optional["StreamMessage[ConsumerRecord]"],
438 ) -> dict[str, str]:
439 if message is None:
440 topic = ",".join(self.topic_names)
441 else:
442 topic = message.raw_message.topic
444 return self.build_log_context(
445 message=message,
446 topic=topic,
447 group_id=self.group_id,
448 )
451class BatchSubscriber(LogicSubscriber[tuple["ConsumerRecord", ...]]):
452 __slots__ = (
453 "batch_timeout_ms",
454 "max_records",
455 "parser",
456 )
458 def __init__(
459 self,
460 config: "KafkaSubscriberConfig",
461 specification: "SubscriberSpecification[Any, Any]",
462 calls: "CallsCollection[tuple[ConsumerRecord, ...]]",
463 batch_timeout_ms: int,
464 max_records: int | None,
465 ) -> None:
466 reg = (
467 Address(config.pattern, KAFKA_ADDRESS_SYNTAX).regex
468 if config.pattern
469 else None
470 )
472 self.parser = AioKafkaBatchParser(
473 msg_class=KafkaMessage if config.ack_first else KafkaAckableMessage,
474 regex=reg,
475 )
476 config.decoder = self.parser.decode_batch
477 config.parser = self.parser.parse_batch
478 super().__init__(config, specification, calls)
480 self.batch_timeout_ms = batch_timeout_ms
481 self.max_records = max_records
483 async def get_msg(
484 self,
485 consumer: "AIOKafkaConsumer",
486 ) -> tuple["ConsumerRecord", ...]:
487 assert consumer, "You should setup subscriber at first."
489 messages = await consumer.getmany(
490 timeout_ms=self.batch_timeout_ms,
491 max_records=self.max_records,
492 )
494 if not messages: # pragma: no cover
495 await anyio.sleep(self.batch_timeout_ms / 1000)
496 return ()
498 return tuple(chain(*messages.values()))
500 def get_log_context(
501 self,
502 message: Optional["StreamMessage[tuple[ConsumerRecord, ...]]"],
503 ) -> dict[str, str]:
504 if message is None:
505 topic = ",".join(self.topic_names)
506 else:
507 topic = message.raw_message[0].topic
509 return self.build_log_context(
510 message=message,
511 topic=topic,
512 group_id=self.group_id,
513 )
516class ConcurrentDefaultSubscriber(ConcurrentMixin["ConsumerRecord"], DefaultSubscriber):
517 __slots__ = ()
519 async def start(self) -> None:
520 await super().start()
521 self.start_consume_task()
523 async def consume_one(self, msg: "ConsumerRecord") -> None:
524 await self._put_msg(msg)
527class ConcurrentBetweenPartitionsSubscriber(DefaultSubscriber):
528 __slots__ = (
529 "consumer_subgroup",
530 "max_workers",
531 )
533 consumer_subgroup: list["AIOKafkaConsumer"]
535 def __init__(
536 self,
537 config: "KafkaSubscriberConfig",
538 specification: "SubscriberSpecification[Any, Any]",
539 calls: "CallsCollection[ConsumerRecord]",
540 max_workers: int,
541 ) -> None:
542 super().__init__(config, specification, calls)
544 self.max_workers = max_workers
545 self.consumer_subgroup = []
547 async def start(self) -> None:
548 """Start the consumer subgroup."""
549 await super(LogicSubscriber, self).start()
550 await self._ensure_topics()
552 if self.calls: 552 ↛ 565line 552 didn't jump to line 565 because the condition on line 552 was always true
553 self.consumer_subgroup = [
554 self.builder(
555 group_id=self.group_id,
556 client_id=self.client_id,
557 **self._connection_args,
558 )
559 for _ in range(self.max_workers)
560 ]
562 else:
563 # We should create single consumer to support
564 # `get_one()` and `__aiter__` methods
565 self.consumer = self.builder(
566 group_id=self.group_id,
567 client_id=self.client_id,
568 **self._connection_args,
569 )
570 self.consumer_subgroup = [self.consumer]
572 # Subscribers starting should be called concurrently
573 # to balance them correctly
574 async with anyio.create_task_group() as tg:
575 for c in self.consumer_subgroup:
576 c.subscribe(
577 topics=self.topic_names_for_subscribe,
578 listener=make_logging_listener(
579 consumer=c,
580 logger=self._outer_config.logger.logger.logger,
581 log_extra=self.get_log_context(None),
582 listener=self._listener,
583 ),
584 )
586 _ = tg.start_soon(c.start)
588 self._post_start()
590 if self.calls: 590 ↛ exitline 590 didn't return from function 'start' because the condition on line 590 was always true
591 for c in self.consumer_subgroup:
592 self.add_task(self._run_consume_loop, (c,))
594 async def stop(self) -> None:
595 if self.consumer_subgroup: 595 ↛ 602line 595 didn't jump to line 602 because the condition on line 595 was always true
596 async with anyio.create_task_group() as tg:
597 for consumer in self.consumer_subgroup:
598 _ = tg.start_soon(consumer.stop)
600 self.consumer_subgroup = []
602 await super().stop()
604 @override
605 async def get_msg(self, consumer: "AIOKafkaConsumer") -> "KafkaRawMessage":
606 assert consumer, "You should setup subscriber at first."
607 message = await consumer.getone()
608 message.consumer = consumer
609 return cast("KafkaRawMessage", message)