Coverage for faststream/confluent/helpers/client.py: 84%
123 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import asyncio
2import logging
3from collections.abc import Callable, Sequence
4from concurrent.futures import ThreadPoolExecutor
5from contextlib import suppress
6from time import time
7from typing import TYPE_CHECKING, Any, cast
9import anyio
10from confluent_kafka import Consumer, KafkaError, KafkaException, Message, Producer
12from faststream._internal.utils.functions import call_or_await, run_in_executor
13from faststream.confluent.schemas import Topic, TopicPartition
14from faststream.exceptions import SetupError
16from . import config as config_module
18if TYPE_CHECKING:
19 from typing_extensions import NotRequired, TypedDict
21 from faststream._internal.logger import LoggerState
23 from .admin import AdminService
25 class _SendKwargs(TypedDict):
26 value: bytes | str | None
27 key: bytes | str | None
28 headers: Sequence[tuple[str, str | bytes | None]] | None
29 partition: NotRequired[int]
30 timestamp: NotRequired[int]
31 on_delivery: NotRequired[Callable[..., None]]
34class _LazyLoggerProxy(logging.Logger):
35 """A logger proxy that lazily delegates to LoggerState.
37 confluent-kafka requires a logger at construction time, but LoggerState
38 may not be initialized yet (it is set up after ``_connect()``). This
39 proxy silently drops log records until the underlying LoggerState is
40 ready, then forwards them to the real logger.
41 """
43 def __init__(self, logger_state: "LoggerState") -> None:
44 super().__init__("faststream.confluent._proxy", logging.NOTSET)
45 self._logger_state = logger_state
47 def handle(self, record: logging.LogRecord) -> None:
48 real_logger = self._logger_state.logger.logger
49 if real_logger is not None and hasattr(real_logger, "handle"):
50 real_logger.handle(record)
53class AsyncConfluentProducer:
54 """An asynchronous Python Kafka client using the "confluent-kafka" package."""
56 __slots__ = (
57 "__running",
58 "_poll_task",
59 "config",
60 "logger_state",
61 "producer",
62 )
64 def __init__(
65 self,
66 *,
67 logger: "LoggerState",
68 config: config_module.ConfluentFastConfig,
69 ) -> None:
70 self.logger_state = logger
72 self.config = config.producer_config
73 self.producer = Producer(
74 self.config,
75 logger=_LazyLoggerProxy(logger),
76 )
78 self.__running = True
79 self._poll_task = asyncio.create_task(self._poll_loop())
81 async def _poll_loop(self) -> None:
82 while self.__running: 82 ↛ exitline 82 didn't return from function '_poll_loop' because the condition on line 82 was always true
83 with suppress(Exception):
84 await call_or_await(self.producer.poll, 0.1)
86 async def stop(self) -> None:
87 """Stop the Kafka producer and flush remaining messages."""
88 if self.__running: 88 ↛ exitline 88 didn't return from function 'stop' because the condition on line 88 was always true
89 self.__running = False
90 if not self._poll_task.done(): 90 ↛ 92line 90 didn't jump to line 92 because the condition on line 90 was always true
91 self._poll_task.cancel()
92 await call_or_await(self.producer.flush)
94 async def flush(self) -> None:
95 await call_or_await(self.producer.flush)
97 async def send(
98 self,
99 topic: str,
100 value: bytes | str | None = None,
101 key: bytes | str | None = None,
102 partition: int | None = None,
103 timestamp_ms: int | None = None,
104 headers: Sequence[tuple[str, str | bytes | None]] | None = None,
105 no_confirm: bool = False,
106 ) -> "asyncio.Future[Message | None] | Message | None":
107 """Sends a single message to a Kafka topic."""
108 kwargs: _SendKwargs = {
109 "value": value,
110 "key": key,
111 "headers": headers,
112 }
114 if partition is not None: 114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true
115 kwargs["partition"] = partition
117 if timestamp_ms is not None:
118 kwargs["timestamp"] = timestamp_ms
120 loop = asyncio.get_running_loop()
121 result_future: asyncio.Future[Message | None] = loop.create_future()
123 def ack_callback(err: Any, msg: Message | None) -> None:
124 if err or (msg is not None and (err := msg.error())): 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true
125 loop.call_soon_threadsafe(
126 result_future.set_exception,
127 KafkaException(err),
128 )
129 else:
130 loop.call_soon_threadsafe(result_future.set_result, msg)
132 kwargs["on_delivery"] = ack_callback
134 # should be sync to prevent segfault
135 # confluent stub expects bytes|None for value/key; we accept str and encode
136 produce_value: bytes | None = (
137 kwargs["value"]
138 if isinstance(kwargs["value"], (bytes, type(None)))
139 else kwargs["value"].encode()
140 )
141 produce_key: bytes | None = (
142 kwargs["key"]
143 if isinstance(kwargs["key"], (bytes, type(None)))
144 else kwargs["key"].encode()
145 )
146 produce_headers = (
147 list(kwargs["headers"]) if kwargs["headers"] is not None else None
148 )
149 produce_kwargs: dict[str, Any] = {
150 "value": produce_value,
151 "key": produce_key,
152 "headers": produce_headers,
153 "on_delivery": kwargs["on_delivery"],
154 }
155 if kwargs.get("partition") is not None: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true
156 produce_kwargs["partition"] = kwargs["partition"]
157 if kwargs.get("timestamp") is not None:
158 produce_kwargs["timestamp"] = kwargs["timestamp"]
159 self.producer.produce(topic, **produce_kwargs)
161 if no_confirm: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 return result_future
163 return await result_future
165 def create_batch(self) -> "BatchBuilder": # ruff: ignore[no-self-use]
166 """Creates a batch for sending multiple messages."""
167 return BatchBuilder()
169 async def send_batch(
170 self,
171 batch: "BatchBuilder",
172 topic: str,
173 *,
174 partition: int | None,
175 no_confirm: bool = False,
176 ) -> None:
177 """Sends a batch of messages to a Kafka topic."""
178 async with anyio.create_task_group() as tg:
179 for msg in batch._builder:
180 _ = tg.start_soon(
181 self.send,
182 topic,
183 msg["value"],
184 msg["key"],
185 partition,
186 msg["timestamp_ms"],
187 msg["headers"],
188 no_confirm,
189 )
191 async def ping(
192 self,
193 timeout: float | None = 5.0,
194 ) -> bool:
195 """Implement ping using `list_topics` information request."""
196 if timeout is None: 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true
197 timeout = -1
199 try:
200 cluster_metadata = await call_or_await(
201 self.producer.list_topics,
202 timeout=timeout,
203 )
205 return bool(cluster_metadata)
207 except Exception:
208 return False
211class AsyncConfluentConsumer:
212 """An asynchronous Python Kafka client for consuming messages using the "confluent-kafka" package."""
214 __slots__ = (
215 "_on_assign",
216 "_on_lost",
217 "_on_revoke",
218 "_thread_pool",
219 "admin_client",
220 "config",
221 "consumer",
222 "logger_state",
223 "partitions",
224 "topics",
225 )
227 def __init__(
228 self,
229 *topics: "Topic",
230 config: config_module.ConfluentFastConfig,
231 logger: "LoggerState",
232 admin_service: "AdminService",
233 # kwargs options
234 partitions: Sequence["TopicPartition"],
235 bootstrap_servers: str | list[str] = "localhost",
236 # consumer options
237 client_id: str | None = "confluent-kafka-consumer",
238 group_id: str | None = None,
239 group_instance_id: str | None = None,
240 fetch_max_wait_ms: int = 500,
241 fetch_max_bytes: int = 52428800,
242 fetch_min_bytes: int = 1,
243 max_partition_fetch_bytes: int = 1 * 1024 * 1024,
244 retry_backoff_ms: int = 100,
245 auto_offset_reset: str = "latest",
246 enable_auto_commit: bool = True,
247 auto_commit_interval_ms: int = 5000,
248 check_crcs: bool = True,
249 metadata_max_age_ms: int = 5 * 60 * 1000,
250 partition_assignment_strategy: str | list[Any] = "roundrobin",
251 max_poll_interval_ms: int = 300000,
252 session_timeout_ms: int = 10000,
253 heartbeat_interval_ms: int = 3000,
254 security_protocol: str = "PLAINTEXT",
255 connections_max_idle_ms: int = 540000,
256 isolation_level: str = "read_uncommitted",
257 allow_auto_create_topics: bool = True,
258 # rebalance callbacks
259 on_assign: Callable[..., None] | None = None,
260 on_revoke: Callable[..., None] | None = None,
261 on_lost: Callable[..., None] | None = None,
262 ) -> None:
263 self.admin_client = admin_service
264 self.logger_state = logger
266 self._on_assign = on_assign
267 self._on_revoke = on_revoke
268 self._on_lost = on_lost
270 self.topics = list(topics)
271 self.partitions = partitions
273 if not isinstance(partition_assignment_strategy, str):
274 partition_assignment_strategy = ",".join(
275 [
276 x if isinstance(x, str) else x().name
277 for x in partition_assignment_strategy
278 ],
279 )
281 config_from_params = {
282 "allow.auto.create.topics": allow_auto_create_topics,
283 "topic.metadata.refresh.interval.ms": 1000,
284 "bootstrap.servers": bootstrap_servers,
285 "client.id": client_id,
286 "group.id": group_id or "faststream-consumer-group",
287 "group.instance.id": group_instance_id,
288 "fetch.wait.max.ms": fetch_max_wait_ms,
289 "fetch.max.bytes": fetch_max_bytes,
290 "fetch.min.bytes": fetch_min_bytes,
291 "max.partition.fetch.bytes": max_partition_fetch_bytes,
292 "fetch.error.backoff.ms": retry_backoff_ms,
293 "auto.offset.reset": auto_offset_reset,
294 "enable.auto.commit": enable_auto_commit,
295 # FastStream enables auto commit only for ACK_FIRST. Other ack
296 # policies must store offsets on ack, not when records are polled.
297 "enable.auto.offset.store": enable_auto_commit,
298 "auto.commit.interval.ms": auto_commit_interval_ms,
299 "check.crcs": check_crcs,
300 "metadata.max.age.ms": metadata_max_age_ms,
301 "partition.assignment.strategy": partition_assignment_strategy,
302 "max.poll.interval.ms": max_poll_interval_ms,
303 "session.timeout.ms": session_timeout_ms,
304 "heartbeat.interval.ms": heartbeat_interval_ms,
305 "security.protocol": security_protocol.lower(),
306 "connections.max.idle.ms": connections_max_idle_ms,
307 "isolation.level": isolation_level,
308 } | config.consumer_config
310 if config_from_params.get("group.protocol") == "consumer":
311 # KIP-848 moves assignment and timers to the broker; librdkafka rejects
312 # these classic options even when they come from the config passthrough.
313 for classic_only in (
314 "partition.assignment.strategy",
315 "session.timeout.ms",
316 "heartbeat.interval.ms",
317 ):
318 config_from_params.pop(classic_only, None)
320 if not enable_auto_commit and config_from_params["enable.auto.offset.store"]:
321 msg = "Manual acknowledgement requires `enable.auto.offset.store=False`."
322 raise SetupError(msg)
324 self.config = config_from_params
325 self.consumer = Consumer(self.config, logger=_LazyLoggerProxy(logger))
327 # A pool with single thread is used in order to execute the commands of the consumer sequentially:
328 # https://github.com/ag2ai/faststream/issues/1904#issuecomment-2506990895
329 self._thread_pool = ThreadPoolExecutor(max_workers=1)
331 @property
332 def topics_to_create(self) -> list["Topic"]:
333 # Conflicting duplicates are reported by `create_subscriber`, the only
334 # public way to get here, so collapsing to the last one is enough.
335 topics: dict[str, Topic] = {t.name: t for t in self.topics}
337 for p in self.partitions:
338 topics.setdefault(p.topic, Topic(p.topic, declare=p.declare))
340 return [t for t in topics.values() if t.declare]
342 async def start(self) -> None:
343 """Starts the Kafka consumer and subscribes to the specified topics."""
344 if self.config.get("allow.auto.create.topics", True):
345 topics_creation_result = await run_in_executor(
346 self._thread_pool,
347 self.admin_client.create_topics,
348 self.topics_to_create,
349 )
351 for create_result in topics_creation_result:
352 if create_result.error:
353 self.logger_state.log(
354 log_level=logging.WARNING,
355 message=f"Failed to create topic {create_result.topic}: {create_result.error}",
356 )
358 else:
359 self.logger_state.log(
360 log_level=logging.WARNING,
361 message="Auto create topics is disabled. Make sure the topics exist.",
362 )
364 if self.topics:
365 subscribe_kwargs: dict[str, Any] = {"topics": [t.name for t in self.topics]}
366 if self._on_assign is not None: 366 ↛ 367line 366 didn't jump to line 367 because the condition on line 366 was never true
367 subscribe_kwargs["on_assign"] = self._on_assign
368 if self._on_revoke is not None: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true
369 subscribe_kwargs["on_revoke"] = self._on_revoke
370 if self._on_lost is not None: 370 ↛ 371line 370 didn't jump to line 371 because the condition on line 370 was never true
371 subscribe_kwargs["on_lost"] = self._on_lost
372 await run_in_executor(
373 self._thread_pool,
374 self.consumer.subscribe,
375 **subscribe_kwargs,
376 )
378 elif self.partitions: 378 ↛ 386line 378 didn't jump to line 386 because the condition on line 378 was always true
379 await run_in_executor(
380 self._thread_pool,
381 self.consumer.assign,
382 [p.to_confluent() for p in self.partitions],
383 )
385 else:
386 msg = "You must provide either `topics` or `partitions` option."
387 raise SetupError(msg)
389 async def commit(self, asynchronous: bool = True) -> None:
390 """Commits the offsets of all messages returned by the last poll operation."""
391 await run_in_executor(
392 self._thread_pool,
393 lambda: self.consumer.commit(asynchronous=asynchronous), # type: ignore[call-overload]
394 )
396 async def stop(self) -> None:
397 """Stops the Kafka consumer and releases all resources."""
398 # NOTE: If we don't explicitly call commit and then close the consumer, the confluent consumer gets stuck.
399 # We are doing this to avoid the issue.
400 enable_auto_commit = self.config.get("enable.auto.commit", True)
402 try:
403 if enable_auto_commit:
404 await self.commit(asynchronous=False)
406 except Exception as e:
407 # No offset stored issue is not a problem - https://github.com/confluentinc/confluent-kafka-python/issues/295#issuecomment-355907183
408 if "No offset stored" in str(e):
409 pass
410 else:
411 self.logger_state.log(
412 log_level=logging.ERROR,
413 message="Consumer closing error occurred.",
414 exc_info=e,
415 )
417 # Wrap calls to async to make method cancelable by timeout
418 # We shouldn't read messages and close consumer concurrently
419 # https://github.com/ag2ai/faststream/issues/1904#issuecomment-2506990895
420 # Now it works without lock due `ThreadPoolExecutor(max_workers=1)`
421 # that makes all calls to consumer sequential
422 await run_in_executor(self._thread_pool, self.consumer.close)
424 self._thread_pool.shutdown(wait=False)
426 async def getone(self, timeout: float = 0.1) -> Message | None:
427 """Consumes a single message from Kafka."""
428 msg = await run_in_executor(self._thread_pool, self.consumer.poll, timeout)
429 return check_msg_error(msg)
431 async def getmany(
432 self,
433 timeout: float = 0.1,
434 max_records: int | None = 10,
435 ) -> tuple[Message, ...]:
436 """Consumes a batch of messages from Kafka and groups them by topic and partition."""
437 raw_messages: list[Message | None] = await run_in_executor(
438 self._thread_pool,
439 cast(
440 "Callable[..., list[Message | None]]",
441 lambda: self.consumer.consume(
442 num_messages=max_records or 10,
443 timeout=timeout,
444 ),
445 ),
446 )
447 return tuple(x for x in map(check_msg_error, raw_messages) if x is not None)
449 async def seek(self, topic: str, partition: int, offset: int) -> None:
450 """Seeks to the specified offset in the specified topic and partition."""
451 topic_partition = TopicPartition(
452 topic=topic,
453 partition=partition,
454 offset=offset,
455 )
456 await run_in_executor(
457 self._thread_pool,
458 self.consumer.seek,
459 topic_partition.to_confluent(),
460 )
462 async def store_offsets(self, *topic_partitions: TopicPartition) -> None:
463 """Stores the offsets to commit for the specified topic partitions."""
464 if not topic_partitions:
465 return
467 await run_in_executor(
468 self._thread_pool,
469 self.consumer.store_offsets,
470 offsets=[
471 topic_partition.to_confluent() for topic_partition in topic_partitions
472 ],
473 )
476def check_msg_error(msg: Message | None) -> Message | None:
477 """Checks for errors in the consumed message."""
478 if msg is None or msg.error():
479 return None
481 return msg
484class BatchBuilder:
485 """A helper class to build a batch of messages to send to Kafka."""
487 __slots__ = ("_builder",)
489 def __init__(self) -> None:
490 """Initializes a new BatchBuilder instance."""
491 self._builder: list[dict[str, Any]] = []
493 def append(
494 self,
495 *,
496 value: bytes | str | None = None,
497 key: bytes | str | None = None,
498 timestamp: int | None = None,
499 headers: Sequence[tuple[str, bytes | None]] | None = None,
500 ) -> None:
501 """Appends a message to the batch with optional timestamp, key, value, and headers."""
502 if key is None and value is None: 502 ↛ 503line 502 didn't jump to line 503 because the condition on line 502 was never true
503 raise KafkaException(
504 KafkaError(40, "Both key and value can't be None"),
505 )
507 self._builder.append(
508 {
509 "timestamp_ms": timestamp or round(time() * 1000),
510 "key": key,
511 "value": value,
512 "headers": headers or [],
513 },
514 )