Coverage for faststream/confluent/helpers/client.py: 84%

123 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1import asyncio 

2import logging 

3from collections.abc import Callable, Sequence 

4from concurrent.futures import ThreadPoolExecutor 

5from contextlib import suppress 

6from time import time 

7from typing import TYPE_CHECKING, Any, cast 

8 

9import anyio 

10from confluent_kafka import Consumer, KafkaError, KafkaException, Message, Producer 

11 

12from faststream._internal.utils.functions import call_or_await, run_in_executor 

13from faststream.confluent.schemas import Topic, TopicPartition 

14from faststream.exceptions import SetupError 

15 

16from . import config as config_module 

17 

18if TYPE_CHECKING: 

19 from typing_extensions import NotRequired, TypedDict 

20 

21 from faststream._internal.logger import LoggerState 

22 

23 from .admin import AdminService 

24 

25 class _SendKwargs(TypedDict): 

26 value: bytes | str | None 

27 key: bytes | str | None 

28 headers: Sequence[tuple[str, str | bytes | None]] | None 

29 partition: NotRequired[int] 

30 timestamp: NotRequired[int] 

31 on_delivery: NotRequired[Callable[..., None]] 

32 

33 

34class _LazyLoggerProxy(logging.Logger): 

35 """A logger proxy that lazily delegates to LoggerState. 

36 

37 confluent-kafka requires a logger at construction time, but LoggerState 

38 may not be initialized yet (it is set up after ``_connect()``). This 

39 proxy silently drops log records until the underlying LoggerState is 

40 ready, then forwards them to the real logger. 

41 """ 

42 

43 def __init__(self, logger_state: "LoggerState") -> None: 

44 super().__init__("faststream.confluent._proxy", logging.NOTSET) 

45 self._logger_state = logger_state 

46 

47 def handle(self, record: logging.LogRecord) -> None: 

48 real_logger = self._logger_state.logger.logger 

49 if real_logger is not None and hasattr(real_logger, "handle"): 

50 real_logger.handle(record) 

51 

52 

53class AsyncConfluentProducer: 

54 """An asynchronous Python Kafka client using the "confluent-kafka" package.""" 

55 

56 __slots__ = ( 

57 "__running", 

58 "_poll_task", 

59 "config", 

60 "logger_state", 

61 "producer", 

62 ) 

63 

64 def __init__( 

65 self, 

66 *, 

67 logger: "LoggerState", 

68 config: config_module.ConfluentFastConfig, 

69 ) -> None: 

70 self.logger_state = logger 

71 

72 self.config = config.producer_config 

73 self.producer = Producer( 

74 self.config, 

75 logger=_LazyLoggerProxy(logger), 

76 ) 

77 

78 self.__running = True 

79 self._poll_task = asyncio.create_task(self._poll_loop()) 

80 

81 async def _poll_loop(self) -> None: 

82 while self.__running: 82 ↛ exitline 82 didn't return from function '_poll_loop' because the condition on line 82 was always true

83 with suppress(Exception): 

84 await call_or_await(self.producer.poll, 0.1) 

85 

86 async def stop(self) -> None: 

87 """Stop the Kafka producer and flush remaining messages.""" 

88 if self.__running: 88 ↛ exitline 88 didn't return from function 'stop' because the condition on line 88 was always true

89 self.__running = False 

90 if not self._poll_task.done(): 90 ↛ 92line 90 didn't jump to line 92 because the condition on line 90 was always true

91 self._poll_task.cancel() 

92 await call_or_await(self.producer.flush) 

93 

94 async def flush(self) -> None: 

95 await call_or_await(self.producer.flush) 

96 

97 async def send( 

98 self, 

99 topic: str, 

100 value: bytes | str | None = None, 

101 key: bytes | str | None = None, 

102 partition: int | None = None, 

103 timestamp_ms: int | None = None, 

104 headers: Sequence[tuple[str, str | bytes | None]] | None = None, 

105 no_confirm: bool = False, 

106 ) -> "asyncio.Future[Message | None] | Message | None": 

107 """Sends a single message to a Kafka topic.""" 

108 kwargs: _SendKwargs = { 

109 "value": value, 

110 "key": key, 

111 "headers": headers, 

112 } 

113 

114 if partition is not None: 114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true

115 kwargs["partition"] = partition 

116 

117 if timestamp_ms is not None: 

118 kwargs["timestamp"] = timestamp_ms 

119 

120 loop = asyncio.get_running_loop() 

121 result_future: asyncio.Future[Message | None] = loop.create_future() 

122 

123 def ack_callback(err: Any, msg: Message | None) -> None: 

124 if err or (msg is not None and (err := msg.error())): 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true

125 loop.call_soon_threadsafe( 

126 result_future.set_exception, 

127 KafkaException(err), 

128 ) 

129 else: 

130 loop.call_soon_threadsafe(result_future.set_result, msg) 

131 

132 kwargs["on_delivery"] = ack_callback 

133 

134 # should be sync to prevent segfault 

135 # confluent stub expects bytes|None for value/key; we accept str and encode 

136 produce_value: bytes | None = ( 

137 kwargs["value"] 

138 if isinstance(kwargs["value"], (bytes, type(None))) 

139 else kwargs["value"].encode() 

140 ) 

141 produce_key: bytes | None = ( 

142 kwargs["key"] 

143 if isinstance(kwargs["key"], (bytes, type(None))) 

144 else kwargs["key"].encode() 

145 ) 

146 produce_headers = ( 

147 list(kwargs["headers"]) if kwargs["headers"] is not None else None 

148 ) 

149 produce_kwargs: dict[str, Any] = { 

150 "value": produce_value, 

151 "key": produce_key, 

152 "headers": produce_headers, 

153 "on_delivery": kwargs["on_delivery"], 

154 } 

155 if kwargs.get("partition") is not None: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true

156 produce_kwargs["partition"] = kwargs["partition"] 

157 if kwargs.get("timestamp") is not None: 

158 produce_kwargs["timestamp"] = kwargs["timestamp"] 

159 self.producer.produce(topic, **produce_kwargs) 

160 

161 if no_confirm: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 return result_future 

163 return await result_future 

164 

165 def create_batch(self) -> "BatchBuilder": # ruff: ignore[no-self-use] 

166 """Creates a batch for sending multiple messages.""" 

167 return BatchBuilder() 

168 

169 async def send_batch( 

170 self, 

171 batch: "BatchBuilder", 

172 topic: str, 

173 *, 

174 partition: int | None, 

175 no_confirm: bool = False, 

176 ) -> None: 

177 """Sends a batch of messages to a Kafka topic.""" 

178 async with anyio.create_task_group() as tg: 

179 for msg in batch._builder: 

180 _ = tg.start_soon( 

181 self.send, 

182 topic, 

183 msg["value"], 

184 msg["key"], 

185 partition, 

186 msg["timestamp_ms"], 

187 msg["headers"], 

188 no_confirm, 

189 ) 

190 

191 async def ping( 

192 self, 

193 timeout: float | None = 5.0, 

194 ) -> bool: 

195 """Implement ping using `list_topics` information request.""" 

196 if timeout is None: 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true

197 timeout = -1 

198 

199 try: 

200 cluster_metadata = await call_or_await( 

201 self.producer.list_topics, 

202 timeout=timeout, 

203 ) 

204 

205 return bool(cluster_metadata) 

206 

207 except Exception: 

208 return False 

209 

210 

211class AsyncConfluentConsumer: 

212 """An asynchronous Python Kafka client for consuming messages using the "confluent-kafka" package.""" 

213 

214 __slots__ = ( 

215 "_on_assign", 

216 "_on_lost", 

217 "_on_revoke", 

218 "_thread_pool", 

219 "admin_client", 

220 "config", 

221 "consumer", 

222 "logger_state", 

223 "partitions", 

224 "topics", 

225 ) 

226 

227 def __init__( 

228 self, 

229 *topics: "Topic", 

230 config: config_module.ConfluentFastConfig, 

231 logger: "LoggerState", 

232 admin_service: "AdminService", 

233 # kwargs options 

234 partitions: Sequence["TopicPartition"], 

235 bootstrap_servers: str | list[str] = "localhost", 

236 # consumer options 

237 client_id: str | None = "confluent-kafka-consumer", 

238 group_id: str | None = None, 

239 group_instance_id: str | None = None, 

240 fetch_max_wait_ms: int = 500, 

241 fetch_max_bytes: int = 52428800, 

242 fetch_min_bytes: int = 1, 

243 max_partition_fetch_bytes: int = 1 * 1024 * 1024, 

244 retry_backoff_ms: int = 100, 

245 auto_offset_reset: str = "latest", 

246 enable_auto_commit: bool = True, 

247 auto_commit_interval_ms: int = 5000, 

248 check_crcs: bool = True, 

249 metadata_max_age_ms: int = 5 * 60 * 1000, 

250 partition_assignment_strategy: str | list[Any] = "roundrobin", 

251 max_poll_interval_ms: int = 300000, 

252 session_timeout_ms: int = 10000, 

253 heartbeat_interval_ms: int = 3000, 

254 security_protocol: str = "PLAINTEXT", 

255 connections_max_idle_ms: int = 540000, 

256 isolation_level: str = "read_uncommitted", 

257 allow_auto_create_topics: bool = True, 

258 # rebalance callbacks 

259 on_assign: Callable[..., None] | None = None, 

260 on_revoke: Callable[..., None] | None = None, 

261 on_lost: Callable[..., None] | None = None, 

262 ) -> None: 

263 self.admin_client = admin_service 

264 self.logger_state = logger 

265 

266 self._on_assign = on_assign 

267 self._on_revoke = on_revoke 

268 self._on_lost = on_lost 

269 

270 self.topics = list(topics) 

271 self.partitions = partitions 

272 

273 if not isinstance(partition_assignment_strategy, str): 

274 partition_assignment_strategy = ",".join( 

275 [ 

276 x if isinstance(x, str) else x().name 

277 for x in partition_assignment_strategy 

278 ], 

279 ) 

280 

281 config_from_params = { 

282 "allow.auto.create.topics": allow_auto_create_topics, 

283 "topic.metadata.refresh.interval.ms": 1000, 

284 "bootstrap.servers": bootstrap_servers, 

285 "client.id": client_id, 

286 "group.id": group_id or "faststream-consumer-group", 

287 "group.instance.id": group_instance_id, 

288 "fetch.wait.max.ms": fetch_max_wait_ms, 

289 "fetch.max.bytes": fetch_max_bytes, 

290 "fetch.min.bytes": fetch_min_bytes, 

291 "max.partition.fetch.bytes": max_partition_fetch_bytes, 

292 "fetch.error.backoff.ms": retry_backoff_ms, 

293 "auto.offset.reset": auto_offset_reset, 

294 "enable.auto.commit": enable_auto_commit, 

295 # FastStream enables auto commit only for ACK_FIRST. Other ack 

296 # policies must store offsets on ack, not when records are polled. 

297 "enable.auto.offset.store": enable_auto_commit, 

298 "auto.commit.interval.ms": auto_commit_interval_ms, 

299 "check.crcs": check_crcs, 

300 "metadata.max.age.ms": metadata_max_age_ms, 

301 "partition.assignment.strategy": partition_assignment_strategy, 

302 "max.poll.interval.ms": max_poll_interval_ms, 

303 "session.timeout.ms": session_timeout_ms, 

304 "heartbeat.interval.ms": heartbeat_interval_ms, 

305 "security.protocol": security_protocol.lower(), 

306 "connections.max.idle.ms": connections_max_idle_ms, 

307 "isolation.level": isolation_level, 

308 } | config.consumer_config 

309 

310 if config_from_params.get("group.protocol") == "consumer": 

311 # KIP-848 moves assignment and timers to the broker; librdkafka rejects 

312 # these classic options even when they come from the config passthrough. 

313 for classic_only in ( 

314 "partition.assignment.strategy", 

315 "session.timeout.ms", 

316 "heartbeat.interval.ms", 

317 ): 

318 config_from_params.pop(classic_only, None) 

319 

320 if not enable_auto_commit and config_from_params["enable.auto.offset.store"]: 

321 msg = "Manual acknowledgement requires `enable.auto.offset.store=False`." 

322 raise SetupError(msg) 

323 

324 self.config = config_from_params 

325 self.consumer = Consumer(self.config, logger=_LazyLoggerProxy(logger)) 

326 

327 # A pool with single thread is used in order to execute the commands of the consumer sequentially: 

328 # https://github.com/ag2ai/faststream/issues/1904#issuecomment-2506990895 

329 self._thread_pool = ThreadPoolExecutor(max_workers=1) 

330 

331 @property 

332 def topics_to_create(self) -> list["Topic"]: 

333 # Conflicting duplicates are reported by `create_subscriber`, the only 

334 # public way to get here, so collapsing to the last one is enough. 

335 topics: dict[str, Topic] = {t.name: t for t in self.topics} 

336 

337 for p in self.partitions: 

338 topics.setdefault(p.topic, Topic(p.topic, declare=p.declare)) 

339 

340 return [t for t in topics.values() if t.declare] 

341 

342 async def start(self) -> None: 

343 """Starts the Kafka consumer and subscribes to the specified topics.""" 

344 if self.config.get("allow.auto.create.topics", True): 

345 topics_creation_result = await run_in_executor( 

346 self._thread_pool, 

347 self.admin_client.create_topics, 

348 self.topics_to_create, 

349 ) 

350 

351 for create_result in topics_creation_result: 

352 if create_result.error: 

353 self.logger_state.log( 

354 log_level=logging.WARNING, 

355 message=f"Failed to create topic {create_result.topic}: {create_result.error}", 

356 ) 

357 

358 else: 

359 self.logger_state.log( 

360 log_level=logging.WARNING, 

361 message="Auto create topics is disabled. Make sure the topics exist.", 

362 ) 

363 

364 if self.topics: 

365 subscribe_kwargs: dict[str, Any] = {"topics": [t.name for t in self.topics]} 

366 if self._on_assign is not None: 366 ↛ 367line 366 didn't jump to line 367 because the condition on line 366 was never true

367 subscribe_kwargs["on_assign"] = self._on_assign 

368 if self._on_revoke is not None: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true

369 subscribe_kwargs["on_revoke"] = self._on_revoke 

370 if self._on_lost is not None: 370 ↛ 371line 370 didn't jump to line 371 because the condition on line 370 was never true

371 subscribe_kwargs["on_lost"] = self._on_lost 

372 await run_in_executor( 

373 self._thread_pool, 

374 self.consumer.subscribe, 

375 **subscribe_kwargs, 

376 ) 

377 

378 elif self.partitions: 378 ↛ 386line 378 didn't jump to line 386 because the condition on line 378 was always true

379 await run_in_executor( 

380 self._thread_pool, 

381 self.consumer.assign, 

382 [p.to_confluent() for p in self.partitions], 

383 ) 

384 

385 else: 

386 msg = "You must provide either `topics` or `partitions` option." 

387 raise SetupError(msg) 

388 

389 async def commit(self, asynchronous: bool = True) -> None: 

390 """Commits the offsets of all messages returned by the last poll operation.""" 

391 await run_in_executor( 

392 self._thread_pool, 

393 lambda: self.consumer.commit(asynchronous=asynchronous), # type: ignore[call-overload] 

394 ) 

395 

396 async def stop(self) -> None: 

397 """Stops the Kafka consumer and releases all resources.""" 

398 # NOTE: If we don't explicitly call commit and then close the consumer, the confluent consumer gets stuck. 

399 # We are doing this to avoid the issue. 

400 enable_auto_commit = self.config.get("enable.auto.commit", True) 

401 

402 try: 

403 if enable_auto_commit: 

404 await self.commit(asynchronous=False) 

405 

406 except Exception as e: 

407 # No offset stored issue is not a problem - https://github.com/confluentinc/confluent-kafka-python/issues/295#issuecomment-355907183 

408 if "No offset stored" in str(e): 

409 pass 

410 else: 

411 self.logger_state.log( 

412 log_level=logging.ERROR, 

413 message="Consumer closing error occurred.", 

414 exc_info=e, 

415 ) 

416 

417 # Wrap calls to async to make method cancelable by timeout 

418 # We shouldn't read messages and close consumer concurrently 

419 # https://github.com/ag2ai/faststream/issues/1904#issuecomment-2506990895 

420 # Now it works without lock due `ThreadPoolExecutor(max_workers=1)` 

421 # that makes all calls to consumer sequential 

422 await run_in_executor(self._thread_pool, self.consumer.close) 

423 

424 self._thread_pool.shutdown(wait=False) 

425 

426 async def getone(self, timeout: float = 0.1) -> Message | None: 

427 """Consumes a single message from Kafka.""" 

428 msg = await run_in_executor(self._thread_pool, self.consumer.poll, timeout) 

429 return check_msg_error(msg) 

430 

431 async def getmany( 

432 self, 

433 timeout: float = 0.1, 

434 max_records: int | None = 10, 

435 ) -> tuple[Message, ...]: 

436 """Consumes a batch of messages from Kafka and groups them by topic and partition.""" 

437 raw_messages: list[Message | None] = await run_in_executor( 

438 self._thread_pool, 

439 cast( 

440 "Callable[..., list[Message | None]]", 

441 lambda: self.consumer.consume( 

442 num_messages=max_records or 10, 

443 timeout=timeout, 

444 ), 

445 ), 

446 ) 

447 return tuple(x for x in map(check_msg_error, raw_messages) if x is not None) 

448 

449 async def seek(self, topic: str, partition: int, offset: int) -> None: 

450 """Seeks to the specified offset in the specified topic and partition.""" 

451 topic_partition = TopicPartition( 

452 topic=topic, 

453 partition=partition, 

454 offset=offset, 

455 ) 

456 await run_in_executor( 

457 self._thread_pool, 

458 self.consumer.seek, 

459 topic_partition.to_confluent(), 

460 ) 

461 

462 async def store_offsets(self, *topic_partitions: TopicPartition) -> None: 

463 """Stores the offsets to commit for the specified topic partitions.""" 

464 if not topic_partitions: 

465 return 

466 

467 await run_in_executor( 

468 self._thread_pool, 

469 self.consumer.store_offsets, 

470 offsets=[ 

471 topic_partition.to_confluent() for topic_partition in topic_partitions 

472 ], 

473 ) 

474 

475 

476def check_msg_error(msg: Message | None) -> Message | None: 

477 """Checks for errors in the consumed message.""" 

478 if msg is None or msg.error(): 

479 return None 

480 

481 return msg 

482 

483 

484class BatchBuilder: 

485 """A helper class to build a batch of messages to send to Kafka.""" 

486 

487 __slots__ = ("_builder",) 

488 

489 def __init__(self) -> None: 

490 """Initializes a new BatchBuilder instance.""" 

491 self._builder: list[dict[str, Any]] = [] 

492 

493 def append( 

494 self, 

495 *, 

496 value: bytes | str | None = None, 

497 key: bytes | str | None = None, 

498 timestamp: int | None = None, 

499 headers: Sequence[tuple[str, bytes | None]] | None = None, 

500 ) -> None: 

501 """Appends a message to the batch with optional timestamp, key, value, and headers.""" 

502 if key is None and value is None: 502 ↛ 503line 502 didn't jump to line 503 because the condition on line 502 was never true

503 raise KafkaException( 

504 KafkaError(40, "Both key and value can't be None"), 

505 ) 

506 

507 self._builder.append( 

508 { 

509 "timestamp_ms": timestamp or round(time() * 1000), 

510 "key": key, 

511 "value": value, 

512 "headers": headers or [], 

513 }, 

514 )