Coverage for faststream/redis/subscriber/usecases/stream_subscriber.py: 93%

209 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 21:20 +0000

1import asyncio 

2import logging 

3import math 

4from collections.abc import AsyncIterator, Awaitable, Callable, Sequence 

5from typing import TYPE_CHECKING, Any, NamedTuple, Optional, TypeAlias 

6 

7import anyio 

8from redis.exceptions import ResponseError 

9from typing_extensions import override 

10 

11from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin 

12from faststream._internal.endpoint.utils import process_msg 

13from faststream.redis.exceptions import ( 

14 StreamClaimUnsupportedError, 

15 StreamGroupNotFoundError, 

16) 

17from faststream.redis.message import ( 

18 BatchStreamMessage, 

19 DefaultStreamMessage, 

20 RedisStreamMessage, 

21) 

22from faststream.redis.parser import ( 

23 RedisBatchStreamParser, 

24 RedisStreamParser, 

25) 

26 

27from .basic import CONSUME_ERROR_BACKOFF_SECONDS, LogicSubscriber 

28 

29if TYPE_CHECKING: 

30 from anyio import Event 

31 

32 from faststream._internal.endpoint.subscriber import SubscriberSpecification 

33 from faststream._internal.endpoint.subscriber.call_item import ( 

34 CallsCollection, 

35 ) 

36 from faststream.message import StreamMessage as BrokerStreamMessage 

37 from faststream.redis.message import _StreamMessage 

38 from faststream.redis.parser.parsers import _StreamParser 

39 from faststream.redis.schemas import StreamSub 

40 from faststream.redis.subscriber.config import RedisSubscriberConfig 

41 

42 

43TopicName: TypeAlias = bytes 

44Offset: TypeAlias = bytes 

45 

46# With `StreamSub.claim_min_idle_time` (XREADGROUP CLAIM), every entry carries 

47# two extra fields: idle time (ms) and previous-delivery count. 

48StreamEntry: TypeAlias = ( 

49 tuple[Offset, dict[bytes, bytes]] | tuple[Offset, dict[bytes, bytes], int, int] 

50) 

51 

52ReadResponse = tuple[ 

53 tuple[ 

54 TopicName, 

55 tuple[StreamEntry, ...], 

56 ], 

57 ..., 

58] 

59ReadCallable = Callable[[str], Awaitable[ReadResponse]] 

60 

61 

62class ClaimMeta(NamedTuple): 

63 """Per-entry XREADGROUP CLAIM metadata.""" 

64 

65 idle_time: int 

66 """Milliseconds since the last delivery.""" 

67 

68 delivery_count: int 

69 """Previous deliveries: 0 for a new entry, XPENDING's `times_delivered` - 1.""" 

70 

71 

72class _StreamHandlerMixin(LogicSubscriber): 

73 __slots__ = ( 

74 "_stream_sub", 

75 "autoclaim_start_id", 

76 "claim_min_idle_time", 

77 "last_id", 

78 "min_idle_time", 

79 "parser", 

80 "read_id", 

81 ) 

82 

83 parser: "_StreamParser" 

84 

85 def __init__( 

86 self, 

87 config: "RedisSubscriberConfig", 

88 specification: "SubscriberSpecification[Any, Any]", 

89 calls: "CallsCollection[Any]", 

90 ) -> None: 

91 super().__init__(config, specification, calls) 

92 

93 assert config.stream_sub 

94 self._stream_sub = config.stream_sub 

95 self.last_id = config.stream_sub.last_id 

96 self.read_id = self.last_id 

97 self.min_idle_time = config.stream_sub.min_idle_time 

98 self.claim_min_idle_time = config.stream_sub.claim_min_idle_time 

99 self.autoclaim_start_id = b"0-0" 

100 

101 @property 

102 def stream_sub(self) -> "StreamSub": 

103 return self._stream_sub.add_prefix(self._outer_config.prefix) 

104 

105 def get_log_context( 

106 self, 

107 message: Optional["BrokerStreamMessage[Any]"], 

108 ) -> dict[str, str]: 

109 return self.build_log_context( 

110 message=message, 

111 channel=self.stream_sub.name, 

112 ) 

113 

114 @override 

115 async def _consume(self, *args: Any, start_signal: "Event") -> None: 

116 if await self._client.ping(): 116 ↛ 119line 116 didn't jump to line 119 because the condition on line 116 was always true

117 start_signal.set() 

118 

119 while self.running: 

120 try: 

121 await self._get_msgs(*args) 

122 

123 except StreamClaimUnsupportedError as e: # ruff: ignore[try-except-in-loop] 

124 await self._stop_on_terminal_error(e, str(e)) 

125 return 

126 

127 except ResponseError as e: 

128 if "NOGROUP" in str(e): 

129 msg = ( 

130 f"Consumer group `{self.stream_sub.group}` for stream " 

131 f"`{self.stream_sub.name}` no longer exists. " 

132 "The stream was likely deleted or flushed. " 

133 "Stopping subscriber — restart the application to recreate the group." 

134 ) 

135 await self._stop_on_terminal_error(StreamGroupNotFoundError(msg), msg) 

136 return 

137 

138 raise 

139 

140 except Exception as e: 

141 self._log( 

142 log_level=logging.ERROR, 

143 message="Message fetch error", 

144 exc_info=e, 

145 ) 

146 await anyio.sleep(CONSUME_ERROR_BACKOFF_SECONDS) 

147 

148 finally: 

149 if not start_signal.is_set(): 149 ↛ 150line 149 didn't jump to line 150 because the condition on line 149 was never true

150 start_signal.set() 

151 

152 async def _stop_on_terminal_error(self, exc: Exception, msg: str) -> None: 

153 # A terminal error: re-running the read cannot fix it, and raising it 

154 # would make the task supervisor restart the read in a hot loop. Stop 

155 # the subscriber for good and, if the broker runs inside an app, exit. 

156 self._log(log_level=logging.CRITICAL, message=msg, exc_info=exc) 

157 await self.stop() 

158 if app := self._outer_config.fd_config.context.get("app"): 158 ↛ exitline 158 didn't return from function '_stop_on_terminal_error' because the condition on line 158 was always true

159 app.exit() 

160 

161 @override 

162 async def start(self) -> None: 

163 client = self._client 

164 

165 stream = self.stream_sub 

166 

167 self.parser._setup(client, stream.group) 

168 

169 read: ReadCallable 

170 

171 if stream.group and stream.consumer: 

172 group_create_id = "$" if self.last_id == ">" else self.last_id 

173 try: 

174 await client.xgroup_create( 

175 name=stream.name, 

176 id=group_create_id, 

177 groupname=stream.group, 

178 mkstream=stream.declare, 

179 ) 

180 except ResponseError as e: 

181 if "already exists" not in str(e): 

182 raise 

183 else: 

184 self.read_id = ">" 

185 

186 self.last_id = self.read_id 

187 

188 if stream.min_idle_time is None: 

189 

190 def read(_: str) -> Awaitable[ReadResponse]: 

191 return self._xreadgroup( 

192 count=stream.max_records, 

193 block=stream.polling_interval, 

194 noack=stream.no_ack, 

195 ) 

196 

197 else: 

198 

199 async def read(_: str) -> ReadResponse: 

200 stream_message = await client.xautoclaim( 

201 name=self.stream_sub.name, 

202 groupname=self.stream_sub.group, 

203 consumername=self.stream_sub.consumer, 

204 min_idle_time=self.min_idle_time, 

205 start_id=self.autoclaim_start_id, 

206 count=1, 

207 ) 

208 stream_name = self.stream_sub.name.encode() 

209 (next_id, messages, *_) = stream_message 

210 

211 # Update start_id for next call 

212 self.autoclaim_start_id = next_id 

213 

214 if next_id == b"0-0" and not messages: 

215 await asyncio.sleep(stream.polling_interval / 1000) # ms to s 

216 return () 

217 

218 return ((stream_name, messages),) 

219 

220 else: 

221 

222 def read( 

223 last_id: str, 

224 ) -> Awaitable[ReadResponse]: 

225 return client.xread( 

226 {stream.name: last_id}, 

227 block=stream.polling_interval, 

228 count=stream.max_records, 

229 ) 

230 

231 await super().start(read) 

232 

233 @override 

234 async def get_one( 

235 self, 

236 *, 

237 timeout: float = 5.0, 

238 ) -> "RedisStreamMessage | None": 

239 assert not self.calls, ( 

240 "You can't use `get_one` method if subscriber has registered handlers." 

241 ) 

242 claim_meta: ClaimMeta | None = None 

243 

244 if self.stream_sub.group and self.stream_sub.consumer: 

245 if self.min_idle_time is None: 

246 stream_message = await self._xreadgroup( 

247 count=1, 

248 block=math.ceil(timeout * 1000), 

249 noack=self.stream_sub.no_ack, 

250 ) 

251 if not stream_message: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true

252 return None 

253 

254 ((stream_name, (entry,)),) = stream_message 

255 message_id, raw_message, claim_meta = self._parse_stream_entry(entry) 

256 else: 

257 stream_message = await self._client.xautoclaim( 

258 name=self.stream_sub.name, 

259 groupname=self.stream_sub.group, 

260 consumername=self.stream_sub.consumer, 

261 min_idle_time=self.min_idle_time, 

262 start_id=self.autoclaim_start_id, 

263 count=1, 

264 ) 

265 (next_id, messages, *_) = stream_message 

266 # Update start_id for next call 

267 self.autoclaim_start_id = next_id 

268 if not messages: 

269 return None 

270 stream_name = self.stream_sub.name.encode() 

271 ((message_id, raw_message),) = messages 

272 else: 

273 stream_message = await self._client.xread( 

274 {self.stream_sub.name: self.last_id}, 

275 block=math.ceil(timeout * 1000), 

276 count=1, 

277 ) 

278 if not stream_message: 

279 return None 

280 

281 ((stream_name, ((message_id, raw_message),)),) = stream_message 

282 

283 self.last_id = message_id.decode() 

284 

285 redis_incoming_msg = DefaultStreamMessage( 

286 type="stream", 

287 channel=stream_name.decode(), 

288 message_ids=[message_id], 

289 data=raw_message, 

290 ) 

291 _attach_claim_metadata(redis_incoming_msg, [claim_meta]) 

292 

293 context = self._outer_config.context 

294 async_parser, async_decoder = self._get_parser_and_decoder() 

295 

296 msg: RedisStreamMessage = await process_msg( # type: ignore[assignment] 

297 msg=redis_incoming_msg, 

298 middlewares=( 

299 m(redis_incoming_msg, context=context) for m in self._broker_middlewares 

300 ), 

301 parser=async_parser, 

302 decoder=async_decoder, 

303 ) 

304 return msg 

305 

306 @override 

307 async def __aiter__(self) -> AsyncIterator["RedisStreamMessage"]: 

308 assert not self.calls, ( 

309 "You can't use iterator if subscriber has registered handlers." 

310 ) 

311 

312 timeout = 5 

313 

314 context = self._outer_config.context 

315 async_parser, async_decoder = self._get_parser_and_decoder() 

316 

317 claim_meta: ClaimMeta | None 

318 

319 while True: 

320 claim_meta = None 

321 

322 if self.stream_sub.group and self.stream_sub.consumer: 

323 if self.min_idle_time is None: 

324 stream_message = await self._xreadgroup( 

325 count=1, 

326 block=math.ceil(timeout * 1000), 

327 noack=self.stream_sub.no_ack, 

328 ) 

329 if not stream_message: 329 ↛ 330line 329 didn't jump to line 330 because the condition on line 329 was never true

330 continue 

331 

332 ((stream_name, (entry,)),) = stream_message 

333 message_id, raw_message, claim_meta = self._parse_stream_entry( 

334 entry, 

335 ) 

336 else: 

337 stream_message = await self._client.xautoclaim( 

338 name=self.stream_sub.name, 

339 groupname=self.stream_sub.group, 

340 consumername=self.stream_sub.consumer, 

341 min_idle_time=self.min_idle_time, 

342 start_id=self.autoclaim_start_id, 

343 count=1, 

344 ) 

345 (next_id, messages, *_) = stream_message 

346 # Update start_id for next call 

347 self.autoclaim_start_id = next_id 

348 if not messages: 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true

349 continue 

350 stream_name = self.stream_sub.name.encode() 

351 ((message_id, raw_message),) = messages 

352 else: 

353 stream_message = await self._client.xread( 

354 {self.stream_sub.name: self.last_id}, 

355 block=math.ceil(timeout * 1000), 

356 count=1, 

357 ) 

358 if not stream_message: 358 ↛ 359line 358 didn't jump to line 359 because the condition on line 358 was never true

359 continue 

360 

361 ((stream_name, ((message_id, raw_message),)),) = stream_message 

362 

363 self.last_id = message_id.decode() 

364 

365 redis_incoming_msg = DefaultStreamMessage( 

366 type="stream", 

367 channel=stream_name.decode(), 

368 message_ids=[message_id], 

369 data=raw_message, 

370 ) 

371 _attach_claim_metadata(redis_incoming_msg, [claim_meta]) 

372 

373 msg: RedisStreamMessage = await process_msg( # type: ignore[assignment] 

374 msg=redis_incoming_msg, 

375 middlewares=( 

376 m(redis_incoming_msg, context=context) 

377 for m in self._broker_middlewares 

378 ), 

379 parser=async_parser, 

380 decoder=async_decoder, 

381 ) 

382 yield msg 

383 

384 async def _xreadgroup( 

385 self, 

386 *, 

387 count: int | None, 

388 block: int | None, 

389 noack: bool = False, 

390 ) -> ReadResponse: 

391 stream = self.stream_sub 

392 

393 response: ReadResponse 

394 

395 if self.claim_min_idle_time is None: 

396 response = await self._client.xreadgroup( 

397 groupname=stream.group, 

398 consumername=stream.consumer, 

399 streams={stream.name: self.read_id}, 

400 count=count, 

401 block=block, 

402 noack=noack, 

403 ) 

404 return response 

405 

406 try: 

407 # types-redis stubs predate `claim_min_idle_time`; StreamSub validation 

408 # guarantees redis-py 7.1.0+ (which accepts it) whenever it is set. 

409 response = await self._client.xreadgroup( # type: ignore[call-arg] 

410 groupname=stream.group, 

411 consumername=stream.consumer, 

412 streams={stream.name: self.read_id}, 

413 count=count, 

414 block=block, 

415 noack=noack, 

416 claim_min_idle_time=self.claim_min_idle_time, 

417 ) 

418 except ResponseError as e: 

419 if "syntax error" not in str(e).lower(): 

420 raise 

421 

422 msg = ( 

423 "Redis server rejected the XREADGROUP CLAIM option for " 

424 f"stream `{stream.name}`. `claim_min_idle_time` " 

425 "requires Redis server 8.4+." 

426 ) 

427 raise StreamClaimUnsupportedError(msg) from e 

428 

429 return response 

430 

431 def _parse_stream_entry( 

432 self, 

433 entry: "StreamEntry", 

434 ) -> tuple[bytes, dict[bytes, bytes], ClaimMeta | None]: 

435 message_id, data, *claim_meta = entry 

436 

437 if self.claim_min_idle_time is None: 

438 return message_id, data, None 

439 

440 if len(claim_meta) == 2: 440 ↛ 445line 440 didn't jump to line 445 because the condition on line 440 was always true

441 return message_id, data, ClaimMeta(*claim_meta) 

442 

443 # Redis ignores CLAIM for an explicit id, so a bare entry means the 

444 # requested recovery is silently off; surface it instead of continuing. 

445 msg = ( 

446 "Stream entry is missing XREADGROUP CLAIM metadata. Redis ignores " 

447 "the CLAIM option when reading with an explicit id, so the " 

448 "requested `claim_min_idle_time` behavior is disabled." 

449 ) 

450 raise ValueError(msg) 

451 

452 

453class StreamSubscriber(_StreamHandlerMixin): 

454 __slots__ = () 

455 

456 def __init__( 

457 self, 

458 config: "RedisSubscriberConfig", 

459 specification: "SubscriberSpecification[Any, Any]", 

460 calls: "CallsCollection[Any]", 

461 ) -> None: 

462 self.parser = RedisStreamParser(config) 

463 config.decoder = self.parser.decode_message 

464 config.parser = self.parser.parse_message 

465 super().__init__(config, specification, calls) 

466 

467 async def _get_msgs( 

468 self, 

469 read: ReadCallable, 

470 ) -> None: 

471 for stream_name, msgs in await read(self.last_id): 

472 if msgs: 472 ↛ 471line 472 didn't jump to line 471 because the condition on line 472 was always true

473 self.last_id = msgs[-1][0].decode() 

474 

475 for entry in msgs: 

476 message_id, raw_msg, claim_meta = self._parse_stream_entry(entry) 

477 

478 msg = DefaultStreamMessage( 

479 type="stream", 

480 channel=stream_name.decode(), 

481 message_ids=[message_id], 

482 data=raw_msg, 

483 ) 

484 _attach_claim_metadata(msg, [claim_meta]) 

485 

486 await self.consume_one(msg) 

487 

488 

489class StreamBatchSubscriber(_StreamHandlerMixin): 

490 __slots__ = () 

491 

492 def __init__( 

493 self, 

494 config: "RedisSubscriberConfig", 

495 specification: "SubscriberSpecification[Any, Any]", 

496 calls: "CallsCollection[Any]", 

497 ) -> None: 

498 self.parser = RedisBatchStreamParser(config) 

499 config.decoder = self.parser.decode_message 

500 config.parser = self.parser.parse_message 

501 super().__init__(config, specification, calls) 

502 

503 async def _get_msgs( 

504 self, 

505 read: ReadCallable, 

506 ) -> None: 

507 for stream_name, msgs in await read(self.last_id): 

508 if msgs: 508 ↛ 507line 508 didn't jump to line 507 because the condition on line 508 was always true

509 self.last_id = msgs[-1][0].decode() 

510 

511 data: list[dict[bytes, bytes]] = [] 

512 ids: list[bytes] = [] 

513 claim_metas: list[ClaimMeta | None] = [] 

514 for entry in msgs: 

515 message_id, i, claim_meta = self._parse_stream_entry(entry) 

516 data.append(i) 

517 ids.append(message_id) 

518 claim_metas.append(claim_meta) 

519 

520 msg = BatchStreamMessage( 

521 type="bstream", 

522 channel=stream_name.decode(), 

523 data=data, 

524 message_ids=ids, 

525 ) 

526 _attach_claim_metadata(msg, claim_metas) 

527 

528 await self.consume_one(msg) 

529 

530 

531class StreamConcurrentSubscriber( 

532 ConcurrentMixin["BrokerStreamMessage[Any]"], 

533 StreamSubscriber, 

534): 

535 __slots__ = () 

536 

537 async def start(self) -> None: 

538 await super().start() 

539 self.start_consume_task() 

540 

541 async def consume_one(self, msg: "BrokerStreamMessage[Any]") -> None: 

542 await self._put_msg(msg) 

543 

544 

545def _attach_claim_metadata( 

546 message: "_StreamMessage", 

547 claim_metas: "Sequence[ClaimMeta | None]", 

548) -> None: 

549 # `_parse_stream_entry` yields metadata for every entry or for none, 

550 # so plain subscribers leave the message untouched. 

551 metas = [m for m in claim_metas if m is not None] 

552 if not metas: 

553 return 

554 

555 message["idle_times"] = [m.idle_time for m in metas] 

556 message["delivery_counts"] = [m.delivery_count for m in metas]