Coverage for faststream/redis/subscriber/usecases/stream_subscriber.py: 93%
209 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 21:20 +0000
1import asyncio
2import logging
3import math
4from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
5from typing import TYPE_CHECKING, Any, NamedTuple, Optional, TypeAlias
7import anyio
8from redis.exceptions import ResponseError
9from typing_extensions import override
11from faststream._internal.endpoint.subscriber.mixins import ConcurrentMixin
12from faststream._internal.endpoint.utils import process_msg
13from faststream.redis.exceptions import (
14 StreamClaimUnsupportedError,
15 StreamGroupNotFoundError,
16)
17from faststream.redis.message import (
18 BatchStreamMessage,
19 DefaultStreamMessage,
20 RedisStreamMessage,
21)
22from faststream.redis.parser import (
23 RedisBatchStreamParser,
24 RedisStreamParser,
25)
27from .basic import CONSUME_ERROR_BACKOFF_SECONDS, LogicSubscriber
29if TYPE_CHECKING:
30 from anyio import Event
32 from faststream._internal.endpoint.subscriber import SubscriberSpecification
33 from faststream._internal.endpoint.subscriber.call_item import (
34 CallsCollection,
35 )
36 from faststream.message import StreamMessage as BrokerStreamMessage
37 from faststream.redis.message import _StreamMessage
38 from faststream.redis.parser.parsers import _StreamParser
39 from faststream.redis.schemas import StreamSub
40 from faststream.redis.subscriber.config import RedisSubscriberConfig
43TopicName: TypeAlias = bytes
44Offset: TypeAlias = bytes
46# With `StreamSub.claim_min_idle_time` (XREADGROUP CLAIM), every entry carries
47# two extra fields: idle time (ms) and previous-delivery count.
48StreamEntry: TypeAlias = (
49 tuple[Offset, dict[bytes, bytes]] | tuple[Offset, dict[bytes, bytes], int, int]
50)
52ReadResponse = tuple[
53 tuple[
54 TopicName,
55 tuple[StreamEntry, ...],
56 ],
57 ...,
58]
59ReadCallable = Callable[[str], Awaitable[ReadResponse]]
62class ClaimMeta(NamedTuple):
63 """Per-entry XREADGROUP CLAIM metadata."""
65 idle_time: int
66 """Milliseconds since the last delivery."""
68 delivery_count: int
69 """Previous deliveries: 0 for a new entry, XPENDING's `times_delivered` - 1."""
72class _StreamHandlerMixin(LogicSubscriber):
73 __slots__ = (
74 "_stream_sub",
75 "autoclaim_start_id",
76 "claim_min_idle_time",
77 "last_id",
78 "min_idle_time",
79 "parser",
80 "read_id",
81 )
83 parser: "_StreamParser"
85 def __init__(
86 self,
87 config: "RedisSubscriberConfig",
88 specification: "SubscriberSpecification[Any, Any]",
89 calls: "CallsCollection[Any]",
90 ) -> None:
91 super().__init__(config, specification, calls)
93 assert config.stream_sub
94 self._stream_sub = config.stream_sub
95 self.last_id = config.stream_sub.last_id
96 self.read_id = self.last_id
97 self.min_idle_time = config.stream_sub.min_idle_time
98 self.claim_min_idle_time = config.stream_sub.claim_min_idle_time
99 self.autoclaim_start_id = b"0-0"
101 @property
102 def stream_sub(self) -> "StreamSub":
103 return self._stream_sub.add_prefix(self._outer_config.prefix)
105 def get_log_context(
106 self,
107 message: Optional["BrokerStreamMessage[Any]"],
108 ) -> dict[str, str]:
109 return self.build_log_context(
110 message=message,
111 channel=self.stream_sub.name,
112 )
114 @override
115 async def _consume(self, *args: Any, start_signal: "Event") -> None:
116 if await self._client.ping(): 116 ↛ 119line 116 didn't jump to line 119 because the condition on line 116 was always true
117 start_signal.set()
119 while self.running:
120 try:
121 await self._get_msgs(*args)
123 except StreamClaimUnsupportedError as e: # ruff: ignore[try-except-in-loop]
124 await self._stop_on_terminal_error(e, str(e))
125 return
127 except ResponseError as e:
128 if "NOGROUP" in str(e):
129 msg = (
130 f"Consumer group `{self.stream_sub.group}` for stream "
131 f"`{self.stream_sub.name}` no longer exists. "
132 "The stream was likely deleted or flushed. "
133 "Stopping subscriber — restart the application to recreate the group."
134 )
135 await self._stop_on_terminal_error(StreamGroupNotFoundError(msg), msg)
136 return
138 raise
140 except Exception as e:
141 self._log(
142 log_level=logging.ERROR,
143 message="Message fetch error",
144 exc_info=e,
145 )
146 await anyio.sleep(CONSUME_ERROR_BACKOFF_SECONDS)
148 finally:
149 if not start_signal.is_set(): 149 ↛ 150line 149 didn't jump to line 150 because the condition on line 149 was never true
150 start_signal.set()
152 async def _stop_on_terminal_error(self, exc: Exception, msg: str) -> None:
153 # A terminal error: re-running the read cannot fix it, and raising it
154 # would make the task supervisor restart the read in a hot loop. Stop
155 # the subscriber for good and, if the broker runs inside an app, exit.
156 self._log(log_level=logging.CRITICAL, message=msg, exc_info=exc)
157 await self.stop()
158 if app := self._outer_config.fd_config.context.get("app"): 158 ↛ exitline 158 didn't return from function '_stop_on_terminal_error' because the condition on line 158 was always true
159 app.exit()
161 @override
162 async def start(self) -> None:
163 client = self._client
165 stream = self.stream_sub
167 self.parser._setup(client, stream.group)
169 read: ReadCallable
171 if stream.group and stream.consumer:
172 group_create_id = "$" if self.last_id == ">" else self.last_id
173 try:
174 await client.xgroup_create(
175 name=stream.name,
176 id=group_create_id,
177 groupname=stream.group,
178 mkstream=stream.declare,
179 )
180 except ResponseError as e:
181 if "already exists" not in str(e):
182 raise
183 else:
184 self.read_id = ">"
186 self.last_id = self.read_id
188 if stream.min_idle_time is None:
190 def read(_: str) -> Awaitable[ReadResponse]:
191 return self._xreadgroup(
192 count=stream.max_records,
193 block=stream.polling_interval,
194 noack=stream.no_ack,
195 )
197 else:
199 async def read(_: str) -> ReadResponse:
200 stream_message = await client.xautoclaim(
201 name=self.stream_sub.name,
202 groupname=self.stream_sub.group,
203 consumername=self.stream_sub.consumer,
204 min_idle_time=self.min_idle_time,
205 start_id=self.autoclaim_start_id,
206 count=1,
207 )
208 stream_name = self.stream_sub.name.encode()
209 (next_id, messages, *_) = stream_message
211 # Update start_id for next call
212 self.autoclaim_start_id = next_id
214 if next_id == b"0-0" and not messages:
215 await asyncio.sleep(stream.polling_interval / 1000) # ms to s
216 return ()
218 return ((stream_name, messages),)
220 else:
222 def read(
223 last_id: str,
224 ) -> Awaitable[ReadResponse]:
225 return client.xread(
226 {stream.name: last_id},
227 block=stream.polling_interval,
228 count=stream.max_records,
229 )
231 await super().start(read)
233 @override
234 async def get_one(
235 self,
236 *,
237 timeout: float = 5.0,
238 ) -> "RedisStreamMessage | None":
239 assert not self.calls, (
240 "You can't use `get_one` method if subscriber has registered handlers."
241 )
242 claim_meta: ClaimMeta | None = None
244 if self.stream_sub.group and self.stream_sub.consumer:
245 if self.min_idle_time is None:
246 stream_message = await self._xreadgroup(
247 count=1,
248 block=math.ceil(timeout * 1000),
249 noack=self.stream_sub.no_ack,
250 )
251 if not stream_message: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true
252 return None
254 ((stream_name, (entry,)),) = stream_message
255 message_id, raw_message, claim_meta = self._parse_stream_entry(entry)
256 else:
257 stream_message = await self._client.xautoclaim(
258 name=self.stream_sub.name,
259 groupname=self.stream_sub.group,
260 consumername=self.stream_sub.consumer,
261 min_idle_time=self.min_idle_time,
262 start_id=self.autoclaim_start_id,
263 count=1,
264 )
265 (next_id, messages, *_) = stream_message
266 # Update start_id for next call
267 self.autoclaim_start_id = next_id
268 if not messages:
269 return None
270 stream_name = self.stream_sub.name.encode()
271 ((message_id, raw_message),) = messages
272 else:
273 stream_message = await self._client.xread(
274 {self.stream_sub.name: self.last_id},
275 block=math.ceil(timeout * 1000),
276 count=1,
277 )
278 if not stream_message:
279 return None
281 ((stream_name, ((message_id, raw_message),)),) = stream_message
283 self.last_id = message_id.decode()
285 redis_incoming_msg = DefaultStreamMessage(
286 type="stream",
287 channel=stream_name.decode(),
288 message_ids=[message_id],
289 data=raw_message,
290 )
291 _attach_claim_metadata(redis_incoming_msg, [claim_meta])
293 context = self._outer_config.context
294 async_parser, async_decoder = self._get_parser_and_decoder()
296 msg: RedisStreamMessage = await process_msg( # type: ignore[assignment]
297 msg=redis_incoming_msg,
298 middlewares=(
299 m(redis_incoming_msg, context=context) for m in self._broker_middlewares
300 ),
301 parser=async_parser,
302 decoder=async_decoder,
303 )
304 return msg
306 @override
307 async def __aiter__(self) -> AsyncIterator["RedisStreamMessage"]:
308 assert not self.calls, (
309 "You can't use iterator if subscriber has registered handlers."
310 )
312 timeout = 5
314 context = self._outer_config.context
315 async_parser, async_decoder = self._get_parser_and_decoder()
317 claim_meta: ClaimMeta | None
319 while True:
320 claim_meta = None
322 if self.stream_sub.group and self.stream_sub.consumer:
323 if self.min_idle_time is None:
324 stream_message = await self._xreadgroup(
325 count=1,
326 block=math.ceil(timeout * 1000),
327 noack=self.stream_sub.no_ack,
328 )
329 if not stream_message: 329 ↛ 330line 329 didn't jump to line 330 because the condition on line 329 was never true
330 continue
332 ((stream_name, (entry,)),) = stream_message
333 message_id, raw_message, claim_meta = self._parse_stream_entry(
334 entry,
335 )
336 else:
337 stream_message = await self._client.xautoclaim(
338 name=self.stream_sub.name,
339 groupname=self.stream_sub.group,
340 consumername=self.stream_sub.consumer,
341 min_idle_time=self.min_idle_time,
342 start_id=self.autoclaim_start_id,
343 count=1,
344 )
345 (next_id, messages, *_) = stream_message
346 # Update start_id for next call
347 self.autoclaim_start_id = next_id
348 if not messages: 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true
349 continue
350 stream_name = self.stream_sub.name.encode()
351 ((message_id, raw_message),) = messages
352 else:
353 stream_message = await self._client.xread(
354 {self.stream_sub.name: self.last_id},
355 block=math.ceil(timeout * 1000),
356 count=1,
357 )
358 if not stream_message: 358 ↛ 359line 358 didn't jump to line 359 because the condition on line 358 was never true
359 continue
361 ((stream_name, ((message_id, raw_message),)),) = stream_message
363 self.last_id = message_id.decode()
365 redis_incoming_msg = DefaultStreamMessage(
366 type="stream",
367 channel=stream_name.decode(),
368 message_ids=[message_id],
369 data=raw_message,
370 )
371 _attach_claim_metadata(redis_incoming_msg, [claim_meta])
373 msg: RedisStreamMessage = await process_msg( # type: ignore[assignment]
374 msg=redis_incoming_msg,
375 middlewares=(
376 m(redis_incoming_msg, context=context)
377 for m in self._broker_middlewares
378 ),
379 parser=async_parser,
380 decoder=async_decoder,
381 )
382 yield msg
384 async def _xreadgroup(
385 self,
386 *,
387 count: int | None,
388 block: int | None,
389 noack: bool = False,
390 ) -> ReadResponse:
391 stream = self.stream_sub
393 response: ReadResponse
395 if self.claim_min_idle_time is None:
396 response = await self._client.xreadgroup(
397 groupname=stream.group,
398 consumername=stream.consumer,
399 streams={stream.name: self.read_id},
400 count=count,
401 block=block,
402 noack=noack,
403 )
404 return response
406 try:
407 # types-redis stubs predate `claim_min_idle_time`; StreamSub validation
408 # guarantees redis-py 7.1.0+ (which accepts it) whenever it is set.
409 response = await self._client.xreadgroup( # type: ignore[call-arg]
410 groupname=stream.group,
411 consumername=stream.consumer,
412 streams={stream.name: self.read_id},
413 count=count,
414 block=block,
415 noack=noack,
416 claim_min_idle_time=self.claim_min_idle_time,
417 )
418 except ResponseError as e:
419 if "syntax error" not in str(e).lower():
420 raise
422 msg = (
423 "Redis server rejected the XREADGROUP CLAIM option for "
424 f"stream `{stream.name}`. `claim_min_idle_time` "
425 "requires Redis server 8.4+."
426 )
427 raise StreamClaimUnsupportedError(msg) from e
429 return response
431 def _parse_stream_entry(
432 self,
433 entry: "StreamEntry",
434 ) -> tuple[bytes, dict[bytes, bytes], ClaimMeta | None]:
435 message_id, data, *claim_meta = entry
437 if self.claim_min_idle_time is None:
438 return message_id, data, None
440 if len(claim_meta) == 2: 440 ↛ 445line 440 didn't jump to line 445 because the condition on line 440 was always true
441 return message_id, data, ClaimMeta(*claim_meta)
443 # Redis ignores CLAIM for an explicit id, so a bare entry means the
444 # requested recovery is silently off; surface it instead of continuing.
445 msg = (
446 "Stream entry is missing XREADGROUP CLAIM metadata. Redis ignores "
447 "the CLAIM option when reading with an explicit id, so the "
448 "requested `claim_min_idle_time` behavior is disabled."
449 )
450 raise ValueError(msg)
453class StreamSubscriber(_StreamHandlerMixin):
454 __slots__ = ()
456 def __init__(
457 self,
458 config: "RedisSubscriberConfig",
459 specification: "SubscriberSpecification[Any, Any]",
460 calls: "CallsCollection[Any]",
461 ) -> None:
462 self.parser = RedisStreamParser(config)
463 config.decoder = self.parser.decode_message
464 config.parser = self.parser.parse_message
465 super().__init__(config, specification, calls)
467 async def _get_msgs(
468 self,
469 read: ReadCallable,
470 ) -> None:
471 for stream_name, msgs in await read(self.last_id):
472 if msgs: 472 ↛ 471line 472 didn't jump to line 471 because the condition on line 472 was always true
473 self.last_id = msgs[-1][0].decode()
475 for entry in msgs:
476 message_id, raw_msg, claim_meta = self._parse_stream_entry(entry)
478 msg = DefaultStreamMessage(
479 type="stream",
480 channel=stream_name.decode(),
481 message_ids=[message_id],
482 data=raw_msg,
483 )
484 _attach_claim_metadata(msg, [claim_meta])
486 await self.consume_one(msg)
489class StreamBatchSubscriber(_StreamHandlerMixin):
490 __slots__ = ()
492 def __init__(
493 self,
494 config: "RedisSubscriberConfig",
495 specification: "SubscriberSpecification[Any, Any]",
496 calls: "CallsCollection[Any]",
497 ) -> None:
498 self.parser = RedisBatchStreamParser(config)
499 config.decoder = self.parser.decode_message
500 config.parser = self.parser.parse_message
501 super().__init__(config, specification, calls)
503 async def _get_msgs(
504 self,
505 read: ReadCallable,
506 ) -> None:
507 for stream_name, msgs in await read(self.last_id):
508 if msgs: 508 ↛ 507line 508 didn't jump to line 507 because the condition on line 508 was always true
509 self.last_id = msgs[-1][0].decode()
511 data: list[dict[bytes, bytes]] = []
512 ids: list[bytes] = []
513 claim_metas: list[ClaimMeta | None] = []
514 for entry in msgs:
515 message_id, i, claim_meta = self._parse_stream_entry(entry)
516 data.append(i)
517 ids.append(message_id)
518 claim_metas.append(claim_meta)
520 msg = BatchStreamMessage(
521 type="bstream",
522 channel=stream_name.decode(),
523 data=data,
524 message_ids=ids,
525 )
526 _attach_claim_metadata(msg, claim_metas)
528 await self.consume_one(msg)
531class StreamConcurrentSubscriber(
532 ConcurrentMixin["BrokerStreamMessage[Any]"],
533 StreamSubscriber,
534):
535 __slots__ = ()
537 async def start(self) -> None:
538 await super().start()
539 self.start_consume_task()
541 async def consume_one(self, msg: "BrokerStreamMessage[Any]") -> None:
542 await self._put_msg(msg)
545def _attach_claim_metadata(
546 message: "_StreamMessage",
547 claim_metas: "Sequence[ClaimMeta | None]",
548) -> None:
549 # `_parse_stream_entry` yields metadata for every entry or for none,
550 # so plain subscribers leave the message untouched.
551 metas = [m for m in claim_metas if m is not None]
552 if not metas:
553 return
555 message["idle_times"] = [m.idle_time for m in metas]
556 message["delivery_counts"] = [m.delivery_count for m in metas]