Coverage for src / zmqtt / _internal / inbound.py: 90%
335 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-11-12 08:49 +0000
« prev ^ index » next coverage.py v7.13.4, created at 2026-11-12 08:49 +0000
1"""Inbound PUBLISH routing, acknowledgement, and persistent-session replay."""
3import asyncio
4import contextlib
5import logging
6from collections import deque
7from collections.abc import Awaitable, Callable
8from typing import Final, Protocol, TypeAlias, cast
10from zmqtt._internal.packets.codec import AnyPacket
11from zmqtt._internal.packets.publish import PubAck, PubComp, Publish, PubRec, PubRel
12from zmqtt._internal.routing import InboundRecipient, RequestRouter
13from zmqtt._internal.state import InboundQoS2Flight, InboundQoS2State, SessionState
14from zmqtt._internal.subscription_index import SubscriptionSelection
15from zmqtt._internal.types.message import Message
16from zmqtt._internal.types.qos import QoS
17from zmqtt.errors import MQTTProtocolError
19log = logging.getLogger("zmqtt.protocol")
21# MQTT 5.0 §3.3.4: DISCONNECT reason code for a server exceeding the client's Receive Maximum.
22_RECEIVE_MAXIMUM_EXCEEDED: Final = 0x93
23# MQTT 5.0 §3.7.2.1: PUBCOMP reason code for a PUBREL the receiver has no state for.
24_PACKET_IDENTIFIER_NOT_FOUND: Final = 0x92
27class InboundConnection(Protocol):
28 """Wire operations needed by the inbound publish flow."""
30 async def send_packet(self, packet: AnyPacket) -> None: ...
32 async def abort(self, reason_code: int | None = None) -> None: ...
35class _NoSessionReplay:
36 """Null Object used when the broker did not resume a session."""
38 async def accept(
39 self,
40 publish: Publish,
41 recipient: InboundRecipient,
42 ) -> tuple[InboundRecipient, "_ReplayState"]:
43 if recipient.request is None and recipient.subscription is None:
44 log.warning("No subscriber for topic %r", publish.topic)
45 return recipient, self
47 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState":
48 del delivery
49 return self
51 def contains_packet_id(self, packet_id: int) -> bool:
52 del packet_id
53 return False
55 def clear(self) -> "_ReplayState":
56 return self
59class _PersistentReplayIdle:
60 """Persistent session with no messages waiting for a local recipient."""
62 def __init__(
63 self,
64 *,
65 buffer_size: int,
66 timeout: float,
67 connection: InboundConnection,
68 owner: "InboundPublishFlow",
69 ) -> None:
70 self._buffer_size = buffer_size
71 self._timeout = timeout
72 self._connection = connection
73 self._owner = owner
75 async def accept(
76 self,
77 publish: Publish,
78 recipient: InboundRecipient,
79 ) -> tuple[InboundRecipient | None, "_ReplayState"]:
80 if recipient.request is not None or recipient.subscription is not None:
81 return recipient, self
83 buffered = _PersistentReplayBuffered(
84 buffer_size=self._buffer_size,
85 timeout=self._timeout,
86 connection=self._connection,
87 owner=self._owner,
88 idle=self,
89 )
90 return await buffered.accept(publish, recipient)
92 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState":
93 del delivery
94 return self
96 def contains_packet_id(self, packet_id: int) -> bool:
97 del packet_id
98 return False
100 def clear(self) -> "_ReplayState":
101 return _NO_SESSION_REPLAY
104class _PersistentReplayBuffered:
105 """Persistent session with unmatched messages awaiting local subscriptions."""
107 def __init__(
108 self,
109 *,
110 buffer_size: int,
111 timeout: float,
112 connection: InboundConnection,
113 owner: "InboundPublishFlow",
114 idle: _PersistentReplayIdle,
115 ) -> None:
116 self._buffer_size = buffer_size
117 self._timeout = timeout
118 self._connection = connection
119 self._owner = owner
120 self._idle = idle
121 self._publishes: deque[Publish] = deque()
122 self._packet_ids: set[int] = set()
123 self._topics: dict[str, int] = {}
124 self._message_count = 0
125 self._delivery_lock = asyncio.Lock()
126 self._expiration_task: asyncio.Task[None] | None = None
128 async def accept(
129 self,
130 publish: Publish,
131 recipient: InboundRecipient,
132 ) -> tuple[InboundRecipient | None, "_ReplayState"]:
133 if recipient.request is not None:
134 return recipient, self
135 if recipient.subscription is not None and publish.topic not in self._topics:
136 return recipient, self
137 if publish.packet_id is not None and publish.packet_id in self._packet_ids:
138 return None, self
139 if self._buffer_size and self._message_count >= self._buffer_size:
140 msg = f"Persistent-session replay buffer limit of {self._buffer_size} messages exceeded"
141 await self._connection.abort()
142 raise MQTTProtocolError(msg)
144 self._publishes.append(publish)
145 self._message_count += 1
146 self._topics[publish.topic] = self._topics.get(publish.topic, 0) + 1
147 if publish.packet_id is not None:
148 self._packet_ids.add(publish.packet_id)
149 if self._expiration_task is None:
150 self._expiration_task = asyncio.create_task(
151 self._expire_after_timeout(),
152 name="zmqtt-session-replay-expiration",
153 )
154 log.debug("Buffered persistent-session replay for topic %r", publish.topic)
155 return None, self
157 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState":
158 async with self._delivery_lock:
159 await self._deliver_pending(delivery)
160 if self._message_count:
161 return self
162 self._cancel_expiration()
163 return self._idle
165 async def expire(self, delivery: "InboundPublishFlow") -> "_ReplayState":
166 """Deliver newly matched messages, then drop the rest without ACK."""
167 async with self._delivery_lock:
168 await self._deliver_pending(delivery)
169 dropped = self._drop_pending()
170 if dropped:
171 log.warning(
172 "Dropped %d persistent-session replay messages after %.1f seconds without delivery",
173 dropped,
174 self._timeout,
175 )
176 return _NO_SESSION_REPLAY
177 return self._idle
179 def contains_packet_id(self, packet_id: int) -> bool:
180 return packet_id in self._packet_ids
182 def clear(self) -> "_ReplayState":
183 self._cancel_expiration()
184 self._publishes.clear()
185 self._packet_ids.clear()
186 self._topics.clear()
187 self._message_count = 0
188 return _NO_SESSION_REPLAY
190 async def _expire_after_timeout(self) -> None:
191 try:
192 await asyncio.sleep(self._timeout)
193 await self._owner.expire_replay(self)
194 except asyncio.CancelledError:
195 return
196 except Exception:
197 log.exception("Failed to expire persistent-session replay buffer")
198 with contextlib.suppress(Exception):
199 await self._connection.abort()
200 finally:
201 if self._expiration_task is asyncio.current_task():
202 self._expiration_task = None
204 def _cancel_expiration(self) -> None:
205 task = self._expiration_task
206 self._expiration_task = None
207 if task is not None and task is not asyncio.current_task():
208 task.cancel()
210 async def _deliver_pending(self, delivery: "InboundPublishFlow") -> None:
211 unmatched: deque[Publish] = deque()
212 pending: deque[Publish] = deque()
213 try:
214 while self._publishes:
215 pending = self._publishes
216 self._publishes = deque()
217 while pending:
218 publish = pending.popleft()
219 recipient = delivery.select_recipient(publish)
220 subscription = recipient.subscription
221 if recipient.request is None and subscription is None:
222 unmatched.append(publish)
223 continue
224 if subscription is not None and subscription[1].queue.full():
225 unmatched.append(publish)
226 continue
227 try:
228 await delivery.deliver_replay(publish, recipient)
229 except BaseException:
230 unmatched.append(publish)
231 raise
232 else:
233 self._discard(publish)
234 finally:
235 unmatched.extend(pending)
236 unmatched.extend(self._publishes)
237 self._publishes = unmatched
239 def _drop_pending(self) -> int:
240 dropped = self._message_count
241 self._publishes.clear()
242 self._packet_ids.clear()
243 self._topics.clear()
244 self._message_count = 0
245 return dropped
247 def _discard(self, publish: Publish) -> None:
248 self._message_count -= 1
249 topic_count = self._topics[publish.topic] - 1
250 if topic_count:
251 self._topics[publish.topic] = topic_count
252 else:
253 self._topics.pop(publish.topic)
254 if publish.packet_id is not None:
255 self._packet_ids.discard(publish.packet_id)
258_ReplayState: TypeAlias = _NoSessionReplay | _PersistentReplayIdle | _PersistentReplayBuffered
259_NO_SESSION_REPLAY: _ReplayState = _NoSessionReplay()
262class InboundPublishFlow:
263 """Own the complete lifecycle of incoming MQTT application messages.
265 The flow selects one application recipient, executes inbound QoS handshakes,
266 and holds unmatched messages replayed by a resumed persistent session until
267 a compatible local subscription becomes available. Connection lifecycle,
268 packet decoding, and outbound publishing remain MQTTProtocol concerns.
270 With *receive_maximum*, the broker may leave at most that many QoS 1/2
271 messages without PUBACK or PUBCOMP (MQTT 5.0 §4.9); exceeding it closes
272 the connection with DISCONNECT 0x93.
273 """
275 def __init__( # noqa: PLR0913
276 self,
277 *,
278 connection: InboundConnection,
279 state: SessionState,
280 request_router: RequestRouter | None,
281 session_replay_buffer_size: int,
282 session_replay_timeout: float,
283 receive_maximum: int | None = None,
284 ) -> None:
285 self._connection = connection
286 self._state = state
287 self._request_router = request_router
288 self._session_replay_buffer_size = session_replay_buffer_size
289 self._session_replay_timeout = session_replay_timeout
290 self._replay: _ReplayState = _NO_SESSION_REPLAY
291 self._receive_maximum: Final = receive_maximum
292 self._unacknowledged: set[int] = set()
294 def begin_session(self, *, session_present: bool) -> None:
295 """Set whether this connection resumed broker-side session state."""
296 self._replay.clear()
297 self._replay = (
298 _PersistentReplayIdle(
299 buffer_size=self._session_replay_buffer_size,
300 timeout=self._session_replay_timeout,
301 connection=self._connection,
302 owner=self,
303 )
304 if session_present
305 else _NO_SESSION_REPLAY
306 )
308 def clear(self) -> None:
309 """Discard connection-local replay state without acknowledging it."""
310 self._replay = self._replay.clear()
312 async def drain(self) -> None:
313 """Deliver held messages for which a recipient now has capacity."""
314 replay = self._replay
315 next_replay = await replay.drain(self)
316 if self._replay is replay:
317 self._replay = next_replay
319 async def expire_replay(self, replay: _PersistentReplayBuffered) -> None:
320 """Expire a buffered state only if it is still active."""
321 if self._replay is not replay:
322 return
323 next_replay = await replay.expire(self)
324 if self._replay is replay:
325 self._replay = next_replay
327 async def handle_publish(self, packet: Publish) -> None:
328 """Route one inbound PUBLISH through its QoS receive flow."""
329 if packet.qos is QoS.AT_MOST_ONCE:
330 recipient = await self._select_recipient_or_buffer(packet)
331 if recipient is not None:
332 await self._deliver(recipient, ack_callback=None)
333 return None
334 if packet.qos is QoS.AT_LEAST_ONCE:
335 return await self._handle_qos1_publish(packet)
336 if packet.qos is QoS.EXACTLY_ONCE:
337 return await self._handle_qos2_publish(packet)
338 return None
340 async def handle_pubrel(self, packet: PubRel) -> None:
341 """Complete an inbound QoS 2 exchange."""
342 flight = self._state.inflight_qos2_in.pop(packet.packet_id, None)
343 if flight is None:
344 log.warning("PUBREL for unknown packet_id %d, completing it", packet.packet_id)
345 await self._connection.send_packet(
346 PubComp(packet_id=packet.packet_id, reason_code=_PACKET_IDENTIFIER_NOT_FOUND),
347 )
348 return
349 await self._complete(PubComp(packet_id=packet.packet_id))
350 if not flight.delivered:
351 await self._deliver(flight.recipient, ack_callback=None)
353 def select_recipient(self, publish: Publish) -> InboundRecipient:
354 """Select the exclusive request or subscription recipient for a PUBLISH."""
355 message = self._make_message(publish)
356 if self._request_router is not None:
357 request = self._request_router.claim(message)
358 if request is not None:
359 return InboundRecipient(
360 message=message,
361 request=request,
362 )
364 selection = self._select_subscription(publish)
365 if selection.identifier_missing:
366 identifier = publish.properties.subscription_identifier if publish.properties else None
367 log.warning(
368 "No subscription with identifier %r for topic %r, falling back to filter matching",
369 identifier,
370 publish.topic,
371 )
372 if selection.tied_filters:
373 log.warning(
374 "Multiple equally-specific subscribers for %r: %s, delivering to first",
375 publish.topic,
376 list(selection.tied_filters),
377 )
378 return InboundRecipient(
379 message=message,
380 subscription=selection.recipient,
381 )
383 def _select_subscription(self, publish: Publish) -> SubscriptionSelection:
384 identifier = publish.properties.subscription_identifier if publish.properties else None
385 if identifier is None:
386 return self._state.subscriptions.select(topic=publish.topic)
387 return self._state.subscriptions.select_by_identifier(
388 topic=publish.topic,
389 identifier=identifier,
390 )
392 async def _select_recipient_or_buffer(self, publish: Publish) -> InboundRecipient | None:
393 replay = self._replay
394 selected = self.select_recipient(publish)
395 recipient, next_replay = await replay.accept(publish, selected)
396 if self._replay is replay:
397 self._replay = next_replay
398 return recipient
400 async def deliver_replay(
401 self,
402 publish: Publish,
403 recipient: InboundRecipient,
404 ) -> None:
405 """Deliver one message selected by the active replay state."""
406 if publish.qos is QoS.AT_MOST_ONCE:
407 await self._deliver(recipient, ack_callback=None)
408 elif publish.qos is QoS.AT_LEAST_ONCE:
409 await self._deliver_qos1_publish(publish, recipient)
410 elif publish.qos is QoS.EXACTLY_ONCE:
411 await self._accept_qos2_publish(publish, recipient)
413 async def _handle_qos1_publish(self, packet: Publish) -> None:
414 if packet.packet_id is None:
415 msg = "Cannot publish without packet id"
416 raise ValueError(msg)
417 await self._hold_receive_quota(packet.packet_id)
418 if self._replay.contains_packet_id(packet.packet_id):
419 return
420 recipient = await self._select_recipient_or_buffer(packet)
421 if recipient is None:
422 return
423 await self._deliver_qos1_publish(packet, recipient)
425 async def _deliver_qos1_publish(
426 self,
427 packet: Publish,
428 recipient: InboundRecipient,
429 ) -> None:
430 if packet.packet_id is None:
431 msg = "Cannot publish without packet id"
432 raise ValueError(msg)
433 if recipient.auto_ack:
434 if await self._deliver(recipient, ack_callback=None):
435 await self._complete(PubAck(packet_id=packet.packet_id))
436 else:
437 acked = False
439 async def _puback() -> None:
440 nonlocal acked
441 if acked:
442 return
443 acked = True
444 await self._complete(PubAck(packet_id=cast("int", packet.packet_id)))
446 await self._deliver(recipient, ack_callback=_puback)
448 async def _handle_qos2_publish(self, packet: Publish) -> None:
449 if packet.packet_id is None:
450 msg = "Cannot publish without packet id"
451 raise ValueError(msg)
452 await self._hold_receive_quota(packet.packet_id)
453 if packet.packet_id in self._state.inflight_qos2_in:
454 # PUBREC already sent — resend it (duplicate PUBLISH after PUBREC)
455 await self._connection.send_packet(PubRec(packet_id=packet.packet_id))
456 return
457 if packet.packet_id in self._state.pending_ack_qos2_in:
458 # Duplicate PUBLISH while app hasn't called msg.ack() yet — ignore
459 return
460 if self._replay.contains_packet_id(packet.packet_id):
461 return
462 recipient = await self._select_recipient_or_buffer(packet)
463 if recipient is None:
464 return
465 await self._accept_qos2_publish(packet, recipient)
467 async def _accept_qos2_publish(
468 self,
469 packet: Publish,
470 recipient: InboundRecipient,
471 ) -> None:
472 if packet.packet_id is None:
473 msg = "Cannot publish without packet id"
474 raise ValueError(msg)
475 if recipient.subscription is not None and recipient.subscription[1].detached:
476 return
477 if recipient.auto_ack:
478 self._state.inflight_qos2_in[packet.packet_id] = InboundQoS2Flight(
479 packet_id=packet.packet_id,
480 recipient=recipient,
481 state=InboundQoS2State.PENDING_PUBREL,
482 )
483 await self._connection.send_packet(PubRec(packet_id=packet.packet_id))
484 else:
485 self._state.pending_ack_qos2_in.add(packet.packet_id)
486 pid = packet.packet_id
488 async def _pubrec() -> None:
489 self._state.pending_ack_qos2_in.discard(pid)
490 self._state.inflight_qos2_in[pid] = InboundQoS2Flight(
491 packet_id=pid,
492 recipient=recipient,
493 state=InboundQoS2State.PENDING_PUBREL,
494 delivered=True,
495 )
496 await self._connection.send_packet(PubRec(packet_id=pid))
498 await self._deliver(recipient, ack_callback=_pubrec)
500 async def _hold_receive_quota(self, packet_id: int) -> None:
501 """Count a QoS 1/2 PUBLISH against Receive Maximum until PUBACK or PUBCOMP.
503 Replayed, buffered, and manually acknowledged messages keep their unit;
504 a duplicate packet ID reuses the unit it already holds.
505 """
506 self._unacknowledged.add(packet_id)
507 limit = self._receive_maximum
508 if limit is not None and len(self._unacknowledged) > limit:
509 await self._connection.abort(_RECEIVE_MAXIMUM_EXCEEDED)
510 msg = f"Broker exceeded the Receive Maximum of {limit} unacknowledged QoS 1/2 messages"
511 raise MQTTProtocolError(msg)
513 async def _complete(self, ack: PubAck | PubComp) -> None:
514 """Send the acknowledgement that ends an inbound QoS 1/2 flow."""
515 # Release first: the broker may answer this ack with a new PUBLISH that
516 # the read loop handles before a manual ack() caller resumes.
517 self._unacknowledged.discard(ack.packet_id)
518 await self._connection.send_packet(ack)
520 async def _deliver(
521 self,
522 recipient: InboundRecipient,
523 ack_callback: Callable[[], Awaitable[None]] | None,
524 ) -> bool:
525 if recipient.request is not None:
526 recipient.request.deliver()
527 return True
529 subscription = recipient.subscription
530 if subscription is None:
531 return True
533 filter_, entry = subscription
534 if entry.detached:
535 return False
536 message = recipient.message
537 if not entry.auto_ack and ack_callback is not None:
538 message._ack_callback = ack_callback # noqa: SLF001 - internal delivery contract
539 await entry.queue.put(message)
540 if entry.detached:
541 if not entry.queue.empty():
542 entry.queue.get_nowait()
543 return False
544 log.debug("Delivered message for topic %r to filter %r", message.topic, filter_)
545 return True
547 @staticmethod
548 def _make_message(publish: Publish) -> Message:
549 return Message(
550 topic=publish.topic,
551 payload=publish.payload,
552 qos=publish.qos,
553 retain=publish.retain,
554 properties=publish.properties,
555 )