Coverage for src / zmqtt / _internal / inbound.py: 90%

335 statements  

« prev     ^ index     » next       coverage.py v7.13.4, created at 2026-11-12 08:49 +0000

1"""Inbound PUBLISH routing, acknowledgement, and persistent-session replay.""" 

2 

3import asyncio 

4import contextlib 

5import logging 

6from collections import deque 

7from collections.abc import Awaitable, Callable 

8from typing import Final, Protocol, TypeAlias, cast 

9 

10from zmqtt._internal.packets.codec import AnyPacket 

11from zmqtt._internal.packets.publish import PubAck, PubComp, Publish, PubRec, PubRel 

12from zmqtt._internal.routing import InboundRecipient, RequestRouter 

13from zmqtt._internal.state import InboundQoS2Flight, InboundQoS2State, SessionState 

14from zmqtt._internal.subscription_index import SubscriptionSelection 

15from zmqtt._internal.types.message import Message 

16from zmqtt._internal.types.qos import QoS 

17from zmqtt.errors import MQTTProtocolError 

18 

19log = logging.getLogger("zmqtt.protocol") 

20 

21# MQTT 5.0 §3.3.4: DISCONNECT reason code for a server exceeding the client's Receive Maximum. 

22_RECEIVE_MAXIMUM_EXCEEDED: Final = 0x93 

23# MQTT 5.0 §3.7.2.1: PUBCOMP reason code for a PUBREL the receiver has no state for. 

24_PACKET_IDENTIFIER_NOT_FOUND: Final = 0x92 

25 

26 

27class InboundConnection(Protocol): 

28 """Wire operations needed by the inbound publish flow.""" 

29 

30 async def send_packet(self, packet: AnyPacket) -> None: ... 

31 

32 async def abort(self, reason_code: int | None = None) -> None: ... 

33 

34 

35class _NoSessionReplay: 

36 """Null Object used when the broker did not resume a session.""" 

37 

38 async def accept( 

39 self, 

40 publish: Publish, 

41 recipient: InboundRecipient, 

42 ) -> tuple[InboundRecipient, "_ReplayState"]: 

43 if recipient.request is None and recipient.subscription is None: 

44 log.warning("No subscriber for topic %r", publish.topic) 

45 return recipient, self 

46 

47 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState": 

48 del delivery 

49 return self 

50 

51 def contains_packet_id(self, packet_id: int) -> bool: 

52 del packet_id 

53 return False 

54 

55 def clear(self) -> "_ReplayState": 

56 return self 

57 

58 

59class _PersistentReplayIdle: 

60 """Persistent session with no messages waiting for a local recipient.""" 

61 

62 def __init__( 

63 self, 

64 *, 

65 buffer_size: int, 

66 timeout: float, 

67 connection: InboundConnection, 

68 owner: "InboundPublishFlow", 

69 ) -> None: 

70 self._buffer_size = buffer_size 

71 self._timeout = timeout 

72 self._connection = connection 

73 self._owner = owner 

74 

75 async def accept( 

76 self, 

77 publish: Publish, 

78 recipient: InboundRecipient, 

79 ) -> tuple[InboundRecipient | None, "_ReplayState"]: 

80 if recipient.request is not None or recipient.subscription is not None: 

81 return recipient, self 

82 

83 buffered = _PersistentReplayBuffered( 

84 buffer_size=self._buffer_size, 

85 timeout=self._timeout, 

86 connection=self._connection, 

87 owner=self._owner, 

88 idle=self, 

89 ) 

90 return await buffered.accept(publish, recipient) 

91 

92 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState": 

93 del delivery 

94 return self 

95 

96 def contains_packet_id(self, packet_id: int) -> bool: 

97 del packet_id 

98 return False 

99 

100 def clear(self) -> "_ReplayState": 

101 return _NO_SESSION_REPLAY 

102 

103 

104class _PersistentReplayBuffered: 

105 """Persistent session with unmatched messages awaiting local subscriptions.""" 

106 

107 def __init__( 

108 self, 

109 *, 

110 buffer_size: int, 

111 timeout: float, 

112 connection: InboundConnection, 

113 owner: "InboundPublishFlow", 

114 idle: _PersistentReplayIdle, 

115 ) -> None: 

116 self._buffer_size = buffer_size 

117 self._timeout = timeout 

118 self._connection = connection 

119 self._owner = owner 

120 self._idle = idle 

121 self._publishes: deque[Publish] = deque() 

122 self._packet_ids: set[int] = set() 

123 self._topics: dict[str, int] = {} 

124 self._message_count = 0 

125 self._delivery_lock = asyncio.Lock() 

126 self._expiration_task: asyncio.Task[None] | None = None 

127 

128 async def accept( 

129 self, 

130 publish: Publish, 

131 recipient: InboundRecipient, 

132 ) -> tuple[InboundRecipient | None, "_ReplayState"]: 

133 if recipient.request is not None: 

134 return recipient, self 

135 if recipient.subscription is not None and publish.topic not in self._topics: 

136 return recipient, self 

137 if publish.packet_id is not None and publish.packet_id in self._packet_ids: 

138 return None, self 

139 if self._buffer_size and self._message_count >= self._buffer_size: 

140 msg = f"Persistent-session replay buffer limit of {self._buffer_size} messages exceeded" 

141 await self._connection.abort() 

142 raise MQTTProtocolError(msg) 

143 

144 self._publishes.append(publish) 

145 self._message_count += 1 

146 self._topics[publish.topic] = self._topics.get(publish.topic, 0) + 1 

147 if publish.packet_id is not None: 

148 self._packet_ids.add(publish.packet_id) 

149 if self._expiration_task is None: 

150 self._expiration_task = asyncio.create_task( 

151 self._expire_after_timeout(), 

152 name="zmqtt-session-replay-expiration", 

153 ) 

154 log.debug("Buffered persistent-session replay for topic %r", publish.topic) 

155 return None, self 

156 

157 async def drain(self, delivery: "InboundPublishFlow") -> "_ReplayState": 

158 async with self._delivery_lock: 

159 await self._deliver_pending(delivery) 

160 if self._message_count: 

161 return self 

162 self._cancel_expiration() 

163 return self._idle 

164 

165 async def expire(self, delivery: "InboundPublishFlow") -> "_ReplayState": 

166 """Deliver newly matched messages, then drop the rest without ACK.""" 

167 async with self._delivery_lock: 

168 await self._deliver_pending(delivery) 

169 dropped = self._drop_pending() 

170 if dropped: 

171 log.warning( 

172 "Dropped %d persistent-session replay messages after %.1f seconds without delivery", 

173 dropped, 

174 self._timeout, 

175 ) 

176 return _NO_SESSION_REPLAY 

177 return self._idle 

178 

179 def contains_packet_id(self, packet_id: int) -> bool: 

180 return packet_id in self._packet_ids 

181 

182 def clear(self) -> "_ReplayState": 

183 self._cancel_expiration() 

184 self._publishes.clear() 

185 self._packet_ids.clear() 

186 self._topics.clear() 

187 self._message_count = 0 

188 return _NO_SESSION_REPLAY 

189 

190 async def _expire_after_timeout(self) -> None: 

191 try: 

192 await asyncio.sleep(self._timeout) 

193 await self._owner.expire_replay(self) 

194 except asyncio.CancelledError: 

195 return 

196 except Exception: 

197 log.exception("Failed to expire persistent-session replay buffer") 

198 with contextlib.suppress(Exception): 

199 await self._connection.abort() 

200 finally: 

201 if self._expiration_task is asyncio.current_task(): 

202 self._expiration_task = None 

203 

204 def _cancel_expiration(self) -> None: 

205 task = self._expiration_task 

206 self._expiration_task = None 

207 if task is not None and task is not asyncio.current_task(): 

208 task.cancel() 

209 

210 async def _deliver_pending(self, delivery: "InboundPublishFlow") -> None: 

211 unmatched: deque[Publish] = deque() 

212 pending: deque[Publish] = deque() 

213 try: 

214 while self._publishes: 

215 pending = self._publishes 

216 self._publishes = deque() 

217 while pending: 

218 publish = pending.popleft() 

219 recipient = delivery.select_recipient(publish) 

220 subscription = recipient.subscription 

221 if recipient.request is None and subscription is None: 

222 unmatched.append(publish) 

223 continue 

224 if subscription is not None and subscription[1].queue.full(): 

225 unmatched.append(publish) 

226 continue 

227 try: 

228 await delivery.deliver_replay(publish, recipient) 

229 except BaseException: 

230 unmatched.append(publish) 

231 raise 

232 else: 

233 self._discard(publish) 

234 finally: 

235 unmatched.extend(pending) 

236 unmatched.extend(self._publishes) 

237 self._publishes = unmatched 

238 

239 def _drop_pending(self) -> int: 

240 dropped = self._message_count 

241 self._publishes.clear() 

242 self._packet_ids.clear() 

243 self._topics.clear() 

244 self._message_count = 0 

245 return dropped 

246 

247 def _discard(self, publish: Publish) -> None: 

248 self._message_count -= 1 

249 topic_count = self._topics[publish.topic] - 1 

250 if topic_count: 

251 self._topics[publish.topic] = topic_count 

252 else: 

253 self._topics.pop(publish.topic) 

254 if publish.packet_id is not None: 

255 self._packet_ids.discard(publish.packet_id) 

256 

257 

258_ReplayState: TypeAlias = _NoSessionReplay | _PersistentReplayIdle | _PersistentReplayBuffered 

259_NO_SESSION_REPLAY: _ReplayState = _NoSessionReplay() 

260 

261 

262class InboundPublishFlow: 

263 """Own the complete lifecycle of incoming MQTT application messages. 

264 

265 The flow selects one application recipient, executes inbound QoS handshakes, 

266 and holds unmatched messages replayed by a resumed persistent session until 

267 a compatible local subscription becomes available. Connection lifecycle, 

268 packet decoding, and outbound publishing remain MQTTProtocol concerns. 

269 

270 With *receive_maximum*, the broker may leave at most that many QoS 1/2 

271 messages without PUBACK or PUBCOMP (MQTT 5.0 §4.9); exceeding it closes 

272 the connection with DISCONNECT 0x93. 

273 """ 

274 

275 def __init__( # noqa: PLR0913 

276 self, 

277 *, 

278 connection: InboundConnection, 

279 state: SessionState, 

280 request_router: RequestRouter | None, 

281 session_replay_buffer_size: int, 

282 session_replay_timeout: float, 

283 receive_maximum: int | None = None, 

284 ) -> None: 

285 self._connection = connection 

286 self._state = state 

287 self._request_router = request_router 

288 self._session_replay_buffer_size = session_replay_buffer_size 

289 self._session_replay_timeout = session_replay_timeout 

290 self._replay: _ReplayState = _NO_SESSION_REPLAY 

291 self._receive_maximum: Final = receive_maximum 

292 self._unacknowledged: set[int] = set() 

293 

294 def begin_session(self, *, session_present: bool) -> None: 

295 """Set whether this connection resumed broker-side session state.""" 

296 self._replay.clear() 

297 self._replay = ( 

298 _PersistentReplayIdle( 

299 buffer_size=self._session_replay_buffer_size, 

300 timeout=self._session_replay_timeout, 

301 connection=self._connection, 

302 owner=self, 

303 ) 

304 if session_present 

305 else _NO_SESSION_REPLAY 

306 ) 

307 

308 def clear(self) -> None: 

309 """Discard connection-local replay state without acknowledging it.""" 

310 self._replay = self._replay.clear() 

311 

312 async def drain(self) -> None: 

313 """Deliver held messages for which a recipient now has capacity.""" 

314 replay = self._replay 

315 next_replay = await replay.drain(self) 

316 if self._replay is replay: 

317 self._replay = next_replay 

318 

319 async def expire_replay(self, replay: _PersistentReplayBuffered) -> None: 

320 """Expire a buffered state only if it is still active.""" 

321 if self._replay is not replay: 

322 return 

323 next_replay = await replay.expire(self) 

324 if self._replay is replay: 

325 self._replay = next_replay 

326 

327 async def handle_publish(self, packet: Publish) -> None: 

328 """Route one inbound PUBLISH through its QoS receive flow.""" 

329 if packet.qos is QoS.AT_MOST_ONCE: 

330 recipient = await self._select_recipient_or_buffer(packet) 

331 if recipient is not None: 

332 await self._deliver(recipient, ack_callback=None) 

333 return None 

334 if packet.qos is QoS.AT_LEAST_ONCE: 

335 return await self._handle_qos1_publish(packet) 

336 if packet.qos is QoS.EXACTLY_ONCE: 

337 return await self._handle_qos2_publish(packet) 

338 return None 

339 

340 async def handle_pubrel(self, packet: PubRel) -> None: 

341 """Complete an inbound QoS 2 exchange.""" 

342 flight = self._state.inflight_qos2_in.pop(packet.packet_id, None) 

343 if flight is None: 

344 log.warning("PUBREL for unknown packet_id %d, completing it", packet.packet_id) 

345 await self._connection.send_packet( 

346 PubComp(packet_id=packet.packet_id, reason_code=_PACKET_IDENTIFIER_NOT_FOUND), 

347 ) 

348 return 

349 await self._complete(PubComp(packet_id=packet.packet_id)) 

350 if not flight.delivered: 

351 await self._deliver(flight.recipient, ack_callback=None) 

352 

353 def select_recipient(self, publish: Publish) -> InboundRecipient: 

354 """Select the exclusive request or subscription recipient for a PUBLISH.""" 

355 message = self._make_message(publish) 

356 if self._request_router is not None: 

357 request = self._request_router.claim(message) 

358 if request is not None: 

359 return InboundRecipient( 

360 message=message, 

361 request=request, 

362 ) 

363 

364 selection = self._select_subscription(publish) 

365 if selection.identifier_missing: 

366 identifier = publish.properties.subscription_identifier if publish.properties else None 

367 log.warning( 

368 "No subscription with identifier %r for topic %r, falling back to filter matching", 

369 identifier, 

370 publish.topic, 

371 ) 

372 if selection.tied_filters: 

373 log.warning( 

374 "Multiple equally-specific subscribers for %r: %s, delivering to first", 

375 publish.topic, 

376 list(selection.tied_filters), 

377 ) 

378 return InboundRecipient( 

379 message=message, 

380 subscription=selection.recipient, 

381 ) 

382 

383 def _select_subscription(self, publish: Publish) -> SubscriptionSelection: 

384 identifier = publish.properties.subscription_identifier if publish.properties else None 

385 if identifier is None: 

386 return self._state.subscriptions.select(topic=publish.topic) 

387 return self._state.subscriptions.select_by_identifier( 

388 topic=publish.topic, 

389 identifier=identifier, 

390 ) 

391 

392 async def _select_recipient_or_buffer(self, publish: Publish) -> InboundRecipient | None: 

393 replay = self._replay 

394 selected = self.select_recipient(publish) 

395 recipient, next_replay = await replay.accept(publish, selected) 

396 if self._replay is replay: 

397 self._replay = next_replay 

398 return recipient 

399 

400 async def deliver_replay( 

401 self, 

402 publish: Publish, 

403 recipient: InboundRecipient, 

404 ) -> None: 

405 """Deliver one message selected by the active replay state.""" 

406 if publish.qos is QoS.AT_MOST_ONCE: 

407 await self._deliver(recipient, ack_callback=None) 

408 elif publish.qos is QoS.AT_LEAST_ONCE: 

409 await self._deliver_qos1_publish(publish, recipient) 

410 elif publish.qos is QoS.EXACTLY_ONCE: 

411 await self._accept_qos2_publish(publish, recipient) 

412 

413 async def _handle_qos1_publish(self, packet: Publish) -> None: 

414 if packet.packet_id is None: 

415 msg = "Cannot publish without packet id" 

416 raise ValueError(msg) 

417 await self._hold_receive_quota(packet.packet_id) 

418 if self._replay.contains_packet_id(packet.packet_id): 

419 return 

420 recipient = await self._select_recipient_or_buffer(packet) 

421 if recipient is None: 

422 return 

423 await self._deliver_qos1_publish(packet, recipient) 

424 

425 async def _deliver_qos1_publish( 

426 self, 

427 packet: Publish, 

428 recipient: InboundRecipient, 

429 ) -> None: 

430 if packet.packet_id is None: 

431 msg = "Cannot publish without packet id" 

432 raise ValueError(msg) 

433 if recipient.auto_ack: 

434 if await self._deliver(recipient, ack_callback=None): 

435 await self._complete(PubAck(packet_id=packet.packet_id)) 

436 else: 

437 acked = False 

438 

439 async def _puback() -> None: 

440 nonlocal acked 

441 if acked: 

442 return 

443 acked = True 

444 await self._complete(PubAck(packet_id=cast("int", packet.packet_id))) 

445 

446 await self._deliver(recipient, ack_callback=_puback) 

447 

448 async def _handle_qos2_publish(self, packet: Publish) -> None: 

449 if packet.packet_id is None: 

450 msg = "Cannot publish without packet id" 

451 raise ValueError(msg) 

452 await self._hold_receive_quota(packet.packet_id) 

453 if packet.packet_id in self._state.inflight_qos2_in: 

454 # PUBREC already sent — resend it (duplicate PUBLISH after PUBREC) 

455 await self._connection.send_packet(PubRec(packet_id=packet.packet_id)) 

456 return 

457 if packet.packet_id in self._state.pending_ack_qos2_in: 

458 # Duplicate PUBLISH while app hasn't called msg.ack() yet — ignore 

459 return 

460 if self._replay.contains_packet_id(packet.packet_id): 

461 return 

462 recipient = await self._select_recipient_or_buffer(packet) 

463 if recipient is None: 

464 return 

465 await self._accept_qos2_publish(packet, recipient) 

466 

467 async def _accept_qos2_publish( 

468 self, 

469 packet: Publish, 

470 recipient: InboundRecipient, 

471 ) -> None: 

472 if packet.packet_id is None: 

473 msg = "Cannot publish without packet id" 

474 raise ValueError(msg) 

475 if recipient.subscription is not None and recipient.subscription[1].detached: 

476 return 

477 if recipient.auto_ack: 

478 self._state.inflight_qos2_in[packet.packet_id] = InboundQoS2Flight( 

479 packet_id=packet.packet_id, 

480 recipient=recipient, 

481 state=InboundQoS2State.PENDING_PUBREL, 

482 ) 

483 await self._connection.send_packet(PubRec(packet_id=packet.packet_id)) 

484 else: 

485 self._state.pending_ack_qos2_in.add(packet.packet_id) 

486 pid = packet.packet_id 

487 

488 async def _pubrec() -> None: 

489 self._state.pending_ack_qos2_in.discard(pid) 

490 self._state.inflight_qos2_in[pid] = InboundQoS2Flight( 

491 packet_id=pid, 

492 recipient=recipient, 

493 state=InboundQoS2State.PENDING_PUBREL, 

494 delivered=True, 

495 ) 

496 await self._connection.send_packet(PubRec(packet_id=pid)) 

497 

498 await self._deliver(recipient, ack_callback=_pubrec) 

499 

500 async def _hold_receive_quota(self, packet_id: int) -> None: 

501 """Count a QoS 1/2 PUBLISH against Receive Maximum until PUBACK or PUBCOMP. 

502 

503 Replayed, buffered, and manually acknowledged messages keep their unit; 

504 a duplicate packet ID reuses the unit it already holds. 

505 """ 

506 self._unacknowledged.add(packet_id) 

507 limit = self._receive_maximum 

508 if limit is not None and len(self._unacknowledged) > limit: 

509 await self._connection.abort(_RECEIVE_MAXIMUM_EXCEEDED) 

510 msg = f"Broker exceeded the Receive Maximum of {limit} unacknowledged QoS 1/2 messages" 

511 raise MQTTProtocolError(msg) 

512 

513 async def _complete(self, ack: PubAck | PubComp) -> None: 

514 """Send the acknowledgement that ends an inbound QoS 1/2 flow.""" 

515 # Release first: the broker may answer this ack with a new PUBLISH that 

516 # the read loop handles before a manual ack() caller resumes. 

517 self._unacknowledged.discard(ack.packet_id) 

518 await self._connection.send_packet(ack) 

519 

520 async def _deliver( 

521 self, 

522 recipient: InboundRecipient, 

523 ack_callback: Callable[[], Awaitable[None]] | None, 

524 ) -> bool: 

525 if recipient.request is not None: 

526 recipient.request.deliver() 

527 return True 

528 

529 subscription = recipient.subscription 

530 if subscription is None: 

531 return True 

532 

533 filter_, entry = subscription 

534 if entry.detached: 

535 return False 

536 message = recipient.message 

537 if not entry.auto_ack and ack_callback is not None: 

538 message._ack_callback = ack_callback # noqa: SLF001 - internal delivery contract 

539 await entry.queue.put(message) 

540 if entry.detached: 

541 if not entry.queue.empty(): 

542 entry.queue.get_nowait() 

543 return False 

544 log.debug("Delivered message for topic %r to filter %r", message.topic, filter_) 

545 return True 

546 

547 @staticmethod 

548 def _make_message(publish: Publish) -> Message: 

549 return Message( 

550 topic=publish.topic, 

551 payload=publish.payload, 

552 qos=publish.qos, 

553 retain=publish.retain, 

554 properties=publish.properties, 

555 )