Coverage for src/copytrading_engine/trading/application/alerts.py: 89%

71 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Alerts leave the committed outboxes on their own task, so a slow or unreachable alert service 

2never holds up copying. A failed send waits longer before each new try, and an alert about 

3something long past is dropped instead of sent.""" 

4 

5import asyncio 

6import datetime as dt 

7import logging 

8from collections import deque 

9from collections.abc import Callable, Iterable, Mapping 

10from typing import Protocol 

11 

12from copytrading_engine.parsing.relay import NotificationOutbox, deliver_notifications 

13from copytrading_engine.shared.notification_models import NotificationPayload 

14 

15log = logging.getLogger(__name__) 

16 

17# An alert says what just happened; one older than this is history, not news. It is marked 

18# delivered without being sent, so turning alerts on never replays weeks of them. 

19ALERT_MAX_AGE = dt.timedelta(hours=1) 

20# After a failed send the next try waits 5 seconds, then longer, up to 5 minutes. 

21ALERT_RETRY_SECONDS: tuple[float, ...] = (5, 15, 30, 60, 120, 300) 

22# Between passes while delivery works. 

23ALERT_IDLE_SECONDS = 1.0 

24 

25 

26class AlertSender(Protocol): 

27 async def send(self, payload: NotificationPayload) -> None: ... 

28 

29 

30class AccountAlerts(Protocol): 

31 async def pending_notifications( 

32 self, 

33 ) -> tuple[tuple[int, str, str, dict[str, object]], ...]: ... 

34 

35 async def confirm_notification(self, notification_id: int) -> None: ... 

36 

37 

38class AlertRelay: 

39 """Deliver the parser's and every running account's alerts, oldest first.""" 

40 

41 def __init__( 

42 self, 

43 sender: AlertSender, 

44 parser: NotificationOutbox, 

45 accounts: Callable[[], Iterable[AccountAlerts]], 

46 *, 

47 announcements: deque[NotificationPayload] | None = None, 

48 clock: Callable[[], dt.datetime] = lambda: dt.datetime.now(dt.UTC), 

49 retry_seconds: tuple[float, ...] = ALERT_RETRY_SECONDS, 

50 idle_seconds: float = ALERT_IDLE_SECONDS, 

51 ) -> None: 

52 if not retry_seconds: 52 ↛ 53line 52 didn't jump to line 53 because the condition on line 52 was never true

53 raise ValueError("Alert retries need at least one wait") 

54 self._sender = sender 

55 self._parser = parser 

56 self._accounts = accounts 

57 # Alerts the runtime raises itself, such as an account that stopped. They are not in any 

58 # outbox: they stay here until sent, and a stopped account has no outbox left to read. 

59 self._announcements = announcements if announcements is not None else deque() 

60 self._clock = clock 

61 self._retry_seconds = retry_seconds 

62 self._idle_seconds = idle_seconds 

63 self._failures = 0 

64 

65 @property 

66 def failing(self) -> bool: 

67 """The last pass could not send an alert.""" 

68 return self._failures > 0 

69 

70 async def run(self, stop: asyncio.Event) -> None: 

71 while not stop.is_set(): 

72 await self.deliver() 

73 wait = ( 

74 self._retry_seconds[min(self._failures, len(self._retry_seconds)) - 1] 

75 if self._failures 

76 else self._idle_seconds 

77 ) 

78 try: 

79 await asyncio.wait_for(stop.wait(), timeout=wait) 

80 except TimeoutError: 

81 pass 

82 

83 async def deliver(self) -> None: 

84 """One pass over every outbox. A failed send leaves its alert for the next pass.""" 

85 try: 

86 while self._announcements: 

87 await self._send(self._announcements[0]) 

88 self._announcements.popleft() 

89 await deliver_notifications(self._parser, lambda item: self._send(item.payload)) 

90 for account in self._accounts(): 

91 for notification_id, _, _, value in await account.pending_notifications(): 

92 await self._send(execution_alert(value)) 

93 await account.confirm_notification(notification_id) 

94 except Exception as exc: # noqa: BLE001 - delivery failure is retried later 

95 self._failures += 1 

96 log.warning( 

97 "notification_delivery_failed type=%s failures=%d", 

98 type(exc).__name__, 

99 self._failures, 

100 ) 

101 return 

102 self._failures = 0 

103 

104 async def _send(self, payload: NotificationPayload) -> None: 

105 if self._clock() - payload.starts_at > ALERT_MAX_AGE: 

106 return 

107 await self._sender.send(payload) 

108 

109 

110def account_stopped_alert(account_id: str, at: dt.datetime) -> NotificationPayload: 

111 """Copying into an account stopped on its own and will not resume until the owner acts.""" 

112 return NotificationPayload( 

113 labels={ 

114 "alertname": "AccountStopped", 

115 "system": "copytrade", 

116 "job": "execution", 

117 "severity": "critical", 

118 "account_id": account_id, 

119 }, 

120 annotations={ 

121 "summary": f"Copying into {account_id} stopped", 

122 "evidence": "CopyTrading hit an error it could not recover from after several tries.", 

123 "impact": "No new orders are placed there. Orders already sent stay at Alpaca.", 

124 "action": "In CopyTrading, stop copying everywhere, then start again to reopen it.", 

125 }, 

126 starts_at=at, 

127 ) 

128 

129 

130def execution_alert(value: Mapping[str, object]) -> NotificationPayload: 

131 """An account outbox row's saved payload, validated.""" 

132 labels = value.get("labels") 

133 annotations = value.get("annotations") 

134 starts_at = value.get("starts_at") 

135 if not isinstance(labels, dict) or not isinstance(annotations, dict): 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true

136 raise ValueError("Saved execution notification is invalid") 

137 if not isinstance(starts_at, str): 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true

138 raise ValueError("Saved execution notification timestamp is invalid") 

139 if not all(isinstance(k, str) and isinstance(v, str) for k, v in labels.items()): 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true

140 raise ValueError("Saved execution notification labels are invalid") 

141 if not all(isinstance(k, str) and isinstance(v, str) for k, v in annotations.items()): 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true

142 raise ValueError("Saved execution notification annotations are invalid") 

143 return NotificationPayload(labels, annotations, dt.datetime.fromisoformat(starts_at))