Coverage for src/copytrading_engine/trading/application/alerts.py: 89%
71 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Alerts leave the committed outboxes on their own task, so a slow or unreachable alert service
2never holds up copying. A failed send waits longer before each new try, and an alert about
3something long past is dropped instead of sent."""
5import asyncio
6import datetime as dt
7import logging
8from collections import deque
9from collections.abc import Callable, Iterable, Mapping
10from typing import Protocol
12from copytrading_engine.parsing.relay import NotificationOutbox, deliver_notifications
13from copytrading_engine.shared.notification_models import NotificationPayload
15log = logging.getLogger(__name__)
17# An alert says what just happened; one older than this is history, not news. It is marked
18# delivered without being sent, so turning alerts on never replays weeks of them.
19ALERT_MAX_AGE = dt.timedelta(hours=1)
20# After a failed send the next try waits 5 seconds, then longer, up to 5 minutes.
21ALERT_RETRY_SECONDS: tuple[float, ...] = (5, 15, 30, 60, 120, 300)
22# Between passes while delivery works.
23ALERT_IDLE_SECONDS = 1.0
26class AlertSender(Protocol):
27 async def send(self, payload: NotificationPayload) -> None: ...
30class AccountAlerts(Protocol):
31 async def pending_notifications(
32 self,
33 ) -> tuple[tuple[int, str, str, dict[str, object]], ...]: ...
35 async def confirm_notification(self, notification_id: int) -> None: ...
38class AlertRelay:
39 """Deliver the parser's and every running account's alerts, oldest first."""
41 def __init__(
42 self,
43 sender: AlertSender,
44 parser: NotificationOutbox,
45 accounts: Callable[[], Iterable[AccountAlerts]],
46 *,
47 announcements: deque[NotificationPayload] | None = None,
48 clock: Callable[[], dt.datetime] = lambda: dt.datetime.now(dt.UTC),
49 retry_seconds: tuple[float, ...] = ALERT_RETRY_SECONDS,
50 idle_seconds: float = ALERT_IDLE_SECONDS,
51 ) -> None:
52 if not retry_seconds: 52 ↛ 53line 52 didn't jump to line 53 because the condition on line 52 was never true
53 raise ValueError("Alert retries need at least one wait")
54 self._sender = sender
55 self._parser = parser
56 self._accounts = accounts
57 # Alerts the runtime raises itself, such as an account that stopped. They are not in any
58 # outbox: they stay here until sent, and a stopped account has no outbox left to read.
59 self._announcements = announcements if announcements is not None else deque()
60 self._clock = clock
61 self._retry_seconds = retry_seconds
62 self._idle_seconds = idle_seconds
63 self._failures = 0
65 @property
66 def failing(self) -> bool:
67 """The last pass could not send an alert."""
68 return self._failures > 0
70 async def run(self, stop: asyncio.Event) -> None:
71 while not stop.is_set():
72 await self.deliver()
73 wait = (
74 self._retry_seconds[min(self._failures, len(self._retry_seconds)) - 1]
75 if self._failures
76 else self._idle_seconds
77 )
78 try:
79 await asyncio.wait_for(stop.wait(), timeout=wait)
80 except TimeoutError:
81 pass
83 async def deliver(self) -> None:
84 """One pass over every outbox. A failed send leaves its alert for the next pass."""
85 try:
86 while self._announcements:
87 await self._send(self._announcements[0])
88 self._announcements.popleft()
89 await deliver_notifications(self._parser, lambda item: self._send(item.payload))
90 for account in self._accounts():
91 for notification_id, _, _, value in await account.pending_notifications():
92 await self._send(execution_alert(value))
93 await account.confirm_notification(notification_id)
94 except Exception as exc: # noqa: BLE001 - delivery failure is retried later
95 self._failures += 1
96 log.warning(
97 "notification_delivery_failed type=%s failures=%d",
98 type(exc).__name__,
99 self._failures,
100 )
101 return
102 self._failures = 0
104 async def _send(self, payload: NotificationPayload) -> None:
105 if self._clock() - payload.starts_at > ALERT_MAX_AGE:
106 return
107 await self._sender.send(payload)
110def account_stopped_alert(account_id: str, at: dt.datetime) -> NotificationPayload:
111 """Copying into an account stopped on its own and will not resume until the owner acts."""
112 return NotificationPayload(
113 labels={
114 "alertname": "AccountStopped",
115 "system": "copytrade",
116 "job": "execution",
117 "severity": "critical",
118 "account_id": account_id,
119 },
120 annotations={
121 "summary": f"Copying into {account_id} stopped",
122 "evidence": "CopyTrading hit an error it could not recover from after several tries.",
123 "impact": "No new orders are placed there. Orders already sent stay at Alpaca.",
124 "action": "In CopyTrading, stop copying everywhere, then start again to reopen it.",
125 },
126 starts_at=at,
127 )
130def execution_alert(value: Mapping[str, object]) -> NotificationPayload:
131 """An account outbox row's saved payload, validated."""
132 labels = value.get("labels")
133 annotations = value.get("annotations")
134 starts_at = value.get("starts_at")
135 if not isinstance(labels, dict) or not isinstance(annotations, dict): 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 raise ValueError("Saved execution notification is invalid")
137 if not isinstance(starts_at, str): 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true
138 raise ValueError("Saved execution notification timestamp is invalid")
139 if not all(isinstance(k, str) and isinstance(v, str) for k, v in labels.items()): 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true
140 raise ValueError("Saved execution notification labels are invalid")
141 if not all(isinstance(k, str) and isinstance(v, str) for k, v in annotations.items()): 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true
142 raise ValueError("Saved execution notification annotations are invalid")
143 return NotificationPayload(labels, annotations, dt.datetime.fromisoformat(starts_at))