Coverage for src/copytrading_engine/trading/application/accounts.py: 94%
222 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Independent account reconciliation and durable signal intake supervisors."""
3import asyncio
4import dataclasses
5import datetime as dt
6import logging
7from collections.abc import Callable
8from contextlib import AbstractContextManager, nullcontext
9from typing import Protocol, runtime_checkable
11from copytrading_engine.execution.application.ports import (
12 AccountFailure,
13 AccountOpenRefused,
14 AccountRuntimeView,
15 BrokerError,
16 ExecutionObservation,
17)
18from copytrading_engine.execution.domain.events import LimitChange
19from copytrading_engine.execution.domain.lifecycle import (
20 AccountControlCommand,
21 AccountControlResult,
22)
23from copytrading_engine.execution.domain.lot_sales import (
24 LotSaleConfirmation,
25 LotSalePreview,
26 LotSalePreviewRequest,
27 LotSaleResult,
28)
29from copytrading_engine.execution.domain.manual_commands import (
30 ManualCommandPage,
31 ManualCommandResult,
32 ManualConfirmationRequest,
33 ManualCorrectionRecord,
34 ManualOrderPreview,
35 ManualPreviewRequest,
36)
37from copytrading_engine.execution.domain.market import EquityHistory, HistoryWindow
38from copytrading_engine.execution.domain.ownership import (
39 OwnershipInspection,
40 OwnershipResolution,
41 OwnershipResolutionRequest,
42)
43from copytrading_engine.execution.domain.signals import CopyConfig
44from copytrading_engine.execution.domain.sizing import DestinationSignal, DestinationTerms
45from copytrading_engine.execution.presentation.account_feed import AccountFeedPage
46from copytrading_engine.execution.presentation.operator_views import (
47 AccountEventPage,
48 AccountOverview,
49 DestinationView,
50)
51from copytrading_engine.parsing.contracts import (
52 DestinationIdentity,
53 DestinationRegistration,
54 PendingSignal,
55)
56from copytrading_engine.shared.correlation import WorkflowAttempt, bind_workflow_attempt
57from copytrading_engine.shared.route_keys import choose_route
58from copytrading_engine.shared.signals import StockSignal
59from copytrading_engine.trading.domain.status import AccountRunState, AccountStatus
61log = logging.getLogger(__name__)
64class AccountUnavailable(Exception):
65 """The account has stopped accepting work; its delivery must remain pending."""
68class AccountOwner(Protocol):
69 async def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None: ...
71 async def cycle(self, now: dt.datetime, *, halted: bool) -> ExecutionObservation: ...
73 async def recover_account(self, now: dt.datetime) -> OwnershipInspection: ...
75 async def control_account(
76 self, command: AccountControlCommand, now: dt.datetime
77 ) -> AccountControlResult: ...
79 async def resolve_ownership(
80 self, request: OwnershipResolutionRequest
81 ) -> OwnershipResolution: ...
83 async def record_manual_correction(
84 self, correction: ManualCorrectionRecord
85 ) -> ManualCorrectionRecord: ...
87 async def preview_lot_sale(
88 self, request: LotSalePreviewRequest, now: dt.datetime
89 ) -> LotSalePreview: ...
91 async def confirm_lot_sale(
92 self, request: LotSaleConfirmation, now: dt.datetime
93 ) -> LotSaleResult: ...
95 async def preview_manual_order(
96 self, request: ManualPreviewRequest, now: dt.datetime
97 ) -> ManualOrderPreview: ...
99 async def confirm_manual_order(
100 self, request: ManualConfirmationRequest, now: dt.datetime
101 ) -> ManualCommandResult: ...
103 async def manual_command_result(self, command_id: str) -> ManualCommandResult: ...
105 async def manual_command_page(
106 self, source_id: str, *, before_command_id: str | None = None, limit: int = 50
107 ) -> ManualCommandPage: ...
109 async def observation(self) -> ExecutionObservation: ...
111 async def update_config(
112 self, config: CopyConfig, now: dt.datetime
113 ) -> tuple[LimitChange, ...]: ...
115 async def account_status(self) -> AccountRuntimeView: ...
117 async def operator_overview(self) -> AccountOverview: ...
119 async def destination_views(self, source_ids: set[str]) -> dict[str, DestinationView]: ...
121 async def event_page(self, before_seq: int | None, limit: int) -> AccountEventPage: ...
123 async def feed_page(self, before_seq: int | None, limit: int) -> AccountFeedPage: ...
125 async def equity_history(self, window: HistoryWindow, now: dt.datetime) -> EquityHistory: ...
127 async def pending_notifications(
128 self,
129 ) -> tuple[tuple[int, str, str, dict[str, object]], ...]: ...
131 async def confirm_notification(self, notification_id: int) -> None: ...
133 def request_stop(self) -> None: ...
135 async def close(self) -> None: ...
138@runtime_checkable
139class WatchesOrders(Protocol):
140 """An owner that can say the moment one of its broker orders changes."""
142 async def watch_orders(
143 self, on_update: Callable[[], None], on_live: Callable[[bool], None], stop: asyncio.Event
144 ) -> None: ...
147# While the order stream is live and nothing waits on a clock, fills arrive the moment they
148# happen, so the periodic check is only a safety net for balances and reconciliation; it stays
149# well inside Alpaca's per-account request limits with many accounts.
150STREAM_SAFETY_SECONDS = 30.0
151# While Alpaca cannot be reached the account keeps trying, first after a second and then twice as
152# long each time, up to a minute; a passing outage must not leave copying stopped until the owner
153# notices.
154UNREACHABLE_FIRST_RETRY_SECONDS = 1.0
155UNREACHABLE_RETRY_SECONDS = 60.0
156# A cycle that fails for any other reason (not Alpaca answering badly) is tried again after each of
157# these waits, about four minutes in all, before the account stays failed. A wake (a new post or an
158# order update) cuts a wait short, so the tries can come sooner and the four minutes can be less.
159# Retrying cannot place an order twice: an order's client ID comes from its post, its intent is
160# committed before it goes out, and a retried cycle only looks orders up by that ID, never
161# resubmits them.
162CYCLE_RETRY_SECONDS: tuple[float, ...] = (15.0, 60.0, 180.0)
165def account_failure(error: BaseException) -> AccountFailure:
166 """Why an account could not open or stopped, in a code its owner can act on: the refusal it
167 was opened with, Alpaca turning the keys down, or Alpaca out of reach. Anything else is
168 `account_unavailable`, and the diagnostics journal holds the error."""
169 if isinstance(error, AccountOpenRefused): 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 return error.reason
171 if isinstance(error, BrokerError):
172 if error.status in {401, 403}:
173 return "broker_keys_rejected"
174 if error.transient: 174 ↛ 176line 174 didn't jump to line 176 because the condition on line 174 was always true
175 return "broker_unreachable"
176 if isinstance(error, TimeoutError):
177 return "broker_unreachable"
178 return "account_unavailable"
181class AccountSupervisor:
182 def __init__(
183 self,
184 account_id: str,
185 owner: AccountOwner | None,
186 poll_seconds: float,
187 stop: asyncio.Event,
188 on_change: Callable[[], None],
189 *,
190 open_failure: str = "account_unavailable",
191 on_failed: Callable[[str, BaseException], None] = lambda _account_id, _error: None,
192 ) -> None:
193 """`open_failure` says why an account without an owner could not open; `on_failed` is told
194 once, with the account's id and the error, when a running account stops on its own."""
195 self.id = account_id
196 self.owner = owner
197 self.poll_seconds = poll_seconds
198 self.stop = stop
199 self.on_change = on_change
200 self.on_failed = on_failed
201 self.state: AccountRunState = "ready" if owner is not None else "failed"
202 self.error_code: str | None = None if owner is not None else open_failure
203 self._account_status = AccountStatus(self.id, self.state, self.error_code)
204 self._task: asyncio.Task[None] | None = None
205 # Set by an order update or a newly received call: the next cycle starts at once.
206 self._wake = asyncio.Event()
207 self._order_watch: asyncio.Task[None] | None = None
208 self._stream_live = False
209 self._outstanding_work = True
211 @property
212 def status(self) -> AccountStatus:
213 return self._account_status
215 async def refresh(self) -> None:
216 if self.owner is None: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true
217 return
218 current = await self.owner.account_status()
219 self._account_status = AccountStatus(
220 self.id,
221 self.state,
222 self.error_code,
223 current.entry_permission,
224 current.recovery_preference,
225 current.readiness,
226 current.account_risk_status,
227 current.account_risk_reason,
228 current.account_activity_status,
229 current.account_activity_reason,
230 )
231 self.on_change()
233 def start(self) -> None:
234 if self.owner is None:
235 return
236 self.state = "running"
237 self._account_status = AccountStatus(
238 self.id,
239 self.state,
240 self.error_code,
241 self._account_status.entry_permission,
242 self._account_status.recovery_preference,
243 self._account_status.readiness,
244 self._account_status.account_risk_status,
245 self._account_status.account_risk_reason,
246 self._account_status.account_activity_status,
247 self._account_status.account_activity_reason,
248 )
249 self._task = asyncio.create_task(self._reconcile())
250 if isinstance(self.owner, WatchesOrders):
251 self._order_watch = asyncio.create_task(
252 self.owner.watch_orders(self._wake.set, self._stream_changed, self.stop)
253 )
254 self.on_change()
256 async def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None:
257 if delivery.terms.connection.account_id != self.id: 257 ↛ 258line 257 didn't jump to line 258 because the condition on line 257 was never true
258 raise ValueError("Destination account identity mismatch")
259 if self.stop.is_set(): 259 ↛ 260line 259 didn't jump to line 260 because the condition on line 259 was never true
260 raise AccountUnavailable
261 if self.state != "running" or self.owner is None:
262 raise AccountUnavailable
263 try:
264 await self.owner.receive(delivery, now)
265 except Exception as exc: # noqa: BLE001 - the supervisor records the account failure
266 self._fail(exc)
267 raise AccountUnavailable from None
268 # A new call is acted on now, not at the next periodic check.
269 self._wake.set()
271 async def _reconcile(self) -> None:
272 assert self.owner is not None
273 failures = 0
274 faults = 0
275 while not self.stop.is_set() and self.state == "running":
276 # Cleared before the cycle: a wake during it starts the next cycle at once.
277 self._wake.clear()
278 try:
279 observation = await self.owner.cycle(dt.datetime.now(dt.UTC), halted=False)
280 self._outstanding_work = observation.ledger.has_outstanding_work
281 faults = 0
282 if failures:
283 failures = 0
284 self._reached()
285 await self.refresh()
286 except BrokerError as exc:
287 if not exc.transient:
288 self._fail(exc)
289 return
290 failures += 1
291 self._unreachable(exc, failures)
292 await self._idle(
293 min(
294 UNREACHABLE_RETRY_SECONDS,
295 UNREACHABLE_FIRST_RETRY_SECONDS * 2.0 ** (failures - 1),
296 )
297 )
298 continue
299 except Exception as exc: # noqa: BLE001 - the supervisor records the account failure
300 if faults >= len(CYCLE_RETRY_SECONDS):
301 self._fail(exc)
302 return
303 self._faulted(exc, faults + 1)
304 wait = CYCLE_RETRY_SECONDS[faults]
305 faults += 1
306 await self._idle(wait)
307 continue
308 await self._idle()
310 @property
311 def check_seconds(self) -> float:
312 """How long until the next periodic check: a slow safety net while the order stream is
313 live and nothing waits on a clock, otherwise the account's own short interval."""
314 if self._stream_live and not self._outstanding_work:
315 return max(self.poll_seconds, STREAM_SAFETY_SECONDS)
316 return self.poll_seconds
318 def _stream_changed(self, live: bool) -> None:
319 self._stream_live = live
320 if not live:
321 # Updates may have been missed while it was down: reconcile now, then check often.
322 self._wake.set()
324 async def _idle(self, seconds: float | None = None) -> None:
325 """Wait for the periodic check, ended early by an order update, a new call, or Stop."""
326 waits = {
327 asyncio.ensure_future(self.stop.wait()),
328 asyncio.ensure_future(self._wake.wait()),
329 }
330 try:
331 await asyncio.wait(
332 waits,
333 timeout=self.check_seconds if seconds is None else seconds,
334 return_when=asyncio.FIRST_COMPLETED,
335 )
336 finally:
337 for wait in waits:
338 wait.cancel()
340 def _unreachable(self, exc: BrokerError, failures: int) -> None:
341 """Alpaca did not answer; the account stays running and tries again."""
342 if failures == 1:
343 log.warning("trading_account_unreachable id=%s status=%s", self.id, exc.status)
344 self.error_code = "broker_unreachable"
345 self._account_status = dataclasses.replace(self._account_status, error_code=self.error_code)
346 self.on_change()
348 def _faulted(self, exc: Exception, attempt: int) -> None:
349 """A cycle failed for a reason that is not the broker's; the account stays running and
350 tries again, and fails only if every try does."""
351 log.warning(
352 "trading_account_cycle_failed id=%s type=%s retry=%d/%d",
353 self.id,
354 type(exc).__name__,
355 attempt,
356 len(CYCLE_RETRY_SECONDS),
357 )
359 def _reached(self) -> None:
360 log.info("trading_account_reached id=%s", self.id)
361 self.error_code = None
362 self._account_status = dataclasses.replace(self._account_status, error_code=None)
364 def _fail(self, exc: Exception) -> None:
365 if self.state == "failed": 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true
366 return
367 if self._order_watch is not None: 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true
368 self._order_watch.cancel()
369 self.state = "failed"
370 self.error_code = account_failure(exc)
371 self._account_status = AccountStatus(self.id, self.state, self.error_code)
372 if self.owner is not None: 372 ↛ 374line 372 didn't jump to line 374 because the condition on line 372 was always true
373 self.owner.request_stop()
374 self.on_failed(self.id, exc)
375 log.error(
376 "trading_account_failed id=%s type=%s status=%s",
377 self.id,
378 type(exc).__name__,
379 getattr(exc, "status", None),
380 )
381 self.on_change()
383 async def close(self) -> None:
384 if self.owner is not None:
385 self.owner.request_stop()
386 if self._task is not None:
387 await self._task
388 if self.owner is not None:
389 await self.owner.close()
392class SignalInbox(Protocol):
393 async def claim_pending_signals(
394 self, limit: int = 100, after_seq: int = 0
395 ) -> tuple[PendingSignal, ...]: ...
397 async def confirm_signal(self, key: str) -> None: ...
399 async def prepare_destinations(
400 self,
401 key: str,
402 registrations: tuple[DestinationRegistration, ...],
403 ) -> tuple[DestinationIdentity, ...]: ...
405 async def reserve_destination_attempt(
406 self, key: str, account_id: str
407 ) -> DestinationIdentity: ...
410class SignalFanout:
411 """Confirm parser delivery only after every configured account commits receipt."""
413 def __init__(
414 self,
415 parser: SignalInbox,
416 routes: dict[str, tuple[DestinationTerms, ...]],
417 accounts: dict[str, AccountSupervisor],
418 stop: asyncio.Event,
419 *,
420 observe_workflow: Callable[[WorkflowAttempt], AbstractContextManager] = lambda _: (
421 nullcontext()
422 ),
423 observe_destination: Callable[
424 [str, WorkflowAttempt], AbstractContextManager
425 ] = lambda _message, _attempt: nullcontext(),
426 ) -> None:
427 self.parser = parser
428 self.routes = routes
429 self.accounts = accounts
430 self.stop = stop
431 self.observe_workflow = observe_workflow
432 self.observe_destination = observe_destination
433 self._cursor = 0
435 async def _receive(
436 self,
437 key: str,
438 signal: StockSignal,
439 terms: DestinationTerms,
440 identity: DestinationIdentity,
441 ) -> None:
442 attempt = WorkflowAttempt(
443 identity.workflow_id, identity.trace_id, identity.destination_id, identity.attempt
444 )
445 with bind_workflow_attempt(attempt), self.observe_destination(key, attempt):
446 await self.accounts[terms.connection.account_id].receive(
447 DestinationSignal(signal=signal, terms=terms), dt.datetime.now(dt.UTC)
448 )
450 async def deliver(self) -> int:
451 confirmed = 0
452 for _ in range(2): 452 ↛ 514line 452 didn't jump to line 514 because the loop on line 452 didn't complete
453 deliveries = await self.parser.claim_pending_signals(after_seq=self._cursor)
454 if not deliveries:
455 self._cursor = 0
456 break
457 for delivery in deliveries:
458 if self.stop.is_set(): 458 ↛ 459line 458 didn't jump to line 459 because the condition on line 458 was never true
459 return confirmed
460 self._cursor = delivery.seq
461 waiting = False
462 route = (
463 choose_route(
464 self.routes,
465 delivery.signal.source,
466 delivery.signal.channel_id,
467 delivery.signal.author_id,
468 ).route
469 or ()
470 )
471 registrations = tuple(
472 DestinationRegistration(
473 terms.connection.account_id, terms.configuration_revision
474 )
475 for terms in route
476 )
477 destinations = await self.parser.prepare_destinations(delivery.key, registrations)
478 if destinations:
479 first = destinations[0]
480 workflow = WorkflowAttempt(first.workflow_id, first.trace_id, None, 0)
481 with bind_workflow_attempt(workflow):
482 with self.observe_workflow(workflow):
483 reserved = []
484 for terms in route:
485 if self.stop.is_set():
486 return confirmed
487 identity = await self.parser.reserve_destination_attempt(
488 delivery.key, terms.connection.account_id
489 )
490 reserved.append((terms, identity))
491 # Each account serializes its own work, so different accounts take
492 # the same post side by side. Every account runs to the end, even if
493 # another fails: cancelling one mid-order could orphan a submission.
494 # The work is built only here, so a stop above never strands a
495 # coroutine that nothing will await.
496 outcomes = await asyncio.gather(
497 *(
498 self._receive(delivery.key, delivery.signal, terms, identity)
499 for terms, identity in reserved
500 ),
501 return_exceptions=True,
502 )
503 for outcome in outcomes:
504 if isinstance(outcome, AccountUnavailable):
505 waiting = True
506 elif isinstance(outcome, BaseException):
507 raise outcome
508 if not waiting:
509 await self.parser.confirm_signal(delivery.key)
510 confirmed += 1
511 if len(deliveries) < 100:
512 self._cursor = 0
513 break
514 return confirmed