Coverage for src/copytrading_engine/trading/application/accounts.py: 94%

222 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Independent account reconciliation and durable signal intake supervisors.""" 

2 

3import asyncio 

4import dataclasses 

5import datetime as dt 

6import logging 

7from collections.abc import Callable 

8from contextlib import AbstractContextManager, nullcontext 

9from typing import Protocol, runtime_checkable 

10 

11from copytrading_engine.execution.application.ports import ( 

12 AccountFailure, 

13 AccountOpenRefused, 

14 AccountRuntimeView, 

15 BrokerError, 

16 ExecutionObservation, 

17) 

18from copytrading_engine.execution.domain.events import LimitChange 

19from copytrading_engine.execution.domain.lifecycle import ( 

20 AccountControlCommand, 

21 AccountControlResult, 

22) 

23from copytrading_engine.execution.domain.lot_sales import ( 

24 LotSaleConfirmation, 

25 LotSalePreview, 

26 LotSalePreviewRequest, 

27 LotSaleResult, 

28) 

29from copytrading_engine.execution.domain.manual_commands import ( 

30 ManualCommandPage, 

31 ManualCommandResult, 

32 ManualConfirmationRequest, 

33 ManualCorrectionRecord, 

34 ManualOrderPreview, 

35 ManualPreviewRequest, 

36) 

37from copytrading_engine.execution.domain.market import EquityHistory, HistoryWindow 

38from copytrading_engine.execution.domain.ownership import ( 

39 OwnershipInspection, 

40 OwnershipResolution, 

41 OwnershipResolutionRequest, 

42) 

43from copytrading_engine.execution.domain.signals import CopyConfig 

44from copytrading_engine.execution.domain.sizing import DestinationSignal, DestinationTerms 

45from copytrading_engine.execution.presentation.account_feed import AccountFeedPage 

46from copytrading_engine.execution.presentation.operator_views import ( 

47 AccountEventPage, 

48 AccountOverview, 

49 DestinationView, 

50) 

51from copytrading_engine.parsing.contracts import ( 

52 DestinationIdentity, 

53 DestinationRegistration, 

54 PendingSignal, 

55) 

56from copytrading_engine.shared.correlation import WorkflowAttempt, bind_workflow_attempt 

57from copytrading_engine.shared.route_keys import choose_route 

58from copytrading_engine.shared.signals import StockSignal 

59from copytrading_engine.trading.domain.status import AccountRunState, AccountStatus 

60 

61log = logging.getLogger(__name__) 

62 

63 

64class AccountUnavailable(Exception): 

65 """The account has stopped accepting work; its delivery must remain pending.""" 

66 

67 

68class AccountOwner(Protocol): 

69 async def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None: ... 

70 

71 async def cycle(self, now: dt.datetime, *, halted: bool) -> ExecutionObservation: ... 

72 

73 async def recover_account(self, now: dt.datetime) -> OwnershipInspection: ... 

74 

75 async def control_account( 

76 self, command: AccountControlCommand, now: dt.datetime 

77 ) -> AccountControlResult: ... 

78 

79 async def resolve_ownership( 

80 self, request: OwnershipResolutionRequest 

81 ) -> OwnershipResolution: ... 

82 

83 async def record_manual_correction( 

84 self, correction: ManualCorrectionRecord 

85 ) -> ManualCorrectionRecord: ... 

86 

87 async def preview_lot_sale( 

88 self, request: LotSalePreviewRequest, now: dt.datetime 

89 ) -> LotSalePreview: ... 

90 

91 async def confirm_lot_sale( 

92 self, request: LotSaleConfirmation, now: dt.datetime 

93 ) -> LotSaleResult: ... 

94 

95 async def preview_manual_order( 

96 self, request: ManualPreviewRequest, now: dt.datetime 

97 ) -> ManualOrderPreview: ... 

98 

99 async def confirm_manual_order( 

100 self, request: ManualConfirmationRequest, now: dt.datetime 

101 ) -> ManualCommandResult: ... 

102 

103 async def manual_command_result(self, command_id: str) -> ManualCommandResult: ... 

104 

105 async def manual_command_page( 

106 self, source_id: str, *, before_command_id: str | None = None, limit: int = 50 

107 ) -> ManualCommandPage: ... 

108 

109 async def observation(self) -> ExecutionObservation: ... 

110 

111 async def update_config( 

112 self, config: CopyConfig, now: dt.datetime 

113 ) -> tuple[LimitChange, ...]: ... 

114 

115 async def account_status(self) -> AccountRuntimeView: ... 

116 

117 async def operator_overview(self) -> AccountOverview: ... 

118 

119 async def destination_views(self, source_ids: set[str]) -> dict[str, DestinationView]: ... 

120 

121 async def event_page(self, before_seq: int | None, limit: int) -> AccountEventPage: ... 

122 

123 async def feed_page(self, before_seq: int | None, limit: int) -> AccountFeedPage: ... 

124 

125 async def equity_history(self, window: HistoryWindow, now: dt.datetime) -> EquityHistory: ... 

126 

127 async def pending_notifications( 

128 self, 

129 ) -> tuple[tuple[int, str, str, dict[str, object]], ...]: ... 

130 

131 async def confirm_notification(self, notification_id: int) -> None: ... 

132 

133 def request_stop(self) -> None: ... 

134 

135 async def close(self) -> None: ... 

136 

137 

138@runtime_checkable 

139class WatchesOrders(Protocol): 

140 """An owner that can say the moment one of its broker orders changes.""" 

141 

142 async def watch_orders( 

143 self, on_update: Callable[[], None], on_live: Callable[[bool], None], stop: asyncio.Event 

144 ) -> None: ... 

145 

146 

147# While the order stream is live and nothing waits on a clock, fills arrive the moment they 

148# happen, so the periodic check is only a safety net for balances and reconciliation; it stays 

149# well inside Alpaca's per-account request limits with many accounts. 

150STREAM_SAFETY_SECONDS = 30.0 

151# While Alpaca cannot be reached the account keeps trying, first after a second and then twice as 

152# long each time, up to a minute; a passing outage must not leave copying stopped until the owner 

153# notices. 

154UNREACHABLE_FIRST_RETRY_SECONDS = 1.0 

155UNREACHABLE_RETRY_SECONDS = 60.0 

156# A cycle that fails for any other reason (not Alpaca answering badly) is tried again after each of 

157# these waits, about four minutes in all, before the account stays failed. A wake (a new post or an 

158# order update) cuts a wait short, so the tries can come sooner and the four minutes can be less. 

159# Retrying cannot place an order twice: an order's client ID comes from its post, its intent is 

160# committed before it goes out, and a retried cycle only looks orders up by that ID, never 

161# resubmits them. 

162CYCLE_RETRY_SECONDS: tuple[float, ...] = (15.0, 60.0, 180.0) 

163 

164 

165def account_failure(error: BaseException) -> AccountFailure: 

166 """Why an account could not open or stopped, in a code its owner can act on: the refusal it 

167 was opened with, Alpaca turning the keys down, or Alpaca out of reach. Anything else is 

168 `account_unavailable`, and the diagnostics journal holds the error.""" 

169 if isinstance(error, AccountOpenRefused): 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 return error.reason 

171 if isinstance(error, BrokerError): 

172 if error.status in {401, 403}: 

173 return "broker_keys_rejected" 

174 if error.transient: 174 ↛ 176line 174 didn't jump to line 176 because the condition on line 174 was always true

175 return "broker_unreachable" 

176 if isinstance(error, TimeoutError): 

177 return "broker_unreachable" 

178 return "account_unavailable" 

179 

180 

181class AccountSupervisor: 

182 def __init__( 

183 self, 

184 account_id: str, 

185 owner: AccountOwner | None, 

186 poll_seconds: float, 

187 stop: asyncio.Event, 

188 on_change: Callable[[], None], 

189 *, 

190 open_failure: str = "account_unavailable", 

191 on_failed: Callable[[str, BaseException], None] = lambda _account_id, _error: None, 

192 ) -> None: 

193 """`open_failure` says why an account without an owner could not open; `on_failed` is told 

194 once, with the account's id and the error, when a running account stops on its own.""" 

195 self.id = account_id 

196 self.owner = owner 

197 self.poll_seconds = poll_seconds 

198 self.stop = stop 

199 self.on_change = on_change 

200 self.on_failed = on_failed 

201 self.state: AccountRunState = "ready" if owner is not None else "failed" 

202 self.error_code: str | None = None if owner is not None else open_failure 

203 self._account_status = AccountStatus(self.id, self.state, self.error_code) 

204 self._task: asyncio.Task[None] | None = None 

205 # Set by an order update or a newly received call: the next cycle starts at once. 

206 self._wake = asyncio.Event() 

207 self._order_watch: asyncio.Task[None] | None = None 

208 self._stream_live = False 

209 self._outstanding_work = True 

210 

211 @property 

212 def status(self) -> AccountStatus: 

213 return self._account_status 

214 

215 async def refresh(self) -> None: 

216 if self.owner is None: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true

217 return 

218 current = await self.owner.account_status() 

219 self._account_status = AccountStatus( 

220 self.id, 

221 self.state, 

222 self.error_code, 

223 current.entry_permission, 

224 current.recovery_preference, 

225 current.readiness, 

226 current.account_risk_status, 

227 current.account_risk_reason, 

228 current.account_activity_status, 

229 current.account_activity_reason, 

230 ) 

231 self.on_change() 

232 

233 def start(self) -> None: 

234 if self.owner is None: 

235 return 

236 self.state = "running" 

237 self._account_status = AccountStatus( 

238 self.id, 

239 self.state, 

240 self.error_code, 

241 self._account_status.entry_permission, 

242 self._account_status.recovery_preference, 

243 self._account_status.readiness, 

244 self._account_status.account_risk_status, 

245 self._account_status.account_risk_reason, 

246 self._account_status.account_activity_status, 

247 self._account_status.account_activity_reason, 

248 ) 

249 self._task = asyncio.create_task(self._reconcile()) 

250 if isinstance(self.owner, WatchesOrders): 

251 self._order_watch = asyncio.create_task( 

252 self.owner.watch_orders(self._wake.set, self._stream_changed, self.stop) 

253 ) 

254 self.on_change() 

255 

256 async def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None: 

257 if delivery.terms.connection.account_id != self.id: 257 ↛ 258line 257 didn't jump to line 258 because the condition on line 257 was never true

258 raise ValueError("Destination account identity mismatch") 

259 if self.stop.is_set(): 259 ↛ 260line 259 didn't jump to line 260 because the condition on line 259 was never true

260 raise AccountUnavailable 

261 if self.state != "running" or self.owner is None: 

262 raise AccountUnavailable 

263 try: 

264 await self.owner.receive(delivery, now) 

265 except Exception as exc: # noqa: BLE001 - the supervisor records the account failure 

266 self._fail(exc) 

267 raise AccountUnavailable from None 

268 # A new call is acted on now, not at the next periodic check. 

269 self._wake.set() 

270 

271 async def _reconcile(self) -> None: 

272 assert self.owner is not None 

273 failures = 0 

274 faults = 0 

275 while not self.stop.is_set() and self.state == "running": 

276 # Cleared before the cycle: a wake during it starts the next cycle at once. 

277 self._wake.clear() 

278 try: 

279 observation = await self.owner.cycle(dt.datetime.now(dt.UTC), halted=False) 

280 self._outstanding_work = observation.ledger.has_outstanding_work 

281 faults = 0 

282 if failures: 

283 failures = 0 

284 self._reached() 

285 await self.refresh() 

286 except BrokerError as exc: 

287 if not exc.transient: 

288 self._fail(exc) 

289 return 

290 failures += 1 

291 self._unreachable(exc, failures) 

292 await self._idle( 

293 min( 

294 UNREACHABLE_RETRY_SECONDS, 

295 UNREACHABLE_FIRST_RETRY_SECONDS * 2.0 ** (failures - 1), 

296 ) 

297 ) 

298 continue 

299 except Exception as exc: # noqa: BLE001 - the supervisor records the account failure 

300 if faults >= len(CYCLE_RETRY_SECONDS): 

301 self._fail(exc) 

302 return 

303 self._faulted(exc, faults + 1) 

304 wait = CYCLE_RETRY_SECONDS[faults] 

305 faults += 1 

306 await self._idle(wait) 

307 continue 

308 await self._idle() 

309 

310 @property 

311 def check_seconds(self) -> float: 

312 """How long until the next periodic check: a slow safety net while the order stream is 

313 live and nothing waits on a clock, otherwise the account's own short interval.""" 

314 if self._stream_live and not self._outstanding_work: 

315 return max(self.poll_seconds, STREAM_SAFETY_SECONDS) 

316 return self.poll_seconds 

317 

318 def _stream_changed(self, live: bool) -> None: 

319 self._stream_live = live 

320 if not live: 

321 # Updates may have been missed while it was down: reconcile now, then check often. 

322 self._wake.set() 

323 

324 async def _idle(self, seconds: float | None = None) -> None: 

325 """Wait for the periodic check, ended early by an order update, a new call, or Stop.""" 

326 waits = { 

327 asyncio.ensure_future(self.stop.wait()), 

328 asyncio.ensure_future(self._wake.wait()), 

329 } 

330 try: 

331 await asyncio.wait( 

332 waits, 

333 timeout=self.check_seconds if seconds is None else seconds, 

334 return_when=asyncio.FIRST_COMPLETED, 

335 ) 

336 finally: 

337 for wait in waits: 

338 wait.cancel() 

339 

340 def _unreachable(self, exc: BrokerError, failures: int) -> None: 

341 """Alpaca did not answer; the account stays running and tries again.""" 

342 if failures == 1: 

343 log.warning("trading_account_unreachable id=%s status=%s", self.id, exc.status) 

344 self.error_code = "broker_unreachable" 

345 self._account_status = dataclasses.replace(self._account_status, error_code=self.error_code) 

346 self.on_change() 

347 

348 def _faulted(self, exc: Exception, attempt: int) -> None: 

349 """A cycle failed for a reason that is not the broker's; the account stays running and 

350 tries again, and fails only if every try does.""" 

351 log.warning( 

352 "trading_account_cycle_failed id=%s type=%s retry=%d/%d", 

353 self.id, 

354 type(exc).__name__, 

355 attempt, 

356 len(CYCLE_RETRY_SECONDS), 

357 ) 

358 

359 def _reached(self) -> None: 

360 log.info("trading_account_reached id=%s", self.id) 

361 self.error_code = None 

362 self._account_status = dataclasses.replace(self._account_status, error_code=None) 

363 

364 def _fail(self, exc: Exception) -> None: 

365 if self.state == "failed": 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true

366 return 

367 if self._order_watch is not None: 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true

368 self._order_watch.cancel() 

369 self.state = "failed" 

370 self.error_code = account_failure(exc) 

371 self._account_status = AccountStatus(self.id, self.state, self.error_code) 

372 if self.owner is not None: 372 ↛ 374line 372 didn't jump to line 374 because the condition on line 372 was always true

373 self.owner.request_stop() 

374 self.on_failed(self.id, exc) 

375 log.error( 

376 "trading_account_failed id=%s type=%s status=%s", 

377 self.id, 

378 type(exc).__name__, 

379 getattr(exc, "status", None), 

380 ) 

381 self.on_change() 

382 

383 async def close(self) -> None: 

384 if self.owner is not None: 

385 self.owner.request_stop() 

386 if self._task is not None: 

387 await self._task 

388 if self.owner is not None: 

389 await self.owner.close() 

390 

391 

392class SignalInbox(Protocol): 

393 async def claim_pending_signals( 

394 self, limit: int = 100, after_seq: int = 0 

395 ) -> tuple[PendingSignal, ...]: ... 

396 

397 async def confirm_signal(self, key: str) -> None: ... 

398 

399 async def prepare_destinations( 

400 self, 

401 key: str, 

402 registrations: tuple[DestinationRegistration, ...], 

403 ) -> tuple[DestinationIdentity, ...]: ... 

404 

405 async def reserve_destination_attempt( 

406 self, key: str, account_id: str 

407 ) -> DestinationIdentity: ... 

408 

409 

410class SignalFanout: 

411 """Confirm parser delivery only after every configured account commits receipt.""" 

412 

413 def __init__( 

414 self, 

415 parser: SignalInbox, 

416 routes: dict[str, tuple[DestinationTerms, ...]], 

417 accounts: dict[str, AccountSupervisor], 

418 stop: asyncio.Event, 

419 *, 

420 observe_workflow: Callable[[WorkflowAttempt], AbstractContextManager] = lambda _: ( 

421 nullcontext() 

422 ), 

423 observe_destination: Callable[ 

424 [str, WorkflowAttempt], AbstractContextManager 

425 ] = lambda _message, _attempt: nullcontext(), 

426 ) -> None: 

427 self.parser = parser 

428 self.routes = routes 

429 self.accounts = accounts 

430 self.stop = stop 

431 self.observe_workflow = observe_workflow 

432 self.observe_destination = observe_destination 

433 self._cursor = 0 

434 

435 async def _receive( 

436 self, 

437 key: str, 

438 signal: StockSignal, 

439 terms: DestinationTerms, 

440 identity: DestinationIdentity, 

441 ) -> None: 

442 attempt = WorkflowAttempt( 

443 identity.workflow_id, identity.trace_id, identity.destination_id, identity.attempt 

444 ) 

445 with bind_workflow_attempt(attempt), self.observe_destination(key, attempt): 

446 await self.accounts[terms.connection.account_id].receive( 

447 DestinationSignal(signal=signal, terms=terms), dt.datetime.now(dt.UTC) 

448 ) 

449 

450 async def deliver(self) -> int: 

451 confirmed = 0 

452 for _ in range(2): 452 ↛ 514line 452 didn't jump to line 514 because the loop on line 452 didn't complete

453 deliveries = await self.parser.claim_pending_signals(after_seq=self._cursor) 

454 if not deliveries: 

455 self._cursor = 0 

456 break 

457 for delivery in deliveries: 

458 if self.stop.is_set(): 458 ↛ 459line 458 didn't jump to line 459 because the condition on line 458 was never true

459 return confirmed 

460 self._cursor = delivery.seq 

461 waiting = False 

462 route = ( 

463 choose_route( 

464 self.routes, 

465 delivery.signal.source, 

466 delivery.signal.channel_id, 

467 delivery.signal.author_id, 

468 ).route 

469 or () 

470 ) 

471 registrations = tuple( 

472 DestinationRegistration( 

473 terms.connection.account_id, terms.configuration_revision 

474 ) 

475 for terms in route 

476 ) 

477 destinations = await self.parser.prepare_destinations(delivery.key, registrations) 

478 if destinations: 

479 first = destinations[0] 

480 workflow = WorkflowAttempt(first.workflow_id, first.trace_id, None, 0) 

481 with bind_workflow_attempt(workflow): 

482 with self.observe_workflow(workflow): 

483 reserved = [] 

484 for terms in route: 

485 if self.stop.is_set(): 

486 return confirmed 

487 identity = await self.parser.reserve_destination_attempt( 

488 delivery.key, terms.connection.account_id 

489 ) 

490 reserved.append((terms, identity)) 

491 # Each account serializes its own work, so different accounts take 

492 # the same post side by side. Every account runs to the end, even if 

493 # another fails: cancelling one mid-order could orphan a submission. 

494 # The work is built only here, so a stop above never strands a 

495 # coroutine that nothing will await. 

496 outcomes = await asyncio.gather( 

497 *( 

498 self._receive(delivery.key, delivery.signal, terms, identity) 

499 for terms, identity in reserved 

500 ), 

501 return_exceptions=True, 

502 ) 

503 for outcome in outcomes: 

504 if isinstance(outcome, AccountUnavailable): 

505 waiting = True 

506 elif isinstance(outcome, BaseException): 

507 raise outcome 

508 if not waiting: 

509 await self.parser.confirm_signal(delivery.key) 

510 confirmed += 1 

511 if len(deliveries) < 100: 

512 self._cursor = 0 

513 break 

514 return confirmed