Coverage for src/copytrading_engine/execution/adapters/owner.py: 85%

206 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Confine the execution engine, broker, and synchronous store to one worker.""" 

2 

3import asyncio 

4import concurrent.futures 

5import datetime as dt 

6import logging 

7import threading 

8from collections.abc import Callable 

9from pathlib import Path 

10from typing import TypeVar, cast 

11 

12from copytrading_engine.execution.adapters.alpaca.broker import ( 

13 AlpacaBroker, 

14 AlpacaCredentials, 

15 Environment, 

16) 

17from copytrading_engine.execution.adapters.alpaca.order_stream import ( 

18 OrderStream, 

19 alpaca_order_stream, 

20) 

21from copytrading_engine.execution.adapters.resources import ( 

22 ExecutionResources, 

23 build_resources, 

24 default_account_lock_root, 

25) 

26from copytrading_engine.execution.application.ports import ( 

27 AccountRuntimeView, 

28 Broker, 

29 ExecutionObservation, 

30 ExecutionObserver, 

31 NoOpObserver, 

32) 

33from copytrading_engine.execution.domain.events import LimitChange 

34from copytrading_engine.execution.domain.lifecycle import ( 

35 AccountControlCommand, 

36 AccountControlResult, 

37 AccountOwnerClosed, 

38) 

39from copytrading_engine.execution.domain.lot_sales import ( 

40 LotSaleConfirmation, 

41 LotSalePreview, 

42 LotSalePreviewRequest, 

43 LotSaleResult, 

44) 

45from copytrading_engine.execution.domain.manual_commands import ( 

46 ManualCommandPage, 

47 ManualCommandResult, 

48 ManualConfirmationRequest, 

49 ManualCorrectionRecord, 

50 ManualOrderPreview, 

51 ManualPreviewRequest, 

52) 

53from copytrading_engine.execution.domain.market import EquityHistory, HistoryWindow 

54from copytrading_engine.execution.domain.ownership import ( 

55 OwnershipInspection, 

56 OwnershipResolution, 

57 OwnershipResolutionRequest, 

58) 

59from copytrading_engine.execution.domain.recovery import ( 

60 IncidentClearanceEvidence, 

61 ManualSale, 

62 ReleaseEvidence, 

63) 

64from copytrading_engine.execution.domain.signals import CopyConfig 

65from copytrading_engine.execution.domain.sizing import DestinationSignal 

66from copytrading_engine.execution.presentation.account_feed import AccountFeedPage 

67from copytrading_engine.execution.presentation.operator_views import ( 

68 AccountEventPage, 

69 AccountOverview, 

70 DestinationView, 

71) 

72 

73log = logging.getLogger(__name__) 

74_ResultT = TypeVar("_ResultT") 

75 

76 

77class ExecutionOwner: 

78 """Serialize async callers over a single worker-confined execution resource set.""" 

79 

80 def __init__(self) -> None: 

81 self._executor = concurrent.futures.ThreadPoolExecutor( 

82 max_workers=1, thread_name_prefix="execution-owner" 

83 ) 

84 self._gate = asyncio.Lock() 

85 self._stop_event = threading.Event() 

86 self._resource: ExecutionResources | None = None 

87 self._closer: Callable[[ExecutionResources], None] | None = None 

88 self._data_dir = Path(".") 

89 self._closing = False 

90 self._closed = False 

91 self._unusable = False 

92 self._close_task: asyncio.Task[None] | None = None 

93 self._order_stream: OrderStream | None = None 

94 

95 @classmethod 

96 async def open( 

97 cls, 

98 data_dir: Path, 

99 credentials: AlpacaCredentials, 

100 config: CopyConfig, 

101 *, 

102 environment: Environment, 

103 broker_factory: Callable[[AlpacaCredentials, Environment], Broker] | None = None, 

104 account_lock_root: Path | None = None, 

105 observer: ExecutionObserver | None = None, 

106 ) -> ExecutionOwner: 

107 data_dir.mkdir(parents=True, exist_ok=True) 

108 if environment not in {"paper", "live"}: 108 ↛ 109line 108 didn't jump to line 109 because the condition on line 108 was never true

109 raise ValueError("Choose Alpaca paper or live explicitly") 

110 lock_root = account_lock_root or default_account_lock_root() 

111 if not lock_root.is_absolute(): 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true

112 raise ValueError("Account lock root must be an absolute path") 

113 factory = broker_factory or (lambda keys, mode: AlpacaBroker(keys, mode)) 

114 return await cls._from_resource_factory( 

115 lambda: build_resources( 

116 data_dir, 

117 credentials, 

118 config, 

119 environment, 

120 factory, 

121 lock_root, 

122 observer if observer is not None else NoOpObserver(), 

123 ), 

124 lambda resource: resource.close(), 

125 data_dir=data_dir, 

126 # Only a real Alpaca account has a live stream; test brokers never open one. 

127 order_stream=( 

128 alpaca_order_stream(credentials, environment) if broker_factory is None else None 

129 ), 

130 ) 

131 

132 async def watch_orders( 

133 self, on_update: Callable[[], None], on_live: Callable[[bool], None], stop: asyncio.Event 

134 ) -> None: 

135 """Call `on_update` whenever Alpaca reports a change to one of this account's orders, and 

136 `on_live` as the stream connects and drops.""" 

137 if self._order_stream is not None: 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true

138 await self._order_stream(on_update, on_live, stop) 

139 

140 @classmethod 

141 async def _from_resource_factory( 

142 cls, 

143 factory: Callable[[], ExecutionResources], 

144 closer: Callable[[ExecutionResources], None], 

145 *, 

146 data_dir: Path = Path("."), 

147 order_stream: OrderStream | None = None, 

148 ) -> ExecutionOwner: 

149 """Shared worker bootstrap used by production and the thread-confinement probe.""" 

150 owner = cls() 

151 owner._data_dir = data_dir 

152 owner._order_stream = order_stream 

153 owner._closer = closer 

154 future = owner._executor.submit(factory) 

155 result, cancelled, failure = await owner._drain(future) 

156 if cancelled: 

157 if failure is None: 157 ↛ 163line 157 didn't jump to line 163 because the condition on line 157 was always true

158 owner._resource = result 

159 cleanup = owner._executor.submit(owner._close_resource) 

160 _, _, cleanup_failure = await owner._drain(cleanup) 

161 if cleanup_failure is not None: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 log.warning("startup_cleanup_failed error=%s", type(cleanup_failure).__name__) 

163 owner._closed = True 

164 owner._executor.shutdown(wait=True) 

165 raise asyncio.CancelledError 

166 if failure is not None: 

167 owner._closed = True 

168 owner._executor.shutdown(wait=True) 

169 raise failure 

170 owner._resource = result 

171 return owner 

172 

173 async def _drain( 

174 self, future: concurrent.futures.Future[_ResultT] 

175 ) -> tuple[_ResultT | None, bool, BaseException | None]: 

176 """Shield one worker call until completion, retaining caller cancellation.""" 

177 cancelled = False 

178 wrapped = asyncio.wrap_future(future) 

179 while True: 

180 try: 

181 await asyncio.shield(wrapped) 

182 break 

183 except asyncio.CancelledError: 

184 cancelled = True 

185 if future.done(): 185 ↛ 186line 185 didn't jump to line 186 because the condition on line 185 was never true

186 break 

187 except BaseException: # noqa: BLE001 - keep draining until the worker settles 

188 break 

189 try: 

190 return future.result(), cancelled, None 

191 except BaseException as exc: # noqa: BLE001 - keep draining until the worker settles 

192 return None, cancelled, exc 

193 

194 def _close_resource(self) -> None: 

195 if self._resource is not None and self._closer is not None: 195 ↛ exitline 195 didn't return from function '_close_resource' because the condition on line 195 was always true

196 self._closer(self._resource) 

197 

198 async def _submit( 

199 self, 

200 operation: Callable[[ExecutionResources], _ResultT], 

201 ) -> _ResultT: 

202 async with self._gate: 

203 if self._closing or self._closed or self._unusable or self._resource is None: 

204 raise AccountOwnerClosed("Execution owner is closed or unusable") 

205 future = self._executor.submit(operation, self._resource) 

206 result, cancelled, failure = await self._drain(future) 

207 if cancelled: 

208 if not self._resource.store.usable: 208 ↛ 209line 208 didn't jump to line 209 because the condition on line 208 was never true

209 self._unusable = True 

210 if failure is not None: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true

211 log.warning("execution_worker_failed error=%s", type(failure).__name__) 

212 raise asyncio.CancelledError 

213 if failure is not None: 

214 if not self._resource.store.usable: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true

215 self._unusable = True 

216 raise failure 

217 return cast(_ResultT, result) 

218 

219 def request_stop(self) -> None: 

220 """Make pre-submit stop checks visible without queueing behind broker work.""" 

221 self._stop_event.set() 

222 

223 async def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None: 

224 await self._submit(lambda resource: resource.receive(delivery, now)) 

225 

226 async def record_rejection(self, payload_hash: str, reason: str, now: dt.datetime) -> None: 

227 await self._submit(lambda resource: resource.record_rejection(payload_hash, reason, now)) 

228 

229 async def cycle(self, now: dt.datetime, *, halted: bool) -> ExecutionObservation: 

230 return await self._submit( 

231 lambda resource: resource.cycle(now, halted=halted, stopping=self._stop_event) 

232 ) 

233 

234 async def observation(self) -> ExecutionObservation: 

235 return await self._submit(lambda resource: resource.observation()) 

236 

237 async def pending_notifications(self) -> tuple[tuple[int, str, str, dict[str, object]], ...]: 

238 return await self._submit(lambda resource: resource.pending_notifications()) 

239 

240 async def confirm_notification(self, notification_id: int) -> None: 

241 await self._submit(lambda resource: resource.confirm_notification(notification_id)) 

242 

243 async def release_uncertain_intent( 

244 self, 

245 client_id: str, 

246 *, 

247 actor: str, 

248 reason: str, 

249 order_history_ref: str, 

250 fill_history_ref: str, 

251 account_id: str, 

252 ) -> ReleaseEvidence: 

253 return await self._submit( 

254 lambda resource: resource.release_uncertain_intent( 

255 client_id, 

256 actor=actor, 

257 reason=reason, 

258 order_history_ref=order_history_ref, 

259 fill_history_ref=fill_history_ref, 

260 account_id=account_id, 

261 ) 

262 ) 

263 

264 async def record_manual_sale(self, sale: ManualSale, *, account_id: str) -> None: 

265 await self._submit( 

266 lambda resource: resource.record_manual_sale(sale, account_id=account_id) 

267 ) 

268 

269 async def record_manual_correction( 

270 self, correction: ManualCorrectionRecord 

271 ) -> ManualCorrectionRecord: 

272 return await self._submit( 

273 lambda resource: resource.record_manual_correction( 

274 correction, stopping=self._stop_event.is_set 

275 ) 

276 ) 

277 

278 async def preview_lot_sale( 

279 self, request: LotSalePreviewRequest, now: dt.datetime 

280 ) -> LotSalePreview: 

281 return await self._submit( 

282 lambda resource: resource.preview_lot_sale( 

283 request, now, stopping=self._stop_event.is_set 

284 ) 

285 ) 

286 

287 async def confirm_lot_sale( 

288 self, request: LotSaleConfirmation, now: dt.datetime 

289 ) -> LotSaleResult: 

290 return await self._submit( 

291 lambda resource: resource.confirm_lot_sale( 

292 request, now, stopping=self._stop_event.is_set 

293 ) 

294 ) 

295 

296 async def preview_manual_order( 

297 self, request: ManualPreviewRequest, now: dt.datetime 

298 ) -> ManualOrderPreview: 

299 return await self._submit( 

300 lambda resource: resource.preview_manual_order( 

301 request, now, stopping=self._stop_event.is_set 

302 ) 

303 ) 

304 

305 async def confirm_manual_order( 

306 self, request: ManualConfirmationRequest, now: dt.datetime 

307 ) -> ManualCommandResult: 

308 return await self._submit( 

309 lambda resource: resource.confirm_manual_order( 

310 request, now, stopping=self._stop_event.is_set 

311 ) 

312 ) 

313 

314 async def manual_command_result(self, command_id: str) -> ManualCommandResult: 

315 return await self._submit( 

316 lambda resource: resource.manual_command_result( 

317 command_id, stopping=self._stop_event.is_set 

318 ) 

319 ) 

320 

321 async def manual_command_page( 

322 self, 

323 source_id: str, 

324 *, 

325 before_command_id: str | None = None, 

326 limit: int = 50, 

327 ) -> ManualCommandPage: 

328 return await self._submit( 

329 lambda resource: resource.manual_command_page( 

330 source_id, 

331 before_command_id=before_command_id, 

332 limit=limit, 

333 stopping=self._stop_event.is_set, 

334 ) 

335 ) 

336 

337 async def inventory_account(self, now: dt.datetime) -> OwnershipInspection: 

338 return await self._submit(lambda resource: resource.inventory_account(now)) 

339 

340 async def inspect_ownership(self) -> OwnershipInspection: 

341 return await self._submit(lambda resource: resource.inspect_ownership()) 

342 

343 async def resolve_ownership(self, request: OwnershipResolutionRequest) -> OwnershipResolution: 

344 return await self._submit(lambda resource: resource.resolve_ownership(request)) 

345 

346 async def recover_account(self, now: dt.datetime) -> OwnershipInspection: 

347 return await self._submit(lambda resource: resource.recover_account(now)) 

348 

349 async def control_account( 

350 self, command: AccountControlCommand, now: dt.datetime 

351 ) -> AccountControlResult: 

352 return await self._submit(lambda resource: resource.control_account(command, now)) 

353 

354 async def update_config(self, config: CopyConfig, now: dt.datetime) -> tuple[LimitChange, ...]: 

355 """Swap the limits between two cycles: the worker runs one operation at a time.""" 

356 return await self._submit(lambda resource: resource.update_config(config, now)) 

357 

358 async def account_status(self) -> AccountRuntimeView: 

359 return await self._submit(lambda resource: resource.account_status()) 

360 

361 async def operator_overview(self) -> AccountOverview: 

362 return await self._submit(lambda resource: resource.operator_overview()) 

363 

364 async def destination_views(self, source_ids: set[str]) -> dict[str, DestinationView]: 

365 return await self._submit(lambda resource: resource.destination_views(source_ids)) 

366 

367 async def event_page(self, before_seq: int | None, limit: int) -> AccountEventPage: 

368 return await self._submit(lambda resource: resource.event_page(before_seq, limit)) 

369 

370 async def feed_page(self, before_seq: int | None, limit: int) -> AccountFeedPage: 

371 return await self._submit(lambda resource: resource.feed_page(before_seq, limit)) 

372 

373 async def equity_history(self, window: HistoryWindow, now: dt.datetime) -> EquityHistory: 

374 return await self._submit(lambda resource: resource.equity_history(window, now)) 

375 

376 async def clear_late_order_incident( 

377 self, 

378 client_id: str, 

379 *, 

380 actor: str, 

381 reason: str, 

382 matched_audit_ref: str, 

383 account_id: str, 

384 ) -> IncidentClearanceEvidence: 

385 return await self._submit( 

386 lambda resource: resource.clear_late_order_incident( 

387 client_id, 

388 actor=actor, 

389 reason=reason, 

390 matched_audit_ref=matched_audit_ref, 

391 account_id=account_id, 

392 ) 

393 ) 

394 

395 async def _close_resources(self) -> None: 

396 failure: BaseException | None = None 

397 async with self._gate: 

398 if self._closed: 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true

399 return 

400 future = self._executor.submit(self._close_resource) 

401 _, cancelled, failure = await self._drain(future) 

402 self._closed = True 

403 self._unusable = True 

404 self._executor.shutdown(wait=True) 

405 if failure is not None: 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true

406 raise failure 

407 if cancelled: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 raise asyncio.CancelledError 

409 

410 async def close(self) -> None: 

411 if self._close_task is None: 

412 if self._closed: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true

413 return 

414 self._closing = True 

415 self.request_stop() 

416 self._close_task = asyncio.create_task(self._close_resources()) 

417 

418 cleanup = self._close_task 

419 cancelled = False 

420 while True: 

421 try: 

422 await asyncio.shield(cleanup) 

423 break 

424 except asyncio.CancelledError: 

425 cancelled = True 

426 if cleanup.done(): 426 ↛ 427line 426 didn't jump to line 427 because the condition on line 426 was never true

427 break 

428 except BaseException: # noqa: BLE001 - keep draining until the worker settles 

429 break 

430 cleanup.result() 

431 if cancelled: 

432 raise asyncio.CancelledError