Coverage for src/copytrading_engine/execution/adapters/resources.py: 89%
275 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""The engine, broker, store, and locks of one account, used only on its owner's worker thread."""
3import datetime as dt
4import fcntl
5import hashlib
6import logging
7import os
8import pwd
9import threading
10from collections.abc import Callable
11from contextlib import ExitStack
12from copy import deepcopy
13from dataclasses import dataclass, field
14from pathlib import Path
16from copytrading_engine.execution.adapters.alpaca.broker import (
17 AlpacaCredentials,
18 Environment,
19)
20from copytrading_engine.execution.adapters.sqlite_ledger import Store
21from copytrading_engine.execution.application.engine import CopyEngine
22from copytrading_engine.execution.application.lot_sales import LotSaleApplication
23from copytrading_engine.execution.application.manual_commands import ManualTradingApplication
24from copytrading_engine.execution.application.ports import (
25 AccountOpenRefused,
26 AccountRuntimeView,
27 Broker,
28 BrokerError,
29 EquityHistoryBroker,
30 ExecutionObservation,
31 ExecutionObserver,
32)
33from copytrading_engine.execution.application.recovery import RecoveryApplication
34from copytrading_engine.execution.domain.events import (
35 JournalEvent,
36 LimitChange,
37 LimitsChanged,
38 SignalRejected,
39)
40from copytrading_engine.execution.domain.lifecycle import (
41 AccountControlCommand,
42 AccountControlResult,
43)
44from copytrading_engine.execution.domain.lot_sales import (
45 LotSaleConfirmation,
46 LotSalePreview,
47 LotSalePreviewRequest,
48 LotSaleResult,
49)
50from copytrading_engine.execution.domain.manual_commands import (
51 ManualCommandPage,
52 ManualCommandResult,
53 ManualConfirmationRequest,
54 ManualCorrectionRecord,
55 ManualOrderPreview,
56 ManualPreviewRequest,
57)
58from copytrading_engine.execution.domain.market import Account, EquityHistory, HistoryWindow
59from copytrading_engine.execution.domain.ownership import (
60 OwnershipInspection,
61 OwnershipResolution,
62 OwnershipResolutionRequest,
63)
64from copytrading_engine.execution.domain.positions import PositionAudit
65from copytrading_engine.execution.domain.recovery import (
66 IncidentClearanceEvidence,
67 ManualSale,
68 ReleaseEvidence,
69)
70from copytrading_engine.execution.domain.signals import CopyConfig
71from copytrading_engine.execution.domain.sizing import DestinationSignal
72from copytrading_engine.execution.domain.values import BrokerAccountNumber
73from copytrading_engine.execution.presentation.account_feed import AccountFeedPage, account_feed
74from copytrading_engine.execution.presentation.operator_views import (
75 AccountEventPage,
76 AccountOverview,
77 DestinationView,
78 account_balance,
79 account_overview,
80 destination_views,
81 event_page,
82)
83from copytrading_engine.shared.owner_facing import OwnerFacingError
85# A ledger not yet bound to a broker account matches no broker account, so manual orders and
86# lot sales refuse until the account has connected.
87_UNBOUND_LEDGER = BrokerAccountNumber("unbound")
89log = logging.getLogger(__name__)
92_identity_guard = threading.Lock()
95_active_identities: set[tuple[Environment, str]] = set()
98_EQUITY_HISTORY_REUSE = dt.timedelta(seconds=60)
101def default_account_lock_root() -> Path:
102 """Stable per-user host lock directory, independent of an installation's state root.
104 This file lock coordinates processes of the same OS user on one host. It is
105 not a distributed lease across users or machines.
106 """
107 # HOME may be overridden independently in two processes owned by this user.
108 user_home = Path(pwd.getpwuid(os.getuid()).pw_dir)
109 return user_home / ".copytrading" / "account-locks"
112def _reserve_identity(environment: Environment, account_id: str) -> Callable[[], None]:
113 identity = (environment, account_id)
114 with _identity_guard:
115 if identity in _active_identities:
116 raise AccountOpenRefused("account_in_use")
117 _active_identities.add(identity)
119 def release() -> None:
120 with _identity_guard:
121 _active_identities.remove(identity)
123 return release
126# Entries the owner turned off or paused skip a buy at once. A restart's recovery only holds it:
127# each cycle tries again, and the signal-age check decides when a post is too old.
128_DELIBERATE_BLOCKS = frozenset({"account_paused", "account_disabled"})
131@dataclass
132class ExecutionResources:
133 stack: ExitStack
134 store: Store
135 broker: Broker
136 engine: CopyEngine
137 account: Account
138 data_dir: Path
139 account_observed_at: dt.datetime
140 position_audit: PositionAudit | None = None
141 _recovery_ready: bool = False
142 last_inspection: OwnershipInspection | None = None
143 _equity_histories: dict[HistoryWindow, tuple[dt.datetime, EquityHistory]] = field(
144 default_factory=dict
145 )
147 @property
148 def runtime_entries_ready(self) -> bool:
149 """Derive buy readiness from current-session recovery and durable permission."""
150 control = self.engine.ledger.state.control
151 return self._recovery_ready and control.entry_permission == "enabled"
153 def account_status(self) -> AccountRuntimeView:
154 control = self.engine.ledger.state.control
155 inspection = self.last_inspection
156 if control.entry_permission == "paused":
157 readiness = "paused"
158 elif control.entry_permission == "disabled":
159 readiness = "disabled"
160 elif not self.runtime_entries_ready:
161 readiness = (
162 "manual_resume_required"
163 if self.engine.config.recovery_preference == "manual"
164 else "recovery_pending"
165 )
166 elif inspection is None or not self._inspection_ready(inspection): 166 ↛ 167line 166 didn't jump to line 167 because the condition on line 166 was never true
167 readiness = "reconciliation_unavailable"
168 else:
169 try:
170 if self.engine.market_session(dt.datetime.now(dt.UTC)).value != "closed":
171 readiness = "enabled"
172 elif self.engine.ledger.state.environment == "paper":
173 # A paper account copies coins at every hour (ADR-0014); only its stocks wait.
174 readiness = "enabled_coins_only"
175 else:
176 readiness = "enabled_waiting_for_session"
177 except Exception: # noqa: BLE001 - readiness degrades instead of failing status
178 readiness = "session_unavailable"
179 return AccountRuntimeView(
180 entry_permission=control.entry_permission,
181 recovery_preference=self.engine.config.recovery_preference,
182 readiness=readiness,
183 account_risk_status=inspection.account_risk_status if inspection else "unavailable",
184 account_risk_reason=inspection.account_risk_reason if inspection else "not_checked",
185 account_activity_status=inspection.account_activity_status
186 if inspection
187 else "unavailable",
188 account_activity_reason=inspection.account_activity_reason
189 if inspection
190 else "not_checked",
191 )
193 def operator_overview(self) -> AccountOverview:
194 return account_overview(
195 self.engine.ledger.snapshot(),
196 self.last_inspection,
197 local_account_id=self.data_dir.name,
198 active_configuration=True,
199 readiness=self.account_status().readiness,
200 balance=account_balance(self.account, self.account_observed_at),
201 recovery_preference=self.engine.config.recovery_preference,
202 )
204 def destination_views(self, source_ids: set[str]) -> dict[str, DestinationView]:
205 return destination_views(
206 self.engine.ledger.snapshot(), source_ids, self.store.message_events(source_ids)
207 )
209 def equity_history(self, window: HistoryWindow, now: dt.datetime) -> EquityHistory:
210 """The broker's curve, fetched at most once a minute so charts never crowd out trading."""
211 if not isinstance(self.broker, EquityHistoryBroker): 211 ↛ 212line 211 didn't jump to line 212 because the condition on line 211 was never true
212 raise RuntimeError("This broker does not report equity history")
213 cached = self._equity_histories.get(window)
214 if cached is not None and now - cached[0] < _EQUITY_HISTORY_REUSE:
215 return cached[1]
216 history = self.broker.equity_history(window)
217 self._equity_histories[window] = (now, history)
218 return history
220 def event_page(self, before_seq: int | None, limit: int) -> AccountEventPage:
221 return event_page(self.data_dir.name, self.store.event_page(before_seq, limit), limit)
223 def feed_page(self, before_seq: int | None, limit: int) -> AccountFeedPage:
224 return account_feed(
225 self.data_dir.name,
226 self.engine.ledger.snapshot(),
227 self.store.feed_events(before_seq, limit),
228 limit,
229 )
231 def entry_block_reason(self) -> str | None:
232 control = self.engine.ledger.state.control
233 if control.entry_permission == "paused":
234 return "account_paused"
235 if control.entry_permission == "disabled":
236 return "account_disabled"
237 if not self.runtime_entries_ready:
238 # Manual recovery waits for the owner's Resume; automatic waits for the checks.
239 return (
240 "manual_resume_required"
241 if self.engine.config.recovery_preference == "manual"
242 else "recovery_pending"
243 )
244 return None
246 def recover_account(self, now: dt.datetime) -> OwnershipInspection:
247 self.engine.reconcile(now)
248 inspection = self.inspect_ownership()
249 self.last_inspection = inspection
250 self.account = self.broker.account()
251 self.account_observed_at = now
252 control = self.engine.ledger.state.control
253 self._recovery_ready = (
254 control.entry_permission in {"enabled", "paused"}
255 and self.engine.config.recovery_preference == "automatic"
256 and self._inspection_ready(inspection)
257 )
258 # A deliberate pause or entries left off skip queued buys; recovery only holds them, and
259 # each cycle's signal-age check decides when one is too old.
260 reason = self.entry_block_reason()
261 if reason in _DELIBERATE_BLOCKS:
262 self.engine.ledger.skip_unpermitted_buys(now, reason)
263 return inspection
265 def _inspection_ready(self, inspection: OwnershipInspection) -> bool:
266 return (
267 inspection.account_id == self.account.id
268 and self.account.active
269 and inspection.account_risk_status == "ready"
270 and inspection.account_activity_status == "ready"
271 and not self.engine.ledger.state.buy_halted
272 )
274 def _resume_refusal(self, inspection: OwnershipInspection) -> str:
275 """Why new buys can't resume yet, as a code the app words in the owner's language."""
276 if inspection.account_activity_reason == "unresolved_account_order": 276 ↛ 278line 276 didn't jump to line 278 because the condition on line 276 was always true
277 return "resume_blocked_outside_order"
278 if not self.account.active:
279 return "resume_blocked_account_inactive"
280 if self.engine.ledger.state.buy_halted:
281 return "resume_blocked_order_unclear"
282 return "resume_blocked_unreadable"
284 def control_account(
285 self, command: AccountControlCommand, now: dt.datetime
286 ) -> AccountControlResult:
287 previous = self.engine.ledger.state.control.commands.get(command.command_id)
288 if previous is not None:
289 return self.engine.ledger.account_control(
290 command, now, local_account_id=self.data_dir.name
291 )
292 if command.action == "resume":
293 self.engine.reconcile(now)
294 inspection = self.inspect_ownership()
295 self.last_inspection = inspection
296 self.account = self.broker.account()
297 self.account_observed_at = now
298 if not self._inspection_ready(inspection):
299 raise OwnerFacingError(self._resume_refusal(inspection))
300 result = self.engine.ledger.account_control(
301 command, now, local_account_id=self.data_dir.name
302 )
303 if command.action == "pause":
304 self.engine.ledger.skip_unpermitted_buys(now, "account_paused")
305 elif command.action == "resume": 305 ↛ 307line 305 didn't jump to line 307 because the condition on line 305 was always true
306 self._recovery_ready = True
307 return result
309 def update_config(self, config: CopyConfig, now: dt.datetime) -> tuple[LimitChange, ...]:
310 """Copy with new limits from the next cycle on, and journal what changed."""
311 before, after = _flat_limits(self.engine.config), _flat_limits(config)
312 changes = tuple(
313 LimitChange(setting=name, before=_text(before[name]), after=_text(value))
314 for name, value in after.items()
315 if before[name] != value
316 )
317 if config.sources != self.engine.config.sources:
318 raise ValueError("A limits update cannot change the account's sources")
319 if not changes:
320 return ()
321 self.engine.ledger.record(JournalEvent(at=now, payload=LimitsChanged(changes=changes)))
322 self.engine.config = config
323 return changes
325 def observation(self) -> ExecutionObservation:
326 return ExecutionObservation(
327 account=deepcopy(self.account),
328 ledger=self.engine.ledger.snapshot(),
329 total_cost_exposure_usd=self.engine.ledger.exposure(),
330 position_audit=deepcopy(self.position_audit),
331 )
333 def cycle(
334 self,
335 now: dt.datetime,
336 *,
337 halted: bool,
338 stopping: threading.Event,
339 ) -> ExecutionObservation:
340 # `now` is when the cycle was asked for, and work queued ahead of it (an approved order,
341 # say) may have run since: every step records the time it runs, never an earlier one.
342 started_at = _not_before(now)
343 self.engine.reconcile(started_at)
344 self.engine.process(
345 now,
346 halted=halted,
347 now_clock=lambda: dt.datetime.now(dt.UTC),
348 halted_now=lambda: (self.data_dir / "HALT").exists(),
349 entry_block_reason=self.entry_block_reason,
350 stopping=stopping.is_set,
351 )
352 # Orders this cycle placed can fill before it ends: book those fills before the broker is
353 # read, so a buy shows as copied and a sale as gone in the same cycle, never as shares at
354 # the broker nobody copied or shares sold that still count. With nothing pending this
355 # costs no broker call. A fill is booked at the time it is read, after the order it fills;
356 # a holdings question this cycle opened still waits for the next cycle's sync to settle.
357 settled_at = _not_before(started_at)
358 positions = self.engine.reconcile(settled_at, settle_before=started_at)
359 self.position_audit = self.engine.audit_positions(positions)
360 self.account = self.broker.account()
361 self.account_observed_at = settled_at
362 self.last_inspection = RecoveryApplication(
363 self.engine.ledger, self.broker
364 ).inspect_ownership(account=self.account, positions=positions)
365 return self.observation()
367 def receive(self, delivery: DestinationSignal, now: dt.datetime) -> None:
368 self.engine.receive(delivery, now)
369 if (reason := self.entry_block_reason()) in _DELIBERATE_BLOCKS:
370 self.engine.ledger.skip_unpermitted_buys(now, reason)
372 def record_rejection(self, payload_hash: str, reason: str, now: dt.datetime) -> None:
373 self.engine.ledger.record(
374 JournalEvent(
375 at=now,
376 payload=SignalRejected(payload_hash=payload_hash, reason=reason),
377 )
378 )
380 def pending_notifications(self) -> tuple[tuple[int, str, str, dict[str, object]], ...]:
381 return deepcopy(self.store.pending_notifications())
383 def confirm_notification(self, notification_id: int) -> None:
384 self.store.confirm_notification(notification_id)
386 def release_uncertain_intent(
387 self,
388 client_id: str,
389 *,
390 actor: str,
391 reason: str,
392 order_history_ref: str,
393 fill_history_ref: str,
394 account_id: str,
395 ) -> ReleaseEvidence:
396 return RecoveryApplication(self.engine.ledger, self.broker).release_uncertain_intent(
397 client_id,
398 actor=actor,
399 reason=reason,
400 order_history_ref=order_history_ref,
401 fill_history_ref=fill_history_ref,
402 account_id=account_id,
403 )
405 def record_manual_sale(self, sale: ManualSale, *, account_id: str) -> None:
406 RecoveryApplication(self.engine.ledger, self.broker).record_manual_sale(
407 sale, account_id=account_id
408 )
410 def _owner_orders_ready(self) -> bool:
411 """Owner-placed orders need finished recovery, a ready inspection, and an account that
412 is not disabled."""
413 inspection_ready = self.last_inspection is not None and self._inspection_ready(
414 self.last_inspection
415 )
416 return (
417 self._recovery_ready
418 and inspection_ready
419 and self.engine.ledger.state.control.entry_permission in {"enabled", "paused"}
420 )
422 def _manual_application(self, stopping: Callable[[], bool]) -> ManualTradingApplication:
423 snapshot = self.engine.ledger.state
424 manual_orders_ready = self._owner_orders_ready()
425 return ManualTradingApplication(
426 self.engine,
427 local_account_id=self.data_dir.name,
428 broker_account_id=snapshot.account_id or _UNBOUND_LEDGER,
429 environment=snapshot.environment or "paper",
430 halted=lambda: (self.data_dir / "HALT").exists() or self.engine.ledger.state.buy_halted,
431 entry_block_reason=self.entry_block_reason,
432 recovery_ready=lambda: manual_orders_ready,
433 stopping=stopping,
434 )
436 def _lot_sale_application(self, stopping: Callable[[], bool]) -> LotSaleApplication:
437 snapshot = self.engine.ledger.state
438 orders_ready = self._owner_orders_ready()
439 return LotSaleApplication(
440 self.engine,
441 local_account_id=self.data_dir.name,
442 broker_account_id=snapshot.account_id or _UNBOUND_LEDGER,
443 environment=snapshot.environment or "paper",
444 halted=lambda: (self.data_dir / "HALT").exists() or self.engine.ledger.state.buy_halted,
445 entry_block_reason=self.entry_block_reason,
446 recovery_ready=lambda: orders_ready,
447 stopping=stopping,
448 )
450 def preview_lot_sale(
451 self, request: LotSalePreviewRequest, now: dt.datetime, *, stopping: Callable[[], bool]
452 ) -> LotSalePreview:
453 return self._lot_sale_application(stopping).preview(request, now)
455 def confirm_lot_sale(
456 self, request: LotSaleConfirmation, now: dt.datetime, *, stopping: Callable[[], bool]
457 ) -> LotSaleResult:
458 return self._lot_sale_application(stopping).confirm(request, now)
460 def record_manual_correction(
461 self, correction: ManualCorrectionRecord, *, stopping: Callable[[], bool]
462 ) -> ManualCorrectionRecord:
463 return self._manual_application(stopping).record_correction(correction)
465 def preview_manual_order(
466 self, request: ManualPreviewRequest, now: dt.datetime, *, stopping: Callable[[], bool]
467 ) -> ManualOrderPreview:
468 return self._manual_application(stopping).preview(request, now)
470 def confirm_manual_order(
471 self,
472 request: ManualConfirmationRequest,
473 now: dt.datetime,
474 *,
475 stopping: Callable[[], bool],
476 ) -> ManualCommandResult:
477 return self._manual_application(stopping).confirm(request, now)
479 def manual_command_result(
480 self, command_id: str, *, stopping: Callable[[], bool]
481 ) -> ManualCommandResult:
482 return self._manual_application(stopping).result(command_id)
484 def manual_command_page(
485 self,
486 source_id: str,
487 *,
488 before_command_id: str | None,
489 limit: int,
490 stopping: Callable[[], bool],
491 ) -> ManualCommandPage:
492 return self._manual_application(stopping).command_page(
493 source_id, before_command_id=before_command_id, limit=limit
494 )
496 def inventory_account(self, now: dt.datetime) -> OwnershipInspection:
497 self.account = self.engine.bind(
498 now, environment=self.engine.ledger.state.environment or "paper"
499 )
500 self.account_observed_at = now
501 return RecoveryApplication(self.engine.ledger, self.broker).inspect_ownership()
503 def inspect_ownership(self) -> OwnershipInspection:
504 return RecoveryApplication(self.engine.ledger, self.broker).inspect_ownership()
506 def resolve_ownership(self, request: OwnershipResolutionRequest) -> OwnershipResolution:
507 result = RecoveryApplication(self.engine.ledger, self.broker).resolve_ownership(request)
508 self.last_inspection = self.inspect_ownership()
509 return result
511 def clear_late_order_incident(
512 self,
513 client_id: str,
514 *,
515 actor: str,
516 reason: str,
517 matched_audit_ref: str,
518 account_id: str,
519 ) -> IncidentClearanceEvidence:
520 return RecoveryApplication(self.engine.ledger, self.broker).clear_late_order_incident(
521 client_id,
522 actor=actor,
523 reason=reason,
524 matched_audit_ref=matched_audit_ref,
525 account_id=account_id,
526 )
528 def close(self) -> None:
529 try:
530 if not self.store.usable: 530 ↛ 531line 530 didn't jump to line 531 because the condition on line 530 was never true
531 return
532 now = dt.datetime.now(dt.UTC)
533 for order in self.engine.pending():
534 if order.type == "stop": 534 ↛ 536line 534 didn't jump to line 536 because the condition on line 534 was always true
535 continue # a stop keeps guarding the shares at Alpaca while copying is off
536 try:
537 self.engine.cancel(order, now, "copying_stopped")
538 except BrokerError as exc:
539 log.warning("shutdown_cancel_unconfirmed error=%s", type(exc).__name__)
540 try:
541 self.engine.reconcile(dt.datetime.now(dt.UTC))
542 except BrokerError as exc:
543 log.warning("shutdown_reconcile_unavailable error=%s", type(exc).__name__)
544 finally:
545 self.stack.close()
548def build_resources(
549 data_dir: Path,
550 credentials: AlpacaCredentials,
551 config: CopyConfig,
552 environment: Environment,
553 broker_factory: Callable[[AlpacaCredentials, Environment], Broker],
554 account_lock_root: Path,
555 observer: ExecutionObserver,
556) -> ExecutionResources:
557 """Acquire the entire synchronous core on the owner thread with rollback cleanup."""
558 stack = ExitStack()
559 try:
560 ownership = stack.enter_context((data_dir / "executor.lock").open("w"))
561 try:
562 fcntl.flock(ownership.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
563 except OSError:
564 raise RuntimeError("Another executor owns this state directory") from None
565 broker = broker_factory(credentials, environment)
566 close_broker = getattr(broker, "close", None)
567 if close_broker is not None:
568 stack.callback(close_broker)
569 account = broker.account()
570 stack.callback(_reserve_identity(environment, account.id))
571 account_key = hashlib.sha256(f"{environment}:{account.id}".encode()).hexdigest()
572 account_lock_root.mkdir(parents=True, exist_ok=True, mode=0o700)
573 identity_lock = stack.enter_context(
574 (account_lock_root / f".execution-account-{account_key}.lock").open("w")
575 )
576 try:
577 fcntl.flock(identity_lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
578 except OSError:
579 raise AccountOpenRefused("account_in_use") from None
580 store = Store(data_dir / "execution.sqlite3")
581 stack.callback(store.close)
582 store.bind_identity(account.id, environment)
583 engine = CopyEngine(store, broker, config, observer=observer)
584 engine.bind(dt.datetime.now(dt.UTC), account=account, environment=environment)
585 return ExecutionResources(
586 stack.pop_all(), store, broker, engine, account, data_dir, dt.datetime.now(dt.UTC)
587 )
588 except BaseException:
589 stack.close()
590 raise
593def _not_before(at: dt.datetime) -> dt.datetime:
594 """The time now, or `at` when a caller's clock runs ahead of it."""
595 return max(at, dt.datetime.now(dt.UTC))
598def _flat_limits(config: CopyConfig) -> dict[str, object]:
599 """Every limit by its account policy name, the pricing ones included."""
600 return config.model_dump(exclude={"sources", "pricing"}) | config.pricing.model_dump()
603def _text(value: object) -> str:
604 return str(value).lower() if isinstance(value, bool) else str(value)