Coverage for src/copytrading_engine/trading/entrypoints/runtime.py: 90%
546 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Explicit trading lifecycle and replay-safe local pipeline composition."""
3import asyncio
4import dataclasses
5import datetime as dt
6import logging
7import os
8from collections import deque
9from collections.abc import AsyncGenerator, Callable, Collection, Mapping
10from contextlib import asynccontextmanager, nullcontext
11from dataclasses import replace
12from functools import partial
13from pathlib import Path
14from uuid import uuid4
16from copytrading_engine.backup.restore.gate import restore_manual_disabled
17from copytrading_engine.execution.adapters.alpaca.broker import AlpacaBroker, AlpacaCredentials
18from copytrading_engine.execution.application.ports import AccountOpenRefused, BrokerError
19from copytrading_engine.execution.domain.pricing import PricePolicy
20from copytrading_engine.execution.domain.signals import CopyConfig
21from copytrading_engine.execution.domain.sizing import DestinationTerms
22from copytrading_engine.parsing.providers.registry import ManagedDecoder
23from copytrading_engine.parsing.readiness import probe_model
24from copytrading_engine.parsing.sqlite import SQLiteExtractionStore
25from copytrading_engine.parsing.worker import ParseWorker
26from copytrading_engine.shared.notification_models import NotificationPayload
27from copytrading_engine.shared.raw_message import RawMessage
28from copytrading_engine.shared.route_keys import route_key
29from copytrading_engine.sources.application import Delivery, ForwardBatch
30from copytrading_engine.sources.session import token_rejected
31from copytrading_engine.sources.sqlite import SQLiteSourceStore
32from copytrading_engine.trading.adapters.account_records import (
33 BrokerIdentity,
34 reconcile_records,
35)
36from copytrading_engine.trading.adapters.activation import (
37 TradingActivationJournal,
38 TradingActivationStatus,
39)
40from copytrading_engine.trading.adapters.capabilities import (
41 CapabilityCheck,
42 TradingCapabilityReport,
43 TradingCapabilityService,
44)
45from copytrading_engine.trading.adapters.channel_history import DiscordChannelHistory
46from copytrading_engine.trading.adapters.operator_queries import SQLiteOperatorEvidence
47from copytrading_engine.trading.adapters.routing import (
48 AccountOwnershipPending,
49 RoutingChangePending,
50 RoutingRevision,
51)
52from copytrading_engine.trading.adapters.telemetry import TradingTelemetry
53from copytrading_engine.trading.application.account_access import AccountAccess
54from copytrading_engine.trading.application.accounts import (
55 AccountOwner,
56 AccountSupervisor,
57 SignalFanout,
58 account_failure,
59)
60from copytrading_engine.trading.application.alerts import AlertRelay, account_stopped_alert
61from copytrading_engine.trading.application.manual_intervention import ManualInterventionService
62from copytrading_engine.trading.application.operator_service import OperatorQueryService
63from copytrading_engine.trading.application.paused_reader import PausedAccountReader, ReadKeys
64from copytrading_engine.trading.application.profile_review import ProfileReviewService
65from copytrading_engine.trading.domain.config import (
66 BrokerCredentials,
67 ConnectionCheck,
68 TradingConfiguration,
69 TradingSecrets,
70)
71from copytrading_engine.trading.domain.status import AccountStatus, TradingStatus
72from copytrading_engine.trading.entrypoints.factories import (
73 SourceSession,
74 TradingFactories,
75 broker_identity,
76 create_decoder,
77 open_owner,
78)
80log = logging.getLogger(__name__)
82# Waits between tries when Alpaca does not answer an account's first connection; they end well
83# inside the time an account has to open.
84OPEN_RETRY_SECONDS = (1.0, 2.0)
87class _ParserPublisher:
88 def __init__(self, parser: SQLiteExtractionStore, stop: asyncio.Event) -> None:
89 self._parser = parser
90 self._stop = stop
92 async def publish(self, delivery: Delivery) -> None:
93 if self._stop.is_set():
94 raise _PausedBeforeReceipt
95 if delivery.mode != "live": 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true
96 raise RuntimeError("Historical capture reached the live parser")
97 event = RawMessage.model_validate_json(delivery.payload)
98 if event.identity != delivery.key: 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true
99 raise ValueError("Source delivery identity changed")
100 await self._parser.add(event)
103class _PausedBeforeReceipt(Exception):
104 """Leave the durable delivery pending when Pause interrupts fanout."""
107def _alpaca_reader(keys: ReadKeys) -> AlpacaBroker:
108 """A broker client the paused reader uses for its three reads, and nothing else."""
109 return AlpacaBroker(AlpacaCredentials(keys.key, keys.secret), keys.environment)
112# Waits before each new try at Discord after it drops; the last repeats until it connects.
113SOURCE_RETRY_SECONDS: tuple[float, ...] = (5, 15, 30, 60, 120, 300)
116class TradingRuntime:
117 """Own real network resources only between explicit Start and Pause commands."""
119 def __init__(
120 self,
121 data_dir: Path,
122 *,
123 factories: TradingFactories | None = None,
124 telemetry: TradingTelemetry | None = None,
125 owner_open_timeout_seconds: float = 10,
126 capability_service: TradingCapabilityService | None = None,
127 restore_gate_path: Path | None = None,
128 source_retry_seconds: tuple[float, ...] = SOURCE_RETRY_SECONDS,
129 ) -> None:
130 self._data_dir = data_dir
131 self._source_retry_seconds = source_retry_seconds
132 self._source_retries = 0
133 self._source_retry_at: float | None = None
134 self._source_failure_code = "source_unavailable"
135 self._open_session: Callable[[], SourceSession] | None = None
136 self._session: SourceSession | None = None
137 self._restore_gate_path = restore_gate_path or data_dir / ".restore-manual-disabled"
138 self._activation_journal = TradingActivationJournal(data_dir / "trading-activation.json")
139 self._activation_id: str | None = None
140 self._telemetry = telemetry
141 if owner_open_timeout_seconds <= 0: 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true
142 raise ValueError("Owner open timeout must be positive")
143 self._owner_open_timeout_seconds = owner_open_timeout_seconds
144 self._capability_service = capability_service or TradingCapabilityService()
145 self._factories = factories or TradingFactories(
146 owner=partial(open_owner, telemetry=telemetry),
147 identity=broker_identity,
148 decoder=partial(
149 create_decoder,
150 diagnostics=telemetry.diagnostic_sink if telemetry is not None else None,
151 ),
152 )
153 self._status = TradingStatus()
154 self._configuration: TradingConfiguration | None = None
155 self._stop = asyncio.Event()
156 self._source_stop = asyncio.Event()
157 self._work_arrived = asyncio.Event()
158 self._source_failed = False
159 self._task: asyncio.Task[None] | None = None
160 self._accounts: dict[str, AccountSupervisor] = {}
161 self._late_owner_opens: set[asyncio.Future[AccountOwner]] = set()
162 self._late_owner_closes: set[asyncio.Task[None]] = set()
163 self._failure_code: str | None = None
164 self._announcements: deque[NotificationPayload] = deque()
165 evidence = SQLiteOperatorEvidence()
166 access = AccountAccess(
167 data_dir,
168 lambda: self._accounts,
169 configuration=lambda: self._configuration,
170 runtime_state=lambda: self._status.state,
171 evidence=evidence,
172 account_error=lambda account_id: next(
173 (item.error_code for item in self._status.accounts if item.id == account_id), None
174 ),
175 )
176 self.operator = OperatorQueryService(access, PausedAccountReader(_alpaca_reader))
177 self.manual = ManualInterventionService(access)
178 self.profiles = ProfileReviewService(
179 data_dir,
180 self._factories.decoder,
181 self._register_secrets,
182 evidence,
183 DiscordChannelHistory(),
184 )
185 self._maintenance_lock = asyncio.Lock()
186 self._maintenance_active = False
187 self._limits_lock = asyncio.Lock()
188 self._worker: ParseWorker | None = None
189 self._source: SQLiteSourceStore | None = None
191 def status(self) -> TradingStatus:
192 return self._status
194 async def _open_owner(
195 self,
196 account_id: str,
197 account_dir: Path,
198 credentials: AlpacaCredentials,
199 policy: CopyConfig,
200 environment: str,
201 ) -> AccountOwner:
202 """Open one account, trying again while Alpaca briefly cannot be reached."""
203 for delay in OPEN_RETRY_SECONDS:
204 try:
205 return await self._factories.owner(account_dir, credentials, policy, environment)
206 except BrokerError as exc:
207 if not exc.transient:
208 raise
209 log.warning("trading_account_open_retry id=%s status=%s", account_id, exc.status)
210 await asyncio.sleep(delay)
211 return await self._factories.owner(account_dir, credentials, policy, environment)
213 async def _place_records(
214 self,
215 account_root: Path,
216 configuration: TradingConfiguration,
217 broker_secrets: Mapping[str, BrokerCredentials],
218 ) -> dict[str, str]:
219 """Put each account's record under its setup name, found by the Alpaca account its keys
220 open, and return why any account cannot open: its keys could not be checked, or another
221 account in the setup uses the same Alpaca account."""
222 identify = self._factories.identity
223 if identify is None:
224 return {}
225 accounts = configuration.accounts
226 checks = await asyncio.gather(
227 *(
228 identify(
229 AlpacaCredentials(
230 broker_secrets[account.id].key, broker_secrets[account.id].secret
231 ),
232 account.environment,
233 )
234 for account in accounts
235 ),
236 return_exceptions=True,
237 )
238 wanted: dict[str, BrokerIdentity | None] = {}
239 refusals: dict[str, str] = {}
240 for account, checked in zip(accounts, checks, strict=True):
241 if isinstance(checked, BaseException):
242 if not isinstance(checked, Exception): 242 ↛ 243line 242 didn't jump to line 243 because the condition on line 242 was never true
243 raise checked
244 refusals[account.id] = account_failure(checked)
245 wanted[account.id] = None
246 else:
247 wanted[account.id] = (account.environment, checked)
248 plan = await asyncio.to_thread(reconcile_records, account_root, wanted)
249 for folder, target in plan.moves.items():
250 log.warning("trading_account_record_moved from=%s to=%s", folder, target)
251 return refusals | plan.refusals
253 def _register_secrets(self, values: Collection[str]) -> None:
254 if self._telemetry is not None:
255 self._telemetry.register_secrets(values)
257 def register_restore_secrets(self, values: tuple[str, ...]) -> None:
258 """Protect one restore preflight's credentials before broker I/O begins."""
259 self._register_secrets(values)
261 def _register_trading_secrets(self, secrets: TradingSecrets) -> None:
262 values = [
263 secrets.discord_token.get_secret_value(),
264 secrets.provider_api_key.get_secret_value(),
265 ]
266 for broker in secrets.brokers:
267 values.extend((broker.key.get_secret_value(), broker.secret.get_secret_value()))
268 if secrets.notification_token is not None:
269 values.append(secrets.notification_token.get_secret_value())
270 self._register_secrets(values)
272 def activation_status(self, activation_id: str) -> TradingActivationStatus:
273 return self._activation_journal.status(activation_id, self._status.state)
275 async def start(
276 self,
277 configuration: TradingConfiguration,
278 secrets: TradingSecrets,
279 activation_id: str | None = None,
280 ) -> TradingStatus:
281 if restore_manual_disabled(self._restore_gate_path):
282 raise ValueError("Trading is unavailable until restore reconciliation completes")
283 if self._maintenance_active:
284 raise ValueError("Trading is unavailable during maintenance")
285 self._register_trading_secrets(secrets)
286 if self._task is not None and not self._task.done(): 286 ↛ 287line 286 didn't jump to line 287 because the condition on line 286 was never true
287 raise ValueError("Trading is already active")
288 if (
289 not secrets.discord_token.get_secret_value()
290 or not secrets.provider_api_key.get_secret_value()
291 ):
292 raise ValueError("Source and provider credentials are required")
293 credential_ids = {broker.account_id for broker in secrets.brokers}
294 if credential_ids != {account.id for account in configuration.accounts}: 294 ↛ 295line 294 didn't jump to line 295 because the condition on line 294 was never true
295 raise ValueError("Every configured account needs its own broker credentials")
296 if configuration.notification is not None and ( 296 ↛ 299line 296 didn't jump to line 299 because the condition on line 296 was never true
297 secrets.notification_token is None or not secrets.notification_token.get_secret_value()
298 ):
299 raise ValueError("Configured notifications require a bot token")
300 activation_id = activation_id or str(uuid4())
301 self._activation_journal.begin(activation_id, configuration.revision())
302 self._activation_id = activation_id
303 self._configuration = configuration
304 self._stop = asyncio.Event()
305 self._source_stop = asyncio.Event()
306 self._work_arrived = asyncio.Event()
307 self._source_failed = False
308 self._source_retries = 0
309 self._source_retry_at = None
310 self._failure_code = None
311 self._announcements.clear()
312 self._status = TradingStatus(
313 state="starting", configured_accounts=len(configuration.accounts)
314 )
315 self._task = asyncio.create_task(self._run(configuration, secrets))
316 return self._status
318 async def update_account_limits(self, configuration: TradingConfiguration) -> str:
319 """Copy with changed account limits from each account's next cycle, without pausing.
321 Only limits may differ from the setup copying now; anything else needs a full check and
322 Start. While paused there is nothing to swap: the next Start copies with the new setup.
323 Returns the new setup's revision, which the app saves alongside it."""
324 async with self._limits_lock:
325 running = self._configuration
326 if self._task is None or self._task.done() or running is None:
327 return configuration.revision()
328 if configuration.without_limits() != running.without_limits():
329 raise ValueError("Only account limits can change while copying")
330 await self._apply_limits(configuration)
331 self._configuration = configuration
332 record = self._activation_journal.record
333 if ( 333 ↛ 340line 333 didn't jump to line 340 because the condition on line 333 was always true
334 self._activation_id is not None
335 and record is not None
336 and record.activation_id == self._activation_id
337 and record.phase in {"starting", "ready"}
338 ):
339 self._activation_journal.adopt(self._activation_id, configuration.revision())
340 return configuration.revision()
342 async def _apply_limits(self, configuration: TradingConfiguration) -> None:
343 """Hand every open account its limits; each swaps them between two of its cycles."""
344 now = dt.datetime.now(dt.UTC)
345 for account in configuration.accounts:
346 supervisor = self._accounts.get(account.id)
347 if supervisor is None: 347 ↛ 348line 347 didn't jump to line 348 because the condition on line 347 was never true
348 continue
349 supervisor.poll_seconds = account.policy.poll_seconds
350 if supervisor.owner is not None: 350 ↛ 345line 350 didn't jump to line 345 because the condition on line 350 was always true
351 changes = await supervisor.owner.update_config(
352 copy_config(configuration, account.id), now
353 )
354 if changes: 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true
355 log.info(
356 "account_limits_changed id=%s settings=%s",
357 account.id,
358 ",".join(change.setting for change in changes),
359 )
361 async def check_connection(self, connection: ConnectionCheck) -> CapabilityCheck:
362 """Check one service with the keys typed for it; nothing is saved or started."""
363 self._register_secrets(connection.secret_values())
364 return await self._capability_service.check(connection)
366 async def validate(
367 self, configuration: TradingConfiguration, secrets: TradingSecrets
368 ) -> TradingCapabilityReport:
369 """Check actual connections and pending revision ownership without activation."""
370 self._register_trading_secrets(secrets)
371 report = await self._capability_service.validate(configuration, secrets)
372 failures = list(report.checks)
373 if self._task is not None and not self._task.done():
374 failures.append(
375 CapabilityCheck(
376 name="configuration",
377 state="failed",
378 adapter="trading_runtime",
379 reason_code="pause_before_configuration_change",
380 )
381 )
382 else:
383 # The setup's own checks read local records only, so they run even when a connection
384 # failed: every problem shows in the one check, not one after another.
385 source = None
386 parser = None
387 routing = None
388 try:
389 database = self._data_dir / "application.db"
390 source = await SQLiteSourceStore.open(database)
391 parser = await SQLiteExtractionStore.open(database)
392 routing = await RoutingRevision.open(database)
393 await routing.validate(configuration)
394 except RoutingChangePending:
395 failures.append(
396 CapabilityCheck(
397 name="configuration",
398 state="failed",
399 adapter="routing_revision",
400 reason_code="routing_change_pending",
401 )
402 )
403 except AccountOwnershipPending as pending:
404 failures.extend(_ownership_checks(pending))
405 except Exception: # noqa: BLE001 - configuration errors become a failed capability check
406 failures.append(
407 CapabilityCheck(
408 name="configuration",
409 state="failed",
410 adapter="routing_revision",
411 reason_code="configuration_preflight_unavailable",
412 )
413 )
414 finally:
415 if routing is not None: 415 ↛ 417line 415 didn't jump to line 417 because the condition on line 415 was always true
416 await routing.close()
417 if parser is not None: 417 ↛ 419line 417 didn't jump to line 419 because the condition on line 417 was always true
418 await parser.close()
419 if source is not None: 419 ↛ 421line 419 didn't jump to line 421 because the condition on line 419 was always true
420 await source.close()
421 return report.model_copy(
422 update={
423 "activatable": report.activatable
424 and all(check.state in {"ready", "not_configured"} for check in failures)
425 and not any(check.name == "configuration" for check in failures),
426 "checks": tuple(failures),
427 }
428 )
430 async def pause(self) -> TradingStatus:
431 if self._task is None or self._task.done():
432 self._status = TradingStatus(
433 configured_accounts=len(self._configuration.accounts) if self._configuration else 0
434 )
435 return self._status
436 self._stop.set()
437 self._source_stop.set()
438 for account in self._accounts.values():
439 if account.owner is not None:
440 account.owner.request_stop()
441 self._status = replace(self._status, state="pausing")
442 return self._status
444 async def shutdown(self) -> None:
445 self.operator.detach_readers()
446 await self.pause()
447 task = self._task
448 if task is not None:
449 await task
450 # An owner that finished opening hands itself to _close_late_owner a moment later, so
451 # wait until every late open has been handed off and every close it started is done.
452 # gather does not yield when everything is already done, and the hand-off runs as a
453 # callback, so give the loop a turn before checking again.
454 while self._late_owner_opens or self._late_owner_closes:
455 await asyncio.gather(
456 *self._late_owner_opens, *self._late_owner_closes, return_exceptions=True
457 )
458 await asyncio.sleep(0)
460 def _watch_late_owner(self, opening: asyncio.Future[AccountOwner]) -> None:
461 """Close an owner that is still opening after Start gave up on it, once it opens."""
462 self._late_owner_opens.add(opening)
463 opening.add_done_callback(self._close_late_owner)
465 def _close_late_owner(self, opened: asyncio.Future[AccountOwner]) -> None:
466 """Close an owner that finished opening after Start gave up on it."""
467 self._late_owner_opens.discard(opened)
468 if opened.cancelled() or opened.exception() is not None: 468 ↛ 469line 468 didn't jump to line 469 because the condition on line 468 was never true
469 return
470 closing = asyncio.create_task(opened.result().close())
471 self._late_owner_closes.add(closing)
472 closing.add_done_callback(self._late_owner_closed)
474 def _late_owner_closed(self, closing: asyncio.Task[None]) -> None:
475 self._late_owner_closes.discard(closing)
476 if not closing.cancelled() and (error := closing.exception()) is not None:
477 log.error("late_account_owner_close_failed type=%s", type(error).__name__)
479 @asynccontextmanager
480 async def maintenance_fence(self) -> AsyncGenerator[None]:
481 """Stop and drain every trading-owned store before a consistent snapshot."""
482 async with self._maintenance_lock:
483 self._maintenance_active = True
484 try:
485 await self.shutdown()
486 yield
487 finally:
488 self._maintenance_active = False
490 async def _run(self, configuration: TradingConfiguration, secrets: TradingSecrets) -> None:
491 source: SQLiteSourceStore | None = None
492 parser: SQLiteExtractionStore | None = None
493 routing: RoutingRevision | None = None
494 decoder: ManagedDecoder | None = None
495 session: SourceSession | None = None
496 self._session = None
497 stage = "storage_unavailable"
498 try:
499 database = self._data_dir / "application.db"
500 source = await SQLiteSourceStore.open(
501 database,
502 diagnostics=(
503 self._telemetry.diagnostic_sink if self._telemetry is not None else None
504 ),
505 on_captured=self._wake,
506 )
507 self._source = source
508 parser = await SQLiteExtractionStore.open(database)
509 routing = await RoutingRevision.open(database)
510 stage = "routing_state_unavailable"
511 await routing.accept(configuration)
512 stage = "provider_unavailable"
513 decoder = await self._factories.decoder(
514 configuration.provider.name,
515 configuration.provider.reader(secrets.provider_api_key, timeout=20),
516 )
517 profile_by_revision = {
518 profile.profile_revision: profile for profile in configuration.profiles
519 }
520 routes = {}
521 for route_binding in configuration.routes:
522 profile = profile_by_revision[route_binding.profile_revision]
523 key = route_key(
524 route_binding.source, route_binding.channel_id, route_binding.author_id
525 )
526 routes[key] = dataclasses.replace(
527 profile.route(), repeat_window_minutes=route_binding.repeat_window_minutes
528 )
529 worker = ParseWorker(
530 parser,
531 decoder,
532 routes,
533 configuration.provider.model,
534 observe=self._telemetry.model_span if self._telemetry else lambda _: nullcontext(),
535 observe_workflow=(
536 self._telemetry.workflow_span if self._telemetry else lambda _: nullcontext()
537 ),
538 )
539 self._worker = worker
540 # Each account that cannot open says why on its own; what fails here is the
541 # accounts' folders on this Mac.
542 stage = "storage_unavailable"
543 account_root = self._data_dir / "accounts"
544 account_root.mkdir(parents=True, exist_ok=True, mode=0o700)
545 os.chmod(account_root, 0o700)
546 broker_secrets = {broker.account_id: broker for broker in secrets.brokers}
547 refusals = await self._place_records(account_root, configuration, broker_secrets)
548 opening: dict[str, asyncio.Future[AccountOwner]] = {}
549 for account in configuration.accounts:
550 if account.id in refusals:
551 continue
552 account_dir = account_root / account.id
553 account_dir.mkdir(mode=0o700, exist_ok=True)
554 os.chmod(account_dir, 0o700)
555 broker = broker_secrets[account.id]
556 policy = copy_config(configuration, account.id)
557 opening[account.id] = asyncio.ensure_future(
558 self._open_owner(
559 account.id,
560 account_dir,
561 AlpacaCredentials(broker.key, broker.secret),
562 policy,
563 account.environment,
564 )
565 )
566 done = (
567 (await asyncio.wait(opening.values(), timeout=self._owner_open_timeout_seconds))[0]
568 if opening
569 else set()
570 )
571 for account in configuration.accounts:
572 task = opening.get(account.id)
573 owner: AccountOwner | None = None
574 # Alpaca not answering in time reads the same as Alpaca out of reach.
575 open_failure: str = "broker_unreachable"
576 if task is None:
577 open_failure = refusals[account.id]
578 log.warning(
579 "trading_account_open_refused id=%s reason=%s", account.id, open_failure
580 )
581 elif task in done:
582 try:
583 owner = task.result()
584 except AccountOpenRefused as exc:
585 log.warning(
586 "trading_account_open_refused id=%s reason=%s", account.id, exc.reason
587 )
588 open_failure = exc.reason
589 except Exception as exc: # noqa: BLE001 - one account's failure is reported per account
590 self._journal_failure(exc)
591 open_failure = account_failure(exc)
592 log.error(
593 "trading_account_open_failed id=%s type=%s reason=%s",
594 account.id,
595 type(exc).__name__,
596 open_failure,
597 )
598 else:
599 log.error("trading_account_open_timed_out id=%s", account.id)
601 self._watch_late_owner(task)
602 self._accounts[account.id] = AccountSupervisor(
603 account.id,
604 owner,
605 account.policy.poll_seconds,
606 self._stop,
607 self._account_changed,
608 open_failure=open_failure,
609 on_failed=self._account_failed,
610 )
611 self._account_changed()
612 # Limits changed while the accounts opened reach them before the first post.
613 async with self._limits_lock:
614 if self._configuration is not None and self._configuration != configuration: 614 ↛ 615line 614 didn't jump to line 615 because the condition on line 614 was never true
615 await self._apply_limits(self._configuration)
616 if not any(account.owner is not None for account in self._accounts.values()):
617 # Copying stops for the accounts' own reason, never a vague "broker unavailable".
618 stage = self._account_failure()
619 raise RuntimeError("No configured account could open")
620 stage = "source_unavailable"
621 channels = {int(channel) for channel in configuration.source.channel_ids}
622 authors = (
623 {int(author) for author in configuration.source.author_ids}
624 if configuration.source.author_ids
625 else None
626 )
627 token = secrets.discord_token.get_secret_value()
629 def open_session() -> SourceSession:
630 # Each try gets a fresh session and stop signal: a Discord session starts once.
631 self._source_stop = asyncio.Event()
632 fresh = self._factories.session(
633 source, channels, authors, self._source_stop, self._report_source_failure
634 )
635 fresh.start(token)
636 return fresh
638 self._open_session = open_session
639 session = self._factories.session(
640 source, channels, authors, self._source_stop, self._report_source_failure
641 )
642 self._session = session
643 notifier = (
644 self._factories.notifier(
645 configuration.notification, secrets.notification_token.get_secret_value()
646 )
647 if configuration.notification is not None and secrets.notification_token is not None
648 else None
649 )
650 for account in self._accounts.values():
651 if account.owner is not None:
652 await account.refresh()
653 account.start()
654 try:
655 session.start(token)
656 except Exception as exc: # noqa: BLE001 - the source failure is recorded and retried
657 self._mark_source_failure(exc)
658 self._status = replace(self._status, state="degraded")
659 stage = "provider_unavailable"
660 await probe_model(decoder, worker.model_health)
661 stage = "processing_unavailable"
662 relay = (
663 AlertRelay(
664 notifier,
665 parser,
666 lambda: tuple(
667 account.owner
668 for account in self._accounts.values()
669 if account.state == "running" and account.owner is not None
670 ),
671 announcements=self._announcements,
672 )
673 if notifier is not None
674 else None
675 )
676 # Alerts go out on their own task: copying never waits for the alert service.
677 alerts = asyncio.create_task(relay.run(self._stop)) if relay is not None else None
678 try:
679 await self._work_loop(configuration, source, parser, worker, relay)
680 finally:
681 if alerts is not None:
682 alerts.cancel()
683 await asyncio.gather(alerts, return_exceptions=True)
684 except RoutingChangePending:
685 self._failure_code = "routing_change_pending"
686 except AccountOwnershipPending:
687 self._failure_code = "account_ownership_pending"
688 except _PausedBeforeReceipt:
689 pass
690 except Exception as exc: # noqa: BLE001 - the runtime records the failed stage and stops
691 self._failure_code = self._failure_code or stage
692 log.error("trading_runtime_failed stage=%s type=%s", stage, type(exc).__name__)
693 finally:
694 self._stop.set()
695 self._source_stop.set()
696 for account in self._accounts.values():
697 if account.owner is not None:
698 account.owner.request_stop()
699 if self._session is not None:
700 try:
701 await self._session.close()
702 except Exception as exc: # noqa: BLE001 - cleanup must not mask the primary outcome
703 log.error("source_close_failed type=%s", type(exc).__name__)
704 for account in self._accounts.values():
705 try:
706 await account.close()
707 except Exception as exc: # noqa: BLE001 - cleanup must not mask the primary outcome
708 log.error("execution_close_failed type=%s", type(exc).__name__)
709 final_accounts = tuple(
710 AccountStatus(
711 id=account.id,
712 state="failed" if account.state == "failed" else "paused",
713 error_code=account.error_code,
714 )
715 for account in self._accounts.values()
716 )
717 self._accounts.clear()
718 self._worker = None
719 self._source = None
720 if decoder is not None:
721 try:
722 await decoder.close()
723 except Exception as exc: # noqa: BLE001 - cleanup must not mask the primary outcome
724 log.error("provider_close_failed type=%s", type(exc).__name__)
725 if parser is not None: 725 ↛ 727line 725 didn't jump to line 727 because the condition on line 725 was always true
726 await parser.close()
727 if source is not None: 727 ↛ 729line 727 didn't jump to line 729 because the condition on line 727 was always true
728 await source.close()
729 if routing is not None: 729 ↛ 731line 729 didn't jump to line 731 because the condition on line 729 was always true
730 await routing.close()
731 if self._failure_code is not None:
732 self._status = replace(
733 self._status,
734 state="failed",
735 active_accounts=0,
736 source_connected=False,
737 model_ready=False,
738 error_code=self._failure_code,
739 accounts=final_accounts,
740 )
741 else:
742 self._status = TradingStatus(
743 configured_accounts=len(configuration.accounts), accounts=final_accounts
744 )
745 activation = self._activation_journal.record
746 activation_id = self._activation_id
747 if ( 747 ↛ exitline 747 didn't return from function '_run' because the condition on line 747 was always true
748 activation is not None
749 and activation_id is not None
750 and activation.activation_id == activation_id
751 ):
752 if activation.phase == "starting":
753 if self._failure_code is not None:
754 self._activation_journal.mark_failed(activation_id, self._failure_code)
755 else:
756 self._activation_journal.mark_stopped(activation_id)
758 async def _work_loop(
759 self,
760 configuration: TradingConfiguration,
761 source: SQLiteSourceStore,
762 parser: SQLiteExtractionStore,
763 worker: ParseWorker,
764 relay: AlertRelay | None,
765 ) -> None:
766 forwarder = ForwardBatch(
767 source,
768 _ParserPublisher(parser, self._stop),
769 observe=self._telemetry.source_span if self._telemetry else nullcontext,
770 )
771 fanout = SignalFanout(
772 parser,
773 {
774 route_key(route.source, route.channel_id, route.author_id): tuple(
775 DestinationTerms(
776 connection=connection,
777 environment=next(
778 account.environment
779 for account in configuration.accounts
780 if account.id == connection.account_id
781 ),
782 configuration_revision=configuration.routing_revision(),
783 guru_id=route.guru_id,
784 profile_revision=route.profile_revision,
785 repeat_window_minutes=route.repeat_window_minutes,
786 )
787 for connection in route.connections
788 )
789 for route in configuration.routes
790 },
791 self._accounts,
792 self._stop,
793 observe_workflow=(
794 self._telemetry.workflow_span if self._telemetry else lambda _: nullcontext()
795 ),
796 observe_destination=(
797 self._telemetry.destination_span
798 if self._telemetry
799 else lambda _message, _attempt: nullcontext()
800 ),
801 )
802 processed = 0
804 while not self._stop.is_set():
805 # Cleared before the pass: a post captured during it wakes the next pass at once.
806 self._work_arrived.clear()
807 await self._retry_source_if_due()
808 session = self._session
809 assert session is not None
810 if not self._source_failed:
811 try:
812 await session.ensure_running()
813 except Exception as exc: # noqa: BLE001 - the source failure is recorded and retried
814 self._mark_source_failure(exc)
815 else:
816 if session.ready: 816 ↛ 818line 816 didn't jump to line 818 because the condition on line 816 was always true
817 self._source_retries = 0
818 if (
819 self._activation_id is not None
820 and session.ready
821 and worker.model_ready
822 and not self._source_failed
823 and all(account.state == "running" for account in self._accounts.values())
824 and self._activation_journal.record is not None
825 and self._activation_journal.record.phase == "starting"
826 ):
827 self._activation_journal.mark_ready(self._activation_id)
828 # Forwarding writes the shared source/parser database. An
829 # uncertain commit must fail the whole runtime.
830 await session.forward_if_ready(forwarder)
831 for _ in range(100):
832 if self._stop.is_set() or not await worker.process_next(dt.datetime.now(dt.UTC)):
833 break
834 if self._stop.is_set():
835 break
836 processed += await fanout.deliver()
837 if self._stop.is_set(): 837 ↛ 838line 837 didn't jump to line 838 because the condition on line 837 was never true
838 break
839 notification_failed = relay is not None and relay.failing
840 self._status = replace(
841 self._status,
842 state="running"
843 if session.ready
844 and not self._source_failed
845 and worker.model_ready
846 and not notification_failed
847 and all(account.state == "running" for account in self._accounts.values())
848 else "degraded",
849 source_connected=session.ready and not self._source_failed,
850 model_ready=worker.model_ready,
851 pending_source=(source_backlog := await source.pending_snapshot()).count,
852 oldest_pending_source_at=source_backlog.oldest_at,
853 pending_signals=(signal_backlog := await parser.pending_snapshot()).count,
854 oldest_pending_signal_at=signal_backlog.oldest_at,
855 processed_signals=processed,
856 error_code=(
857 self._account_failure()
858 if any(account.state == "failed" for account in self._accounts.values())
859 else self._source_failure_code
860 if self._source_failed
861 else "notification_unavailable"
862 if notification_failed
863 else None
864 ),
865 accounts=tuple(account.status for account in self._accounts.values()),
866 active_accounts=sum(
867 account.state == "running" for account in self._accounts.values()
868 ),
869 )
870 await self._idle(0.5)
872 def _wake(self) -> None:
873 self._work_arrived.set()
875 async def _idle(self, seconds: float) -> None:
876 """Wait for the next pass: a captured post or Stop ends the wait early."""
877 waits = {
878 asyncio.ensure_future(self._stop.wait()),
879 asyncio.ensure_future(self._work_arrived.wait()),
880 }
881 try:
882 await asyncio.wait(waits, timeout=seconds, return_when=asyncio.FIRST_COMPLETED)
883 finally:
884 for wait in waits:
885 wait.cancel()
887 async def _retry_source_if_due(self) -> None:
888 """Open Discord again once its wait is over, unless Discord turned the token down."""
889 if not self._source_failed or self._source_retry_at is None or self._open_session is None:
890 return
891 if asyncio.get_running_loop().time() < self._source_retry_at:
892 return
893 self._source_retry_at = None
894 previous, self._session = self._session, None
895 if previous is not None: 895 ↛ 900line 895 didn't jump to line 900 because the condition on line 895 was always true
896 try:
897 await previous.close()
898 except Exception as exc: # noqa: BLE001 - the old session is gone either way
899 log.warning("source_close_failed type=%s", type(exc).__name__)
900 self._source_retries += 1
901 log.info("trading_source_retry attempt=%d", self._source_retries)
902 self._source_failed = False
903 try:
904 self._session = self._open_session()
905 except Exception as exc: # noqa: BLE001 - recorded, and tried again after the next wait
906 self._session = previous
907 self._mark_source_failure(exc)
908 return
909 self._status = replace(self._status, error_code=None)
911 def _report_source_failure(self, event: str, error: BaseException | None) -> None:
912 del event
913 self._mark_source_failure(error)
915 def _mark_source_failure(self, error: BaseException | None) -> None:
916 self._source_failed = True
917 self._source_stop.set()
918 rejected = token_rejected(error)
919 code = "source_unauthorized" if rejected else "source_unavailable"
920 self._source_failure_code = code
921 if not rejected:
922 waits = self._source_retry_seconds
923 wait = waits[min(self._source_retries, len(waits) - 1)] if waits else None
924 self._source_retry_at = (
925 None if wait is None else asyncio.get_running_loop().time() + wait
926 )
927 self._fail_startup_if_pending(code)
928 self._status = replace(
929 self._status,
930 state="degraded",
931 source_connected=False,
932 error_code=code,
933 )
934 log.error(
935 "trading_source_failed type=%s",
936 type(error).__name__ if error is not None else "unknown",
937 )
939 def _account_failure(self) -> str:
940 """The failed accounts' shared reason, such as another app owning the account, or
941 `accounts_unavailable` when they fail for different reasons; each account's status still
942 carries its own."""
943 reasons = {
944 account.error_code for account in self._accounts.values() if account.state == "failed"
945 }
946 if len(reasons) > 1:
947 return "accounts_unavailable"
948 return reason if reasons and (reason := reasons.pop()) else "account_unavailable"
950 def _journal_failure(self, error: BaseException) -> None:
951 if self._telemetry is not None:
952 self._telemetry.account_failed(error)
954 def _account_failed(self, account_id: str, error: BaseException) -> None:
955 """A running account stopped: the journal records the error, and the owner's alert channel
956 hears of it once."""
957 self._journal_failure(error)
958 self._announcements.append(account_stopped_alert(account_id, dt.datetime.now(dt.UTC)))
960 def _account_changed(self) -> None:
961 statuses = tuple(account.status for account in self._accounts.values())
962 if any(account.state == "failed" for account in self._accounts.values()):
963 self._fail_startup_if_pending(self._account_failure())
964 self._status = replace(
965 self._status,
966 accounts=statuses,
967 active_accounts=sum(account.state == "running" for account in self._accounts.values()),
968 state="degraded"
969 if any(account.state == "failed" for account in self._accounts.values())
970 and self._status.state in {"running", "degraded"}
971 else self._status.state,
972 error_code=self._account_failure()
973 if any(account.state == "failed" for account in self._accounts.values())
974 else self._status.error_code,
975 )
977 def _fail_startup_if_pending(self, error_code: str) -> None:
978 record = self._activation_journal.record
979 if record is None or record.phase != "starting":
980 return
981 try:
982 self._activation_journal.mark_failed(self._activation_id or "", error_code)
983 except Exception: # noqa: BLE001 - an undurable failure stops the runtime
984 # If the failure cannot be made durable, stop before any caller can
985 # treat an uncertain activation as committed.
986 self._failure_code = "activation_journal_unavailable"
987 self._source_stop.set()
988 self._stop.set()
989 for account in self._accounts.values():
990 if account.owner is not None:
991 account.owner.request_stop()
994def _ownership_checks(pending: AccountOwnershipPending) -> tuple[CapabilityCheck, ...]:
995 """One check per removed account that still has work in flight, naming the account and the
996 stocks: unfinished orders first, then a post still being copied. Shares it holds never block:
997 they stay at the broker."""
998 checks = []
999 for account_id, held in pending.accounts:
1000 if held is None:
1001 reason, symbols = "removed_account_unreadable", ()
1002 elif held.order_symbols:
1003 reason, symbols = "removed_account_has_open_orders", held.order_symbols
1004 else:
1005 reason, symbols = "removed_account_needs_review", ()
1006 checks.append(
1007 CapabilityCheck(
1008 name="configuration",
1009 state="failed",
1010 subject=account_id,
1011 adapter="routing_revision",
1012 reason_code=reason,
1013 symbols=symbols,
1014 )
1015 )
1016 return tuple(checks)
1019def copy_config(configuration: TradingConfiguration, account_id: str) -> CopyConfig:
1020 account = next(account for account in configuration.accounts if account.id == account_id)
1021 pricing = {"max_above_signal_pct", "max_below_signal_pct"}
1022 return CopyConfig(
1023 sources=tuple(f"discord:{channel}" for channel in configuration.source.channel_ids),
1024 pricing=PricePolicy(**account.policy.model_dump(include=pricing)),
1025 **account.policy.model_dump(exclude=pricing),
1026 )