Coverage for src/copytrading_engine/diagnostics/telemetry/local.py: 80%

241 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""The diagnostics sink: redacted stage, payload, and trading records in a local journal.""" 

2 

3import json 

4import queue 

5import re 

6import threading 

7import time 

8from collections.abc import Callable, Iterable, Mapping 

9from dataclasses import asdict, dataclass 

10from datetime import UTC, datetime 

11 

12from copytrading_engine.diagnostics.capture import ( 

13 DiagnosticCaptureHealth, 

14 DiagnosticHealth, 

15 DiagnosticState, 

16) 

17from copytrading_engine.diagnostics.redaction import ( 

18 MAX_DIAGNOSTIC_PAYLOAD_BYTES, 

19 JsonValue, 

20 admit_diagnostic_payload, 

21) 

22from copytrading_engine.diagnostics.telemetry.attributes import ( 

23 bounded_label, 

24 bounded_opaque_id, 

25 capture_counter_keys, 

26 stages, 

27 validated_hex_id, 

28) 

29from copytrading_engine.diagnostics.telemetry.config import TelemetryConfig 

30from copytrading_engine.diagnostics.telemetry.events import ( 

31 DiagnosticEvent, 

32 DiagnosticPayloadEvent, 

33 TradingDiagnosticEvent, 

34 safe_ledger_event, 

35) 

36from copytrading_engine.diagnostics.telemetry.journal import journal_logger, prune_journal 

37from copytrading_engine.host.self_test.model import Stage 

38from copytrading_engine.shared.model_providers import MODEL_PROVIDERS 

39from copytrading_engine.shared.payload_capture import CaptureKind, CaptureStatus, ProviderName 

40 

41_MAX_KNOWN_SECRETS = 256 

42_MAX_KNOWN_SECRET_BYTES = 4096 

43_PRUNE_INTERVAL_SECONDS = 3_600.0 

44 

45_TRADING_OPERATIONS = frozenset( 

46 { 

47 "source.forward", 

48 "workflow", 

49 "model.parse", 

50 "destination.receive", 

51 "execution.risk_checks", 

52 "execution.order_submission", 

53 "execution.quote_snapshot", 

54 "execution.ledger_event", 

55 "account.failed", 

56 } 

57) 

58_TRADING_OUTCOMES = frozenset({"returned", "raised", "observed"}) 

59 

60type JournalEvent = DiagnosticEvent | TradingDiagnosticEvent | DiagnosticPayloadEvent 

61 

62 

63@dataclass(slots=True) 

64class _FlushBarrier: 

65 completed: threading.Event 

66 successful: bool = False 

67 

68 

69def journal_line(event: JournalEvent) -> str: 

70 """One self-describing JSON line: its kind, an ISO time, then the event's own fields.""" 

71 kind = ( 

72 "stage" 

73 if isinstance(event, DiagnosticEvent) 

74 else "payload" 

75 if isinstance(event, DiagnosticPayloadEvent) 

76 else "trading" 

77 ) 

78 at = datetime.fromtimestamp(event.created_at_ns / 1e9, UTC).isoformat(timespec="milliseconds") 

79 return json.dumps( 

80 {"kind": kind, "at": at, **asdict(event)}, ensure_ascii=False, separators=(",", ":") 

81 ) 

82 

83 

84_FACT_PATTERNS = { 

85 "symbol": r"[A-Z][A-Z0-9.\-]{0,9}", 

86 "side": r"buy|sell", 

87 "qty": r"\d{1,12}(\.\d{1,10})?", 

88 "limit_price": r"\d{1,12}(\.\d{1,10})?", 

89 "source_price": r"\d{1,12}(\.\d{1,10})?", 

90 "status": r"[a-z_]{1,40}", 

91 "filled_qty": r"\d{1,12}(\.\d{1,10})?", 

92 "filled_avg_price": r"\d{1,12}(\.\d{1,10})?", 

93 "reason": r"[a-z_]{1,64}", 

94} 

95 

96 

97def trading_facts(facts: Mapping[str, object] | None) -> dict[str, str]: 

98 """The trading facts worth reading in a ledger event, each kept only when it is the plain 

99 value its pattern allows, so no free text can ride along.""" 

100 if not facts: 

101 return {} 

102 kept = {} 

103 for name, pattern in _FACT_PATTERNS.items(): 

104 value = facts.get(name) 

105 if isinstance(value, (str, int, float)) and not isinstance(value, bool): 

106 text = str(value) 

107 if re.fullmatch(pattern, text): 

108 kept[name] = text 

109 return kept 

110 

111 

112class LocalTelemetry: 

113 """Best-effort diagnostics queue independent from workflow transactions.""" 

114 

115 def __init__(self, config: TelemetryConfig) -> None: 

116 self._config = config 

117 self._queue: queue.Queue[JournalEvent | _FlushBarrier] = queue.Queue( 

118 maxsize=config.queue_capacity 

119 ) 

120 self._lock = threading.Lock() 

121 self._secrets_lock = threading.Lock() 

122 self._known_secrets: set[str] = set() 

123 self._secret_registry_overflow = False 

124 self._stop_requested = threading.Event() 

125 self._state: DiagnosticState = "healthy" 

126 self._dropped_records = 0 

127 self._last_error_code: str | None = None 

128 self._anchor_callback: Callable[[str], None] | None = None 

129 self._capture_counts = { 

130 "source_events": 0, 

131 "source_event_gaps": 0, 

132 "model_requests": 0, 

133 "model_request_gaps": 0, 

134 "model_responses": 0, 

135 "model_response_gaps": 0, 

136 } 

137 self._closed = False 

138 self._logger, error = journal_logger( 

139 config.journal_directory, 

140 retention_age_days=config.retention_age_days, 

141 max_bytes=config.max_bytes, 

142 on_error=lambda: self._record_failure(1, "journal_write_failed"), 

143 ) 

144 if error is not None: 

145 self._state = "degraded" 

146 self._last_error_code = "journal_unavailable" 

147 self._worker = threading.Thread(target=self._run, name="diagnostics-journal", daemon=True) 

148 self._worker.start() 

149 

150 def register_secrets(self, values: Iterable[str]) -> bool: 

151 """Add credentials to this process' bounded redaction set. 

152 

153 Registrations are additive for the sink lifetime so credentials from old 

154 activations remain protected while their callbacks may still be running. 

155 If the registry bound is exceeded, payload capture switches to gap-only 

156 records rather than journaling data with an incomplete redaction set. 

157 """ 

158 candidate: set[str] = set() 

159 for value in values: 

160 if not isinstance(value, str): 160 ↛ 161line 160 didn't jump to line 161 because the condition on line 160 was never true

161 raise TypeError("diagnostic redaction values must be strings") 

162 if not value: 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true

163 continue 

164 if len(value.encode("utf-8")) > _MAX_KNOWN_SECRET_BYTES: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true

165 with self._secrets_lock: 

166 self._secret_registry_overflow = True 

167 return False 

168 candidate.add(value) 

169 if len(candidate) > _MAX_KNOWN_SECRETS: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 with self._secrets_lock: 

171 self._secret_registry_overflow = True 

172 return False 

173 with self._secrets_lock: 

174 if self._secret_registry_overflow: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true

175 return False 

176 if len(self._known_secrets | candidate) > _MAX_KNOWN_SECRETS: 

177 self._secret_registry_overflow = True 

178 return False 

179 self._known_secrets.update(candidate) 

180 return True 

181 

182 def _redaction_snapshot(self) -> tuple[tuple[str, ...], bool]: 

183 with self._secrets_lock: 

184 return tuple(self._known_secrets), self._secret_registry_overflow 

185 

186 def record_stage( 

187 self, 

188 *, 

189 command_id: str, 

190 trace_id: str, 

191 stage: Stage, 

192 outcome: str, 

193 payload: JsonValue, 

194 previous_stage: Stage | None = None, 

195 anchor_exported: bool | None = None, 

196 ) -> None: 

197 """Redact and enqueue a committed stage without blocking its caller.""" 

198 known_secrets, registry_overflow = self._redaction_snapshot() 

199 admission = ( 

200 admit_diagnostic_payload( 

201 payload, secrets=known_secrets, max_bytes=MAX_DIAGNOSTIC_PAYLOAD_BYTES 

202 ) 

203 if not registry_overflow 

204 else None 

205 ) 

206 self._enqueue( 

207 DiagnosticEvent( 

208 command_id=bounded_opaque_id(command_id, 128), 

209 trace_id=bounded_opaque_id(trace_id, 128), 

210 stage=bounded_label(stage.value, stages()), 

211 outcome=bounded_label( 

212 outcome, ("accepted", "parsed", "completed", "failed", "error") 

213 ), 

214 payload=admission.payload if admission is not None else None, 

215 previous_stage=previous_stage.value if previous_stage else None, 

216 created_at_ns=time.time_ns(), 

217 anchor_exported=anchor_exported, 

218 capture_status=( 

219 "incomplete" 

220 if admission is None 

221 else "oversize" 

222 if admission.status == "oversize" 

223 else "complete" 

224 ), 

225 payload_bytes=admission.original_bytes if admission is not None else 0, 

226 redacted_fields=admission.redacted_fields if admission is not None else 0, 

227 ) 

228 ) 

229 

230 def record_payload( 

231 self, 

232 *, 

233 capture_kind: CaptureKind, 

234 workflow_id: str | None, 

235 trace_id: str | None, 

236 destination_id: str | None, 

237 attempt: int | None, 

238 provider: ProviderName, 

239 payload: JsonValue | None, 

240 source_bytes: int | None = None, 

241 max_bytes: int = MAX_DIAGNOSTIC_PAYLOAD_BYTES, 

242 status: CaptureStatus = "complete", 

243 ) -> None: 

244 """Admit a redacted, bounded copy of one source/provider wire payload.""" 

245 kind = ( 

246 capture_kind 

247 if capture_kind in {"source_event", "model_request", "model_response"} 

248 else "other" 

249 ) 

250 safe_provider = provider if provider in MODEL_PROVIDERS else "other" 

251 known_secrets, registry_overflow = self._redaction_snapshot() 

252 admission = ( 

253 admit_diagnostic_payload( 

254 payload, secrets=known_secrets, max_bytes=max_bytes, source_bytes=source_bytes 

255 ) 

256 if not registry_overflow 

257 else None 

258 ) 

259 safe_status: CaptureStatus 

260 if registry_overflow: 

261 safe_status = "incomplete" 

262 elif status in {"incomplete", "missing", "oversize"}: 

263 safe_status = status 

264 elif admission is None: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true

265 safe_status = "missing" 

266 elif admission.status != "complete": 

267 safe_status = admission.status 

268 else: 

269 safe_status = "complete" 

270 event = DiagnosticPayloadEvent( 

271 capture_kind=kind, 

272 capture_status=safe_status, 

273 workflow_id=validated_hex_id(workflow_id, 32), 

274 trace_id=validated_hex_id(trace_id, 32), 

275 destination_id=validated_hex_id(destination_id, 32), 

276 attempt=attempt if type(attempt) is int and attempt > 0 else None, 

277 provider=safe_provider, 

278 payload=( 

279 admission.payload if admission is not None and safe_status == "complete" else None 

280 ), 

281 payload_bytes=admission.original_bytes 

282 if admission is not None 

283 else (source_bytes or 0), 

284 redacted_fields=admission.redacted_fields if admission is not None else 0, 

285 created_at_ns=time.time_ns(), 

286 ) 

287 count_key, gap_key = capture_counter_keys(kind) 

288 with self._lock: 

289 if safe_status == "complete" and count_key is not None: 

290 self._capture_counts[count_key] += 1 

291 elif gap_key is not None: 291 ↛ 293line 291 didn't jump to line 293

292 self._capture_counts[gap_key] += 1 

293 if not self._enqueue(event) and gap_key is not None: 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true

294 with self._lock: 

295 if safe_status == "complete" and count_key is not None: 

296 self._capture_counts[count_key] -= 1 

297 self._capture_counts[gap_key] += 1 

298 

299 def record_trading_operation( 

300 self, 

301 *, 

302 operation: str, 

303 outcome: str, 

304 trace_id: str | None = None, 

305 signal_ref: str | None = None, 

306 ledger_event: str | None = None, 

307 workflow_id: str | None = None, 

308 destination_id: str | None = None, 

309 attempt: int | None = None, 

310 facts: Mapping[str, object] | None = None, 

311 ) -> None: 

312 """Queue only finite labels, opaque correlation, and plain trading facts that match 

313 their patterns, never source/model text.""" 

314 self._enqueue( 

315 TradingDiagnosticEvent( 

316 **trading_facts(facts), 

317 operation=operation if operation in _TRADING_OPERATIONS else "other", 

318 outcome=outcome if outcome in _TRADING_OUTCOMES else "other", 

319 trace_id=validated_hex_id(trace_id, 32), 

320 signal_ref=signal_ref 

321 if signal_ref and re.fullmatch(r"[0-9a-f]{32}", signal_ref) 

322 else None, 

323 ledger_event=safe_ledger_event(ledger_event) if ledger_event is not None else None, 

324 created_at_ns=time.time_ns(), 

325 workflow_id=validated_hex_id(workflow_id, 32), 

326 destination_id=validated_hex_id(destination_id, 32), 

327 attempt=attempt if type(attempt) is int and attempt > 0 else None, 

328 ) 

329 ) 

330 

331 def capture_health(self) -> DiagnosticCaptureHealth: 

332 """Snapshot capture completeness separately from journal health.""" 

333 with self._lock: 

334 return DiagnosticCaptureHealth(**self._capture_counts) 

335 

336 def snapshot(self) -> DiagnosticHealth: 

337 with self._lock: 

338 return DiagnosticHealth( 

339 state=self._state, 

340 dropped_records=self._dropped_records, 

341 last_error_code=self._last_error_code, 

342 ) 

343 

344 def report_trading_observer_failure(self) -> None: 

345 self._record_failure(1, "trading_observer_failed") 

346 

347 def set_anchor_export_callback(self, callback: Callable[[str], None]) -> None: 

348 """Receive each self-test command id once its captured stage is in the journal.""" 

349 self._anchor_callback = callback 

350 

351 def flush(self, *, timeout_seconds: float = 5.0) -> bool: 

352 """Wait until all events accepted before the barrier have been journaled.""" 

353 if timeout_seconds <= 0: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true

354 return False 

355 with self._lock: 

356 if self._closed: 

357 return False 

358 barrier = _FlushBarrier(threading.Event()) 

359 deadline = time.monotonic() + timeout_seconds 

360 try: 

361 self._queue.put(barrier, timeout=timeout_seconds) 

362 except queue.Full: 

363 return False 

364 completed = barrier.completed.wait(max(0.0, deadline - time.monotonic())) 

365 return completed and barrier.successful 

366 

367 def close(self, timeout_seconds: float) -> None: 

368 """Drain and stop the journal writer within the caller's shutdown budget.""" 

369 if timeout_seconds <= 0: 369 ↛ 370line 369 didn't jump to line 370 because the condition on line 369 was never true

370 raise ValueError("telemetry shutdown timeout must be positive") 

371 deadline = time.monotonic() + timeout_seconds 

372 with self._lock: 

373 if self._closed: 373 ↛ 374line 373 didn't jump to line 374 because the condition on line 373 was never true

374 return 

375 self._closed = True 

376 self._stop_requested.set() 

377 self._worker.join(max(0.0, deadline - time.monotonic())) 

378 if self._worker.is_alive(): 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true

379 self._discard_queued_on_shutdown() 

380 

381 def _enqueue(self, event: JournalEvent) -> bool: 

382 with self._lock: 

383 if self._closed: 

384 self._dropped_records += 1 

385 self._last_error_code = "sink_closed" 

386 self._state = "degraded" 

387 return False 

388 try: 

389 self._queue.put_nowait(event) 

390 except queue.Full: 

391 self._dropped_records += 1 

392 self._last_error_code = "queue_full" 

393 self._state = "degraded" 

394 return False 

395 return True 

396 

397 def _run(self) -> None: 

398 next_prune = time.monotonic() + _PRUNE_INTERVAL_SECONDS 

399 try: 

400 while True: 

401 if time.monotonic() >= next_prune: 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true

402 next_prune = time.monotonic() + _PRUNE_INTERVAL_SECONDS 

403 self._prune() 

404 try: 

405 item = self._queue.get(timeout=0.01) 

406 except queue.Empty: 

407 if self._stop_requested.is_set(): 

408 return 

409 continue 

410 if isinstance(item, _FlushBarrier): 

411 item.successful = True 

412 item.completed.set() 

413 self._queue.task_done() 

414 continue 

415 written = self._write(item) 

416 if ( 

417 written 

418 and self._anchor_callback is not None 

419 and isinstance(item, DiagnosticEvent) 

420 and item.stage == Stage.CAPTURED.value 

421 and item.anchor_exported is False 

422 ): 

423 try: 

424 self._anchor_callback(item.command_id) 

425 except Exception: # noqa: BLE001 - telemetry failure must not reach operational code 

426 self._record_failure(0, "journal_receipt_failed") 

427 self._queue.task_done() 

428 finally: 

429 self._close_journal() 

430 

431 def _write(self, event: JournalEvent) -> bool: 

432 if self._logger is None: 

433 self._record_failure(1, "journal_unavailable") 

434 return False 

435 try: 

436 self._logger.info(journal_line(event)) 

437 except OSError, TypeError, ValueError: 

438 self._record_failure(1, "journal_write_failed") 

439 return False 

440 return True 

441 

442 def _prune(self) -> None: 

443 """Expire whole days of journal while the engine keeps running for weeks.""" 

444 try: 

445 prune_journal( 

446 self._config.journal_directory, 

447 retention_age_days=self._config.retention_age_days, 

448 ) 

449 except OSError: 

450 self._record_failure(0, "journal_prune_failed") 

451 

452 def _discard_queued_on_shutdown(self) -> None: 

453 dropped = 0 

454 while True: 

455 try: 

456 item = self._queue.get_nowait() 

457 except queue.Empty: 

458 break 

459 if isinstance(item, _FlushBarrier): 

460 item.completed.set() 

461 else: 

462 dropped += 1 

463 self._queue.task_done() 

464 self._record_failure(dropped, "shutdown_timeout") 

465 

466 def _close_journal(self) -> None: 

467 if self._logger is None: 

468 return 

469 for handler in self._logger.handlers[:]: 

470 try: 

471 handler.flush() 

472 handler.close() 

473 except Exception: # noqa: BLE001 - telemetry failure must not reach operational code 

474 pass 

475 self._logger.removeHandler(handler) 

476 

477 def _record_failure(self, count: int, code: str) -> None: 

478 with self._lock: 

479 self._dropped_records += count 

480 self._state = "degraded" 

481 self._last_error_code = code