Coverage for src/copytrading_engine/diagnostics/telemetry/local.py: 80%
241 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""The diagnostics sink: redacted stage, payload, and trading records in a local journal."""
3import json
4import queue
5import re
6import threading
7import time
8from collections.abc import Callable, Iterable, Mapping
9from dataclasses import asdict, dataclass
10from datetime import UTC, datetime
12from copytrading_engine.diagnostics.capture import (
13 DiagnosticCaptureHealth,
14 DiagnosticHealth,
15 DiagnosticState,
16)
17from copytrading_engine.diagnostics.redaction import (
18 MAX_DIAGNOSTIC_PAYLOAD_BYTES,
19 JsonValue,
20 admit_diagnostic_payload,
21)
22from copytrading_engine.diagnostics.telemetry.attributes import (
23 bounded_label,
24 bounded_opaque_id,
25 capture_counter_keys,
26 stages,
27 validated_hex_id,
28)
29from copytrading_engine.diagnostics.telemetry.config import TelemetryConfig
30from copytrading_engine.diagnostics.telemetry.events import (
31 DiagnosticEvent,
32 DiagnosticPayloadEvent,
33 TradingDiagnosticEvent,
34 safe_ledger_event,
35)
36from copytrading_engine.diagnostics.telemetry.journal import journal_logger, prune_journal
37from copytrading_engine.host.self_test.model import Stage
38from copytrading_engine.shared.model_providers import MODEL_PROVIDERS
39from copytrading_engine.shared.payload_capture import CaptureKind, CaptureStatus, ProviderName
41_MAX_KNOWN_SECRETS = 256
42_MAX_KNOWN_SECRET_BYTES = 4096
43_PRUNE_INTERVAL_SECONDS = 3_600.0
45_TRADING_OPERATIONS = frozenset(
46 {
47 "source.forward",
48 "workflow",
49 "model.parse",
50 "destination.receive",
51 "execution.risk_checks",
52 "execution.order_submission",
53 "execution.quote_snapshot",
54 "execution.ledger_event",
55 "account.failed",
56 }
57)
58_TRADING_OUTCOMES = frozenset({"returned", "raised", "observed"})
60type JournalEvent = DiagnosticEvent | TradingDiagnosticEvent | DiagnosticPayloadEvent
63@dataclass(slots=True)
64class _FlushBarrier:
65 completed: threading.Event
66 successful: bool = False
69def journal_line(event: JournalEvent) -> str:
70 """One self-describing JSON line: its kind, an ISO time, then the event's own fields."""
71 kind = (
72 "stage"
73 if isinstance(event, DiagnosticEvent)
74 else "payload"
75 if isinstance(event, DiagnosticPayloadEvent)
76 else "trading"
77 )
78 at = datetime.fromtimestamp(event.created_at_ns / 1e9, UTC).isoformat(timespec="milliseconds")
79 return json.dumps(
80 {"kind": kind, "at": at, **asdict(event)}, ensure_ascii=False, separators=(",", ":")
81 )
84_FACT_PATTERNS = {
85 "symbol": r"[A-Z][A-Z0-9.\-]{0,9}",
86 "side": r"buy|sell",
87 "qty": r"\d{1,12}(\.\d{1,10})?",
88 "limit_price": r"\d{1,12}(\.\d{1,10})?",
89 "source_price": r"\d{1,12}(\.\d{1,10})?",
90 "status": r"[a-z_]{1,40}",
91 "filled_qty": r"\d{1,12}(\.\d{1,10})?",
92 "filled_avg_price": r"\d{1,12}(\.\d{1,10})?",
93 "reason": r"[a-z_]{1,64}",
94}
97def trading_facts(facts: Mapping[str, object] | None) -> dict[str, str]:
98 """The trading facts worth reading in a ledger event, each kept only when it is the plain
99 value its pattern allows, so no free text can ride along."""
100 if not facts:
101 return {}
102 kept = {}
103 for name, pattern in _FACT_PATTERNS.items():
104 value = facts.get(name)
105 if isinstance(value, (str, int, float)) and not isinstance(value, bool):
106 text = str(value)
107 if re.fullmatch(pattern, text):
108 kept[name] = text
109 return kept
112class LocalTelemetry:
113 """Best-effort diagnostics queue independent from workflow transactions."""
115 def __init__(self, config: TelemetryConfig) -> None:
116 self._config = config
117 self._queue: queue.Queue[JournalEvent | _FlushBarrier] = queue.Queue(
118 maxsize=config.queue_capacity
119 )
120 self._lock = threading.Lock()
121 self._secrets_lock = threading.Lock()
122 self._known_secrets: set[str] = set()
123 self._secret_registry_overflow = False
124 self._stop_requested = threading.Event()
125 self._state: DiagnosticState = "healthy"
126 self._dropped_records = 0
127 self._last_error_code: str | None = None
128 self._anchor_callback: Callable[[str], None] | None = None
129 self._capture_counts = {
130 "source_events": 0,
131 "source_event_gaps": 0,
132 "model_requests": 0,
133 "model_request_gaps": 0,
134 "model_responses": 0,
135 "model_response_gaps": 0,
136 }
137 self._closed = False
138 self._logger, error = journal_logger(
139 config.journal_directory,
140 retention_age_days=config.retention_age_days,
141 max_bytes=config.max_bytes,
142 on_error=lambda: self._record_failure(1, "journal_write_failed"),
143 )
144 if error is not None:
145 self._state = "degraded"
146 self._last_error_code = "journal_unavailable"
147 self._worker = threading.Thread(target=self._run, name="diagnostics-journal", daemon=True)
148 self._worker.start()
150 def register_secrets(self, values: Iterable[str]) -> bool:
151 """Add credentials to this process' bounded redaction set.
153 Registrations are additive for the sink lifetime so credentials from old
154 activations remain protected while their callbacks may still be running.
155 If the registry bound is exceeded, payload capture switches to gap-only
156 records rather than journaling data with an incomplete redaction set.
157 """
158 candidate: set[str] = set()
159 for value in values:
160 if not isinstance(value, str): 160 ↛ 161line 160 didn't jump to line 161 because the condition on line 160 was never true
161 raise TypeError("diagnostic redaction values must be strings")
162 if not value: 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true
163 continue
164 if len(value.encode("utf-8")) > _MAX_KNOWN_SECRET_BYTES: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 with self._secrets_lock:
166 self._secret_registry_overflow = True
167 return False
168 candidate.add(value)
169 if len(candidate) > _MAX_KNOWN_SECRETS: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 with self._secrets_lock:
171 self._secret_registry_overflow = True
172 return False
173 with self._secrets_lock:
174 if self._secret_registry_overflow: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true
175 return False
176 if len(self._known_secrets | candidate) > _MAX_KNOWN_SECRETS:
177 self._secret_registry_overflow = True
178 return False
179 self._known_secrets.update(candidate)
180 return True
182 def _redaction_snapshot(self) -> tuple[tuple[str, ...], bool]:
183 with self._secrets_lock:
184 return tuple(self._known_secrets), self._secret_registry_overflow
186 def record_stage(
187 self,
188 *,
189 command_id: str,
190 trace_id: str,
191 stage: Stage,
192 outcome: str,
193 payload: JsonValue,
194 previous_stage: Stage | None = None,
195 anchor_exported: bool | None = None,
196 ) -> None:
197 """Redact and enqueue a committed stage without blocking its caller."""
198 known_secrets, registry_overflow = self._redaction_snapshot()
199 admission = (
200 admit_diagnostic_payload(
201 payload, secrets=known_secrets, max_bytes=MAX_DIAGNOSTIC_PAYLOAD_BYTES
202 )
203 if not registry_overflow
204 else None
205 )
206 self._enqueue(
207 DiagnosticEvent(
208 command_id=bounded_opaque_id(command_id, 128),
209 trace_id=bounded_opaque_id(trace_id, 128),
210 stage=bounded_label(stage.value, stages()),
211 outcome=bounded_label(
212 outcome, ("accepted", "parsed", "completed", "failed", "error")
213 ),
214 payload=admission.payload if admission is not None else None,
215 previous_stage=previous_stage.value if previous_stage else None,
216 created_at_ns=time.time_ns(),
217 anchor_exported=anchor_exported,
218 capture_status=(
219 "incomplete"
220 if admission is None
221 else "oversize"
222 if admission.status == "oversize"
223 else "complete"
224 ),
225 payload_bytes=admission.original_bytes if admission is not None else 0,
226 redacted_fields=admission.redacted_fields if admission is not None else 0,
227 )
228 )
230 def record_payload(
231 self,
232 *,
233 capture_kind: CaptureKind,
234 workflow_id: str | None,
235 trace_id: str | None,
236 destination_id: str | None,
237 attempt: int | None,
238 provider: ProviderName,
239 payload: JsonValue | None,
240 source_bytes: int | None = None,
241 max_bytes: int = MAX_DIAGNOSTIC_PAYLOAD_BYTES,
242 status: CaptureStatus = "complete",
243 ) -> None:
244 """Admit a redacted, bounded copy of one source/provider wire payload."""
245 kind = (
246 capture_kind
247 if capture_kind in {"source_event", "model_request", "model_response"}
248 else "other"
249 )
250 safe_provider = provider if provider in MODEL_PROVIDERS else "other"
251 known_secrets, registry_overflow = self._redaction_snapshot()
252 admission = (
253 admit_diagnostic_payload(
254 payload, secrets=known_secrets, max_bytes=max_bytes, source_bytes=source_bytes
255 )
256 if not registry_overflow
257 else None
258 )
259 safe_status: CaptureStatus
260 if registry_overflow:
261 safe_status = "incomplete"
262 elif status in {"incomplete", "missing", "oversize"}:
263 safe_status = status
264 elif admission is None: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true
265 safe_status = "missing"
266 elif admission.status != "complete":
267 safe_status = admission.status
268 else:
269 safe_status = "complete"
270 event = DiagnosticPayloadEvent(
271 capture_kind=kind,
272 capture_status=safe_status,
273 workflow_id=validated_hex_id(workflow_id, 32),
274 trace_id=validated_hex_id(trace_id, 32),
275 destination_id=validated_hex_id(destination_id, 32),
276 attempt=attempt if type(attempt) is int and attempt > 0 else None,
277 provider=safe_provider,
278 payload=(
279 admission.payload if admission is not None and safe_status == "complete" else None
280 ),
281 payload_bytes=admission.original_bytes
282 if admission is not None
283 else (source_bytes or 0),
284 redacted_fields=admission.redacted_fields if admission is not None else 0,
285 created_at_ns=time.time_ns(),
286 )
287 count_key, gap_key = capture_counter_keys(kind)
288 with self._lock:
289 if safe_status == "complete" and count_key is not None:
290 self._capture_counts[count_key] += 1
291 elif gap_key is not None: 291 ↛ 293line 291 didn't jump to line 293
292 self._capture_counts[gap_key] += 1
293 if not self._enqueue(event) and gap_key is not None: 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true
294 with self._lock:
295 if safe_status == "complete" and count_key is not None:
296 self._capture_counts[count_key] -= 1
297 self._capture_counts[gap_key] += 1
299 def record_trading_operation(
300 self,
301 *,
302 operation: str,
303 outcome: str,
304 trace_id: str | None = None,
305 signal_ref: str | None = None,
306 ledger_event: str | None = None,
307 workflow_id: str | None = None,
308 destination_id: str | None = None,
309 attempt: int | None = None,
310 facts: Mapping[str, object] | None = None,
311 ) -> None:
312 """Queue only finite labels, opaque correlation, and plain trading facts that match
313 their patterns, never source/model text."""
314 self._enqueue(
315 TradingDiagnosticEvent(
316 **trading_facts(facts),
317 operation=operation if operation in _TRADING_OPERATIONS else "other",
318 outcome=outcome if outcome in _TRADING_OUTCOMES else "other",
319 trace_id=validated_hex_id(trace_id, 32),
320 signal_ref=signal_ref
321 if signal_ref and re.fullmatch(r"[0-9a-f]{32}", signal_ref)
322 else None,
323 ledger_event=safe_ledger_event(ledger_event) if ledger_event is not None else None,
324 created_at_ns=time.time_ns(),
325 workflow_id=validated_hex_id(workflow_id, 32),
326 destination_id=validated_hex_id(destination_id, 32),
327 attempt=attempt if type(attempt) is int and attempt > 0 else None,
328 )
329 )
331 def capture_health(self) -> DiagnosticCaptureHealth:
332 """Snapshot capture completeness separately from journal health."""
333 with self._lock:
334 return DiagnosticCaptureHealth(**self._capture_counts)
336 def snapshot(self) -> DiagnosticHealth:
337 with self._lock:
338 return DiagnosticHealth(
339 state=self._state,
340 dropped_records=self._dropped_records,
341 last_error_code=self._last_error_code,
342 )
344 def report_trading_observer_failure(self) -> None:
345 self._record_failure(1, "trading_observer_failed")
347 def set_anchor_export_callback(self, callback: Callable[[str], None]) -> None:
348 """Receive each self-test command id once its captured stage is in the journal."""
349 self._anchor_callback = callback
351 def flush(self, *, timeout_seconds: float = 5.0) -> bool:
352 """Wait until all events accepted before the barrier have been journaled."""
353 if timeout_seconds <= 0: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 return False
355 with self._lock:
356 if self._closed:
357 return False
358 barrier = _FlushBarrier(threading.Event())
359 deadline = time.monotonic() + timeout_seconds
360 try:
361 self._queue.put(barrier, timeout=timeout_seconds)
362 except queue.Full:
363 return False
364 completed = barrier.completed.wait(max(0.0, deadline - time.monotonic()))
365 return completed and barrier.successful
367 def close(self, timeout_seconds: float) -> None:
368 """Drain and stop the journal writer within the caller's shutdown budget."""
369 if timeout_seconds <= 0: 369 ↛ 370line 369 didn't jump to line 370 because the condition on line 369 was never true
370 raise ValueError("telemetry shutdown timeout must be positive")
371 deadline = time.monotonic() + timeout_seconds
372 with self._lock:
373 if self._closed: 373 ↛ 374line 373 didn't jump to line 374 because the condition on line 373 was never true
374 return
375 self._closed = True
376 self._stop_requested.set()
377 self._worker.join(max(0.0, deadline - time.monotonic()))
378 if self._worker.is_alive(): 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true
379 self._discard_queued_on_shutdown()
381 def _enqueue(self, event: JournalEvent) -> bool:
382 with self._lock:
383 if self._closed:
384 self._dropped_records += 1
385 self._last_error_code = "sink_closed"
386 self._state = "degraded"
387 return False
388 try:
389 self._queue.put_nowait(event)
390 except queue.Full:
391 self._dropped_records += 1
392 self._last_error_code = "queue_full"
393 self._state = "degraded"
394 return False
395 return True
397 def _run(self) -> None:
398 next_prune = time.monotonic() + _PRUNE_INTERVAL_SECONDS
399 try:
400 while True:
401 if time.monotonic() >= next_prune: 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true
402 next_prune = time.monotonic() + _PRUNE_INTERVAL_SECONDS
403 self._prune()
404 try:
405 item = self._queue.get(timeout=0.01)
406 except queue.Empty:
407 if self._stop_requested.is_set():
408 return
409 continue
410 if isinstance(item, _FlushBarrier):
411 item.successful = True
412 item.completed.set()
413 self._queue.task_done()
414 continue
415 written = self._write(item)
416 if (
417 written
418 and self._anchor_callback is not None
419 and isinstance(item, DiagnosticEvent)
420 and item.stage == Stage.CAPTURED.value
421 and item.anchor_exported is False
422 ):
423 try:
424 self._anchor_callback(item.command_id)
425 except Exception: # noqa: BLE001 - telemetry failure must not reach operational code
426 self._record_failure(0, "journal_receipt_failed")
427 self._queue.task_done()
428 finally:
429 self._close_journal()
431 def _write(self, event: JournalEvent) -> bool:
432 if self._logger is None:
433 self._record_failure(1, "journal_unavailable")
434 return False
435 try:
436 self._logger.info(journal_line(event))
437 except OSError, TypeError, ValueError:
438 self._record_failure(1, "journal_write_failed")
439 return False
440 return True
442 def _prune(self) -> None:
443 """Expire whole days of journal while the engine keeps running for weeks."""
444 try:
445 prune_journal(
446 self._config.journal_directory,
447 retention_age_days=self._config.retention_age_days,
448 )
449 except OSError:
450 self._record_failure(0, "journal_prune_failed")
452 def _discard_queued_on_shutdown(self) -> None:
453 dropped = 0
454 while True:
455 try:
456 item = self._queue.get_nowait()
457 except queue.Empty:
458 break
459 if isinstance(item, _FlushBarrier):
460 item.completed.set()
461 else:
462 dropped += 1
463 self._queue.task_done()
464 self._record_failure(dropped, "shutdown_timeout")
466 def _close_journal(self) -> None:
467 if self._logger is None:
468 return
469 for handler in self._logger.handlers[:]:
470 try:
471 handler.flush()
472 handler.close()
473 except Exception: # noqa: BLE001 - telemetry failure must not reach operational code
474 pass
475 self._logger.removeHandler(handler)
477 def _record_failure(self, count: int, code: str) -> None:
478 with self._lock:
479 self._dropped_records += count
480 self._state = "degraded"
481 self._last_error_code = code