Coverage for src/copytrading_engine/diagnostics/redaction.py: 93%
123 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Recursive credential removal before diagnostic admission."""
3import base64
4import binascii
5import html
6import json
7import re
8from dataclasses import dataclass
9from typing import Literal
10from urllib.parse import quote, quote_plus, unquote_plus
12from copytrading_engine.shared.payload_capture import JsonValue
14_SENSITIVE_FRAGMENTS = (
15 "authorization",
16 "cookie",
17 "credential",
18 "password",
19 "passwd",
20 "secret",
21 "token",
22 "apikey",
23 "privatekey",
24)
25_REDACTED = "[REDACTED]"
26MAX_DIAGNOSTIC_PAYLOAD_BYTES = 48 * 1024
27_CREDENTIAL_HEADER = re.compile(
28 r"(?i)(?P<field>\b(?:authorization|proxy-authorization|x-api-key)\s*[:=]\s*)"
29 r"(?:(?:bearer|basic)\s+)?(?P<value>[^\s,;\]}\"']+)"
30)
31_AUTH_SCHEME = re.compile(r"(?i)\b(?P<scheme>bearer|basic)\s+(?P<value>[A-Za-z0-9._~+/=-]{8,})")
32_QUERY_PARAMETER = re.compile(r"(?P<prefix>[?&;]|^)(?P<key>[^?&;=\s]+)=(?P<value>[^&#;\s]*)")
33_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")
36@dataclass(frozen=True, slots=True)
37class RedactionResult:
38 value: JsonValue
39 redacted_fields: int
42type DiagnosticPayloadStatus = Literal["complete", "oversize", "missing"]
45@dataclass(frozen=True, slots=True)
46class DiagnosticPayloadAdmission:
47 payload: JsonValue | None
48 status: DiagnosticPayloadStatus
49 original_bytes: int
50 redacted_fields: int
53def redact_with_count(value: JsonValue, *, secrets: tuple[str, ...]) -> RedactionResult:
54 """Redact recursively and report how many values or fields were removed."""
55 replacements = _secret_forms(secrets)
56 return _redact(value, replacements)
59def admit_diagnostic_payload(
60 value: JsonValue | None,
61 *,
62 secrets: tuple[str, ...],
63 max_bytes: int = MAX_DIAGNOSTIC_PAYLOAD_BYTES,
64 source_bytes: int | None = None,
65) -> DiagnosticPayloadAdmission:
66 """Redact first, then admit a complete payload only within the fixed byte cap."""
67 if type(max_bytes) is not int or not 1 <= max_bytes <= MAX_DIAGNOSTIC_PAYLOAD_BYTES:
68 raise ValueError("diagnostic payload limit is outside the supported bounds")
69 if source_bytes is not None and (type(source_bytes) is not int or source_bytes < 0): 69 ↛ 70line 69 didn't jump to line 70 because the condition on line 69 was never true
70 raise ValueError("diagnostic payload byte count must be nonnegative")
71 if value is None:
72 return DiagnosticPayloadAdmission(None, "missing", source_bytes or 0, 0)
74 safe = redact_with_count(value, secrets=secrets)
75 serialized = json.dumps(safe.value, ensure_ascii=False, separators=(",", ":"))
76 original_bytes = (
77 source_bytes
78 if source_bytes is not None
79 else len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
80 )
81 if original_bytes > max_bytes or len(serialized.encode("utf-8")) > max_bytes:
82 return DiagnosticPayloadAdmission(None, "oversize", original_bytes, safe.redacted_fields)
83 return DiagnosticPayloadAdmission(safe.value, "complete", original_bytes, safe.redacted_fields)
86def _redact(value: JsonValue, secrets: tuple[str, ...]) -> RedactionResult:
87 if isinstance(value, dict):
88 result: dict[str, JsonValue] = {}
89 count = 0
90 for key, item in value.items():
91 if _is_sensitive_key(key):
92 result[key] = _REDACTED
93 count += 1
94 else:
95 safe = _redact(item, secrets)
96 result[key] = safe.value
97 count += safe.redacted_fields
98 return RedactionResult(result, count)
99 if isinstance(value, list):
100 result = []
101 count = 0
102 for item in value:
103 safe = _redact(item, secrets)
104 result.append(safe.value)
105 count += safe.redacted_fields
106 return RedactionResult(result, count)
107 if isinstance(value, str):
108 return _redact_text(value, secrets)
109 return RedactionResult(value, 0)
112def _redact_text(value: str, secrets: tuple[str, ...]) -> RedactionResult:
113 structured = _parse_structured(value)
114 if structured is not None:
115 safe = _redact(structured, secrets)
116 return RedactionResult(
117 json.dumps(safe.value, ensure_ascii=False, sort_keys=True, separators=(",", ":")),
118 safe.redacted_fields,
119 )
121 result = value
122 count = 0
123 for secret in secrets:
124 occurrences = result.count(secret)
125 if occurrences:
126 result = result.replace(secret, _REDACTED)
127 count += occurrences
129 def redact_header(match: re.Match[str]) -> str:
130 nonlocal count
131 count += 1
132 return match.group("field") + _REDACTED
134 result = _CREDENTIAL_HEADER.sub(redact_header, result)
136 def redact_auth_scheme(match: re.Match[str]) -> str:
137 nonlocal count
138 count += 1
139 return f"{match.group('scheme')} {_REDACTED}"
141 result = _AUTH_SCHEME.sub(redact_auth_scheme, result)
143 def redact_query(match: re.Match[str]) -> str:
144 nonlocal count
145 key = unquote_plus(match.group("key"))
146 if not _is_sensitive_key(key) and _normalized_key(key) not in _SENSITIVE_QUERY_NAMES:
147 return match.group(0)
148 count += 1
149 return f"{match.group('prefix')}{match.group('key')}={_REDACTED}"
151 result = _QUERY_PARAMETER.sub(redact_query, result)
152 return RedactionResult(result, count)
155_SENSITIVE_QUERY_NAMES = frozenset(
156 {"apikey", "auth", "authorization", "key", "password", "signature", "sig"}
157)
160def _parse_structured(value: str) -> dict[str, JsonValue] | list[JsonValue] | None:
161 try:
162 parsed = json.loads(value)
163 except json.JSONDecodeError, TypeError:
164 parsed = None
165 if isinstance(parsed, dict | list):
166 return parsed
168 # Some clients serialize JSON as a quoted or escaped string before logging it.
169 if isinstance(parsed, str) and parsed != value: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 return _parse_structured(parsed)
172 compact = value.strip()
173 if len(compact) >= 32 and re.fullmatch(r"[A-Za-z0-9_-]+={0,2}", compact):
174 try:
175 decoded = base64.urlsafe_b64decode(compact + "=" * (-len(compact) % 4))
176 parsed = json.loads(decoded)
177 except binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError:
178 return None
179 if isinstance(parsed, dict | list):
180 return parsed
181 return None
184def _secret_forms(secrets: tuple[str, ...]) -> tuple[str, ...]:
185 forms: set[str] = set()
186 for secret in secrets:
187 if not secret: 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true
188 continue
189 encoded = secret.encode("utf-8")
190 forms.update(
191 {
192 secret,
193 quote(secret, safe=""),
194 quote_plus(secret),
195 _lower_percent_escapes(quote(secret, safe="")),
196 base64.b64encode(encoded).decode("ascii"),
197 base64.urlsafe_b64encode(encoded).decode("ascii"),
198 base64.b64encode(encoded).decode("ascii").rstrip("="),
199 base64.urlsafe_b64encode(encoded).decode("ascii").rstrip("="),
200 json.dumps(secret, ensure_ascii=True)[1:-1],
201 html.escape(secret, quote=True),
202 encoded.hex(),
203 }
204 )
205 return tuple(sorted(forms, key=len, reverse=True))
208def _lower_percent_escapes(value: str) -> str:
209 return _PERCENT_ESCAPE.sub(lambda match: match.group(0).lower(), value)
212def _normalized_key(key: str) -> str:
213 return re.sub(r"[^a-z0-9]", "", key.casefold())
216def _is_sensitive_key(key: str) -> bool:
217 normalized = _normalized_key(key)
218 return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS)