Coverage for src/copytrading_engine/diagnostics/redaction.py: 93%

123 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Recursive credential removal before diagnostic admission.""" 

2 

3import base64 

4import binascii 

5import html 

6import json 

7import re 

8from dataclasses import dataclass 

9from typing import Literal 

10from urllib.parse import quote, quote_plus, unquote_plus 

11 

12from copytrading_engine.shared.payload_capture import JsonValue 

13 

14_SENSITIVE_FRAGMENTS = ( 

15 "authorization", 

16 "cookie", 

17 "credential", 

18 "password", 

19 "passwd", 

20 "secret", 

21 "token", 

22 "apikey", 

23 "privatekey", 

24) 

25_REDACTED = "[REDACTED]" 

26MAX_DIAGNOSTIC_PAYLOAD_BYTES = 48 * 1024 

27_CREDENTIAL_HEADER = re.compile( 

28 r"(?i)(?P<field>\b(?:authorization|proxy-authorization|x-api-key)\s*[:=]\s*)" 

29 r"(?:(?:bearer|basic)\s+)?(?P<value>[^\s,;\]}\"']+)" 

30) 

31_AUTH_SCHEME = re.compile(r"(?i)\b(?P<scheme>bearer|basic)\s+(?P<value>[A-Za-z0-9._~+/=-]{8,})") 

32_QUERY_PARAMETER = re.compile(r"(?P<prefix>[?&;]|^)(?P<key>[^?&;=\s]+)=(?P<value>[^&#;\s]*)") 

33_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}") 

34 

35 

36@dataclass(frozen=True, slots=True) 

37class RedactionResult: 

38 value: JsonValue 

39 redacted_fields: int 

40 

41 

42type DiagnosticPayloadStatus = Literal["complete", "oversize", "missing"] 

43 

44 

45@dataclass(frozen=True, slots=True) 

46class DiagnosticPayloadAdmission: 

47 payload: JsonValue | None 

48 status: DiagnosticPayloadStatus 

49 original_bytes: int 

50 redacted_fields: int 

51 

52 

53def redact_with_count(value: JsonValue, *, secrets: tuple[str, ...]) -> RedactionResult: 

54 """Redact recursively and report how many values or fields were removed.""" 

55 replacements = _secret_forms(secrets) 

56 return _redact(value, replacements) 

57 

58 

59def admit_diagnostic_payload( 

60 value: JsonValue | None, 

61 *, 

62 secrets: tuple[str, ...], 

63 max_bytes: int = MAX_DIAGNOSTIC_PAYLOAD_BYTES, 

64 source_bytes: int | None = None, 

65) -> DiagnosticPayloadAdmission: 

66 """Redact first, then admit a complete payload only within the fixed byte cap.""" 

67 if type(max_bytes) is not int or not 1 <= max_bytes <= MAX_DIAGNOSTIC_PAYLOAD_BYTES: 

68 raise ValueError("diagnostic payload limit is outside the supported bounds") 

69 if source_bytes is not None and (type(source_bytes) is not int or source_bytes < 0): 69 ↛ 70line 69 didn't jump to line 70 because the condition on line 69 was never true

70 raise ValueError("diagnostic payload byte count must be nonnegative") 

71 if value is None: 

72 return DiagnosticPayloadAdmission(None, "missing", source_bytes or 0, 0) 

73 

74 safe = redact_with_count(value, secrets=secrets) 

75 serialized = json.dumps(safe.value, ensure_ascii=False, separators=(",", ":")) 

76 original_bytes = ( 

77 source_bytes 

78 if source_bytes is not None 

79 else len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) 

80 ) 

81 if original_bytes > max_bytes or len(serialized.encode("utf-8")) > max_bytes: 

82 return DiagnosticPayloadAdmission(None, "oversize", original_bytes, safe.redacted_fields) 

83 return DiagnosticPayloadAdmission(safe.value, "complete", original_bytes, safe.redacted_fields) 

84 

85 

86def _redact(value: JsonValue, secrets: tuple[str, ...]) -> RedactionResult: 

87 if isinstance(value, dict): 

88 result: dict[str, JsonValue] = {} 

89 count = 0 

90 for key, item in value.items(): 

91 if _is_sensitive_key(key): 

92 result[key] = _REDACTED 

93 count += 1 

94 else: 

95 safe = _redact(item, secrets) 

96 result[key] = safe.value 

97 count += safe.redacted_fields 

98 return RedactionResult(result, count) 

99 if isinstance(value, list): 

100 result = [] 

101 count = 0 

102 for item in value: 

103 safe = _redact(item, secrets) 

104 result.append(safe.value) 

105 count += safe.redacted_fields 

106 return RedactionResult(result, count) 

107 if isinstance(value, str): 

108 return _redact_text(value, secrets) 

109 return RedactionResult(value, 0) 

110 

111 

112def _redact_text(value: str, secrets: tuple[str, ...]) -> RedactionResult: 

113 structured = _parse_structured(value) 

114 if structured is not None: 

115 safe = _redact(structured, secrets) 

116 return RedactionResult( 

117 json.dumps(safe.value, ensure_ascii=False, sort_keys=True, separators=(",", ":")), 

118 safe.redacted_fields, 

119 ) 

120 

121 result = value 

122 count = 0 

123 for secret in secrets: 

124 occurrences = result.count(secret) 

125 if occurrences: 

126 result = result.replace(secret, _REDACTED) 

127 count += occurrences 

128 

129 def redact_header(match: re.Match[str]) -> str: 

130 nonlocal count 

131 count += 1 

132 return match.group("field") + _REDACTED 

133 

134 result = _CREDENTIAL_HEADER.sub(redact_header, result) 

135 

136 def redact_auth_scheme(match: re.Match[str]) -> str: 

137 nonlocal count 

138 count += 1 

139 return f"{match.group('scheme')} {_REDACTED}" 

140 

141 result = _AUTH_SCHEME.sub(redact_auth_scheme, result) 

142 

143 def redact_query(match: re.Match[str]) -> str: 

144 nonlocal count 

145 key = unquote_plus(match.group("key")) 

146 if not _is_sensitive_key(key) and _normalized_key(key) not in _SENSITIVE_QUERY_NAMES: 

147 return match.group(0) 

148 count += 1 

149 return f"{match.group('prefix')}{match.group('key')}={_REDACTED}" 

150 

151 result = _QUERY_PARAMETER.sub(redact_query, result) 

152 return RedactionResult(result, count) 

153 

154 

155_SENSITIVE_QUERY_NAMES = frozenset( 

156 {"apikey", "auth", "authorization", "key", "password", "signature", "sig"} 

157) 

158 

159 

160def _parse_structured(value: str) -> dict[str, JsonValue] | list[JsonValue] | None: 

161 try: 

162 parsed = json.loads(value) 

163 except json.JSONDecodeError, TypeError: 

164 parsed = None 

165 if isinstance(parsed, dict | list): 

166 return parsed 

167 

168 # Some clients serialize JSON as a quoted or escaped string before logging it. 

169 if isinstance(parsed, str) and parsed != value: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 return _parse_structured(parsed) 

171 

172 compact = value.strip() 

173 if len(compact) >= 32 and re.fullmatch(r"[A-Za-z0-9_-]+={0,2}", compact): 

174 try: 

175 decoded = base64.urlsafe_b64decode(compact + "=" * (-len(compact) % 4)) 

176 parsed = json.loads(decoded) 

177 except binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError: 

178 return None 

179 if isinstance(parsed, dict | list): 

180 return parsed 

181 return None 

182 

183 

184def _secret_forms(secrets: tuple[str, ...]) -> tuple[str, ...]: 

185 forms: set[str] = set() 

186 for secret in secrets: 

187 if not secret: 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true

188 continue 

189 encoded = secret.encode("utf-8") 

190 forms.update( 

191 { 

192 secret, 

193 quote(secret, safe=""), 

194 quote_plus(secret), 

195 _lower_percent_escapes(quote(secret, safe="")), 

196 base64.b64encode(encoded).decode("ascii"), 

197 base64.urlsafe_b64encode(encoded).decode("ascii"), 

198 base64.b64encode(encoded).decode("ascii").rstrip("="), 

199 base64.urlsafe_b64encode(encoded).decode("ascii").rstrip("="), 

200 json.dumps(secret, ensure_ascii=True)[1:-1], 

201 html.escape(secret, quote=True), 

202 encoded.hex(), 

203 } 

204 ) 

205 return tuple(sorted(forms, key=len, reverse=True)) 

206 

207 

208def _lower_percent_escapes(value: str) -> str: 

209 return _PERCENT_ESCAPE.sub(lambda match: match.group(0).lower(), value) 

210 

211 

212def _normalized_key(key: str) -> str: 

213 return re.sub(r"[^a-z0-9]", "", key.casefold()) 

214 

215 

216def _is_sensitive_key(key: str) -> bool: 

217 normalized = _normalized_key(key) 

218 return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS)