Coverage for src/copytrading_engine/sources/attachments.py: 76%

166 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Fetch only bounded Discord CDN attachments into private operational storage.""" 

2 

3import asyncio 

4import hashlib 

5import os 

6import re 

7import stat 

8from collections.abc import Sequence 

9from dataclasses import dataclass 

10from pathlib import Path 

11from urllib.parse import urljoin, urlsplit 

12from uuid import uuid4 

13 

14import httpx2 as httpx 

15 

16from copytrading_engine.sources.application import AttachmentReference, AttachmentStatus 

17from copytrading_engine.sources.evidence import MAX_ATTACHMENTS_PER_MESSAGE 

18 

19# A chart screenshot is commonly 1-3 MB. 

20MAX_ATTACHMENT_BYTES = 4 * 1024 * 1024 

21MAX_MESSAGE_ATTACHMENT_BYTES = 8 * 1024 * 1024 

22MAX_ATTACHMENT_REDIRECTS = 3 

23ATTACHMENT_CAPTURE_TIMEOUT_SECONDS = 15 

24ALLOWED_ATTACHMENT_HOSTS = frozenset({"cdn.discordapp.com", "media.discordapp.net"}) 

25# Discord's own copy of a picture linked from elsewhere, as a relay's embed shows it. 

26_DISCORD_IMAGE_PROXY = re.compile(r"^images-ext-[0-9]+\.discordapp\.net$") 

27 

28 

29def discord_host(host: str | None) -> bool: 

30 """Whether Discord itself serves this host's files.""" 

31 host = (host or "").casefold() 

32 return host in ALLOWED_ATTACHMENT_HOSTS or bool(_DISCORD_IMAGE_PROXY.fullmatch(host)) 

33 

34 

35_REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) 

36_CONTENT_ADDRESS = re.compile(r"^attachments/([0-9a-f]{2})/([0-9a-f]{64})$") 

37 

38 

39@dataclass(frozen=True, slots=True) 

40class AttachmentCaptureResult: 

41 status: AttachmentStatus 

42 byte_size: int | None = None 

43 sha256: str | None = None 

44 managed_path: str | None = None 

45 

46 

47async def capture_attachments( 

48 references: Sequence[AttachmentReference], 

49 managed_root: Path, 

50 *, 

51 transport: httpx.AsyncBaseTransport | None = None, 

52) -> tuple[AttachmentCaptureResult, ...]: 

53 """Capture at most ten files, with one shared message time and byte budget.""" 

54 references = tuple(references[:MAX_ATTACHMENTS_PER_MESSAGE]) 

55 results: list[AttachmentCaptureResult] = [] 

56 captured_bytes = 0 

57 timeout = httpx.Timeout(5.0, connect=2.0) 

58 async with httpx.AsyncClient( 

59 transport=transport, 

60 timeout=timeout, 

61 follow_redirects=False, 

62 trust_env=False, 

63 ) as client: 

64 try: 

65 async with asyncio.timeout(ATTACHMENT_CAPTURE_TIMEOUT_SECONDS): 

66 for reference in references: 

67 if not reference.url: 

68 results.append(AttachmentCaptureResult("missing")) 

69 continue 

70 if type(reference.declared_size) is not int or reference.declared_size < 0: 70 ↛ 71line 70 didn't jump to line 71 because the condition on line 70 was never true

71 results.append(AttachmentCaptureResult("download_failed")) 

72 continue 

73 remaining = MAX_MESSAGE_ATTACHMENT_BYTES - captured_bytes 

74 if ( 

75 reference.declared_size > MAX_ATTACHMENT_BYTES 

76 or reference.declared_size > remaining 

77 ): 

78 results.append(AttachmentCaptureResult("oversize")) 

79 continue 

80 result = await _capture_one( 

81 client, 

82 reference.url, 

83 managed_root, 

84 maximum_bytes=min(MAX_ATTACHMENT_BYTES, remaining), 

85 ) 

86 results.append(result) 

87 if result.status == "available" and result.byte_size is not None: 

88 captured_bytes += result.byte_size 

89 except TimeoutError: 

90 results.extend( 

91 AttachmentCaptureResult("timed_out") for _ in range(len(results), len(references)) 

92 ) 

93 return tuple(results) 

94 

95 

96async def _capture_one( 

97 client: httpx.AsyncClient, 

98 initial_url: str, 

99 managed_root: Path, 

100 *, 

101 maximum_bytes: int, 

102) -> AttachmentCaptureResult: 

103 if not _allowed_url(initial_url): 

104 return AttachmentCaptureResult("origin_rejected") 

105 url = initial_url 

106 for redirect_count in range(MAX_ATTACHMENT_REDIRECTS + 1): 106 ↛ 144line 106 didn't jump to line 144 because the loop on line 106 didn't complete

107 try: 

108 async with client.stream("GET", url) as response: 

109 final_url = str(response.request.url) 

110 if not _allowed_url(final_url): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true

111 return AttachmentCaptureResult("origin_rejected") 

112 if response.status_code in _REDIRECT_CODES: 

113 location = response.headers.get("location") 

114 if not location or redirect_count == MAX_ATTACHMENT_REDIRECTS: 114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true

115 return AttachmentCaptureResult("download_failed") 

116 next_url = urljoin(final_url, location) 

117 if not _allowed_url(next_url): 117 ↛ 119line 117 didn't jump to line 119 because the condition on line 117 was always true

118 return AttachmentCaptureResult("origin_rejected") 

119 url = next_url 

120 continue 

121 if not 200 <= response.status_code < 300: 121 ↛ 122line 121 didn't jump to line 122 because the condition on line 121 was never true

122 return AttachmentCaptureResult("download_failed") 

123 length = response.headers.get("content-length") 

124 if length is not None: 124 ↛ 130line 124 didn't jump to line 130 because the condition on line 124 was always true

125 try: 

126 if int(length) > maximum_bytes: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 return AttachmentCaptureResult("oversize") 

128 except ValueError: 

129 pass 

130 body = bytearray() 

131 async for chunk in response.aiter_bytes(): 

132 if len(body) + len(chunk) > maximum_bytes: 132 ↛ 133line 132 didn't jump to line 133 because the condition on line 132 was never true

133 return AttachmentCaptureResult("oversize") 

134 body.extend(chunk) 

135 digest = hashlib.sha256(body).hexdigest() 

136 relative_path = await asyncio.to_thread( 

137 _store_content, managed_root, bytes(body), digest 

138 ) 

139 return AttachmentCaptureResult("available", len(body), digest, relative_path) 

140 except httpx.TimeoutException: 

141 return AttachmentCaptureResult("timed_out") 

142 except httpx.RequestError, OSError, ValueError: 

143 return AttachmentCaptureResult("download_failed") 

144 return AttachmentCaptureResult("download_failed") 

145 

146 

147def _allowed_url(value: str) -> bool: 

148 try: 

149 parsed = urlsplit(value) 

150 return ( 

151 parsed.scheme == "https" 

152 and discord_host(parsed.hostname) 

153 and parsed.port in {None, 443} 

154 and parsed.username is None 

155 and parsed.password is None 

156 ) 

157 except ValueError: 

158 return False 

159 

160 

161def _store_content(root: Path, content: bytes, digest: str) -> str: 

162 root.mkdir(mode=0o700, parents=True, exist_ok=True) 

163 directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) 

164 nofollow = getattr(os, "O_NOFOLLOW", 0) 

165 root_fd = os.open(root, directory_flags) 

166 try: 

167 os.fchmod(root_fd, 0o700) 

168 bucket = digest[:2] 

169 try: 

170 os.mkdir(bucket, mode=0o700, dir_fd=root_fd) 

171 except FileExistsError: 

172 pass 

173 bucket_fd = os.open(bucket, directory_flags, dir_fd=root_fd) 

174 try: 

175 os.fchmod(bucket_fd, 0o700) 

176 name = digest 

177 temporary = f".tmp-{uuid4().hex}" 

178 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow 

179 fd = os.open(temporary, flags, 0o600, dir_fd=bucket_fd) 

180 with os.fdopen(fd, "wb") as stream: 

181 stream.write(content) 

182 stream.flush() 

183 os.fsync(stream.fileno()) 

184 try: 

185 os.link( 

186 temporary, 

187 name, 

188 src_dir_fd=bucket_fd, 

189 dst_dir_fd=bucket_fd, 

190 follow_symlinks=False, 

191 ) 

192 except FileExistsError: 

193 existing_fd = os.open(name, os.O_RDONLY | nofollow, dir_fd=bucket_fd) 

194 try: 

195 existing_stat = os.fstat(existing_fd) 

196 if not stat.S_ISREG(existing_stat.st_mode) or existing_stat.st_size != len( 

197 content 

198 ): 

199 raise OSError("managed attachment content-address collision") 

200 with os.fdopen(os.dup(existing_fd), "rb") as existing: 

201 existing_digest = hashlib.sha256( 

202 existing.read(MAX_ATTACHMENT_BYTES + 1) 

203 ).hexdigest() 

204 if existing_digest != digest: 

205 raise OSError("managed attachment content-address collision") 

206 finally: 

207 os.close(existing_fd) 

208 finally: 

209 os.unlink(temporary, dir_fd=bucket_fd) 

210 os.fsync(bucket_fd) 

211 finally: 

212 os.close(bucket_fd) 

213 finally: 

214 os.close(root_fd) 

215 return (Path("attachments") / digest[:2] / digest).as_posix() 

216 

217 

218def read_managed_content( 

219 root: Path, 

220 relative_path: str, 

221 *, 

222 expected_digest: str, 

223 expected_size: int, 

224) -> bytes: 

225 """Read a registered content-addressed file without following untrusted paths.""" 

226 match = _CONTENT_ADDRESS.fullmatch(relative_path) 

227 if ( 227 ↛ 235line 227 didn't jump to line 235 because the condition on line 227 was never true

228 match is None 

229 or match.group(1) != expected_digest[:2] 

230 or match.group(2) != expected_digest 

231 or not re.fullmatch(r"[0-9a-f]{64}", expected_digest) 

232 or type(expected_size) is not int 

233 or not 0 <= expected_size <= MAX_ATTACHMENT_BYTES 

234 ): 

235 raise ValueError("attachment_evidence_unavailable") 

236 directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) 

237 nofollow = getattr(os, "O_NOFOLLOW", 0) 

238 root_fd = os.open(root, directory_flags) 

239 try: 

240 bucket_fd = os.open(match.group(1), directory_flags, dir_fd=root_fd) 

241 try: 

242 fd = os.open(match.group(2), os.O_RDONLY | nofollow, dir_fd=bucket_fd) 

243 try: 

244 info = os.fstat(fd) 

245 if not stat.S_ISREG(info.st_mode) or info.st_size != expected_size: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true

246 raise ValueError("attachment_evidence_unavailable") 

247 with os.fdopen(os.dup(fd), "rb") as stream: 

248 content = stream.read(MAX_ATTACHMENT_BYTES + 1) 

249 finally: 

250 os.close(fd) 

251 finally: 

252 os.close(bucket_fd) 

253 finally: 

254 os.close(root_fd) 

255 if len(content) != expected_size or hashlib.sha256(content).hexdigest() != expected_digest: 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 raise ValueError("attachment_evidence_unavailable") 

257 return content