Coverage for src/copytrading_engine/sources/attachments.py: 76%
166 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Fetch only bounded Discord CDN attachments into private operational storage."""
3import asyncio
4import hashlib
5import os
6import re
7import stat
8from collections.abc import Sequence
9from dataclasses import dataclass
10from pathlib import Path
11from urllib.parse import urljoin, urlsplit
12from uuid import uuid4
14import httpx2 as httpx
16from copytrading_engine.sources.application import AttachmentReference, AttachmentStatus
17from copytrading_engine.sources.evidence import MAX_ATTACHMENTS_PER_MESSAGE
19# A chart screenshot is commonly 1-3 MB.
20MAX_ATTACHMENT_BYTES = 4 * 1024 * 1024
21MAX_MESSAGE_ATTACHMENT_BYTES = 8 * 1024 * 1024
22MAX_ATTACHMENT_REDIRECTS = 3
23ATTACHMENT_CAPTURE_TIMEOUT_SECONDS = 15
24ALLOWED_ATTACHMENT_HOSTS = frozenset({"cdn.discordapp.com", "media.discordapp.net"})
25# Discord's own copy of a picture linked from elsewhere, as a relay's embed shows it.
26_DISCORD_IMAGE_PROXY = re.compile(r"^images-ext-[0-9]+\.discordapp\.net$")
29def discord_host(host: str | None) -> bool:
30 """Whether Discord itself serves this host's files."""
31 host = (host or "").casefold()
32 return host in ALLOWED_ATTACHMENT_HOSTS or bool(_DISCORD_IMAGE_PROXY.fullmatch(host))
35_REDIRECT_CODES = frozenset({301, 302, 303, 307, 308})
36_CONTENT_ADDRESS = re.compile(r"^attachments/([0-9a-f]{2})/([0-9a-f]{64})$")
39@dataclass(frozen=True, slots=True)
40class AttachmentCaptureResult:
41 status: AttachmentStatus
42 byte_size: int | None = None
43 sha256: str | None = None
44 managed_path: str | None = None
47async def capture_attachments(
48 references: Sequence[AttachmentReference],
49 managed_root: Path,
50 *,
51 transport: httpx.AsyncBaseTransport | None = None,
52) -> tuple[AttachmentCaptureResult, ...]:
53 """Capture at most ten files, with one shared message time and byte budget."""
54 references = tuple(references[:MAX_ATTACHMENTS_PER_MESSAGE])
55 results: list[AttachmentCaptureResult] = []
56 captured_bytes = 0
57 timeout = httpx.Timeout(5.0, connect=2.0)
58 async with httpx.AsyncClient(
59 transport=transport,
60 timeout=timeout,
61 follow_redirects=False,
62 trust_env=False,
63 ) as client:
64 try:
65 async with asyncio.timeout(ATTACHMENT_CAPTURE_TIMEOUT_SECONDS):
66 for reference in references:
67 if not reference.url:
68 results.append(AttachmentCaptureResult("missing"))
69 continue
70 if type(reference.declared_size) is not int or reference.declared_size < 0: 70 ↛ 71line 70 didn't jump to line 71 because the condition on line 70 was never true
71 results.append(AttachmentCaptureResult("download_failed"))
72 continue
73 remaining = MAX_MESSAGE_ATTACHMENT_BYTES - captured_bytes
74 if (
75 reference.declared_size > MAX_ATTACHMENT_BYTES
76 or reference.declared_size > remaining
77 ):
78 results.append(AttachmentCaptureResult("oversize"))
79 continue
80 result = await _capture_one(
81 client,
82 reference.url,
83 managed_root,
84 maximum_bytes=min(MAX_ATTACHMENT_BYTES, remaining),
85 )
86 results.append(result)
87 if result.status == "available" and result.byte_size is not None:
88 captured_bytes += result.byte_size
89 except TimeoutError:
90 results.extend(
91 AttachmentCaptureResult("timed_out") for _ in range(len(results), len(references))
92 )
93 return tuple(results)
96async def _capture_one(
97 client: httpx.AsyncClient,
98 initial_url: str,
99 managed_root: Path,
100 *,
101 maximum_bytes: int,
102) -> AttachmentCaptureResult:
103 if not _allowed_url(initial_url):
104 return AttachmentCaptureResult("origin_rejected")
105 url = initial_url
106 for redirect_count in range(MAX_ATTACHMENT_REDIRECTS + 1): 106 ↛ 144line 106 didn't jump to line 144 because the loop on line 106 didn't complete
107 try:
108 async with client.stream("GET", url) as response:
109 final_url = str(response.request.url)
110 if not _allowed_url(final_url): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true
111 return AttachmentCaptureResult("origin_rejected")
112 if response.status_code in _REDIRECT_CODES:
113 location = response.headers.get("location")
114 if not location or redirect_count == MAX_ATTACHMENT_REDIRECTS: 114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true
115 return AttachmentCaptureResult("download_failed")
116 next_url = urljoin(final_url, location)
117 if not _allowed_url(next_url): 117 ↛ 119line 117 didn't jump to line 119 because the condition on line 117 was always true
118 return AttachmentCaptureResult("origin_rejected")
119 url = next_url
120 continue
121 if not 200 <= response.status_code < 300: 121 ↛ 122line 121 didn't jump to line 122 because the condition on line 121 was never true
122 return AttachmentCaptureResult("download_failed")
123 length = response.headers.get("content-length")
124 if length is not None: 124 ↛ 130line 124 didn't jump to line 130 because the condition on line 124 was always true
125 try:
126 if int(length) > maximum_bytes: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true
127 return AttachmentCaptureResult("oversize")
128 except ValueError:
129 pass
130 body = bytearray()
131 async for chunk in response.aiter_bytes():
132 if len(body) + len(chunk) > maximum_bytes: 132 ↛ 133line 132 didn't jump to line 133 because the condition on line 132 was never true
133 return AttachmentCaptureResult("oversize")
134 body.extend(chunk)
135 digest = hashlib.sha256(body).hexdigest()
136 relative_path = await asyncio.to_thread(
137 _store_content, managed_root, bytes(body), digest
138 )
139 return AttachmentCaptureResult("available", len(body), digest, relative_path)
140 except httpx.TimeoutException:
141 return AttachmentCaptureResult("timed_out")
142 except httpx.RequestError, OSError, ValueError:
143 return AttachmentCaptureResult("download_failed")
144 return AttachmentCaptureResult("download_failed")
147def _allowed_url(value: str) -> bool:
148 try:
149 parsed = urlsplit(value)
150 return (
151 parsed.scheme == "https"
152 and discord_host(parsed.hostname)
153 and parsed.port in {None, 443}
154 and parsed.username is None
155 and parsed.password is None
156 )
157 except ValueError:
158 return False
161def _store_content(root: Path, content: bytes, digest: str) -> str:
162 root.mkdir(mode=0o700, parents=True, exist_ok=True)
163 directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
164 nofollow = getattr(os, "O_NOFOLLOW", 0)
165 root_fd = os.open(root, directory_flags)
166 try:
167 os.fchmod(root_fd, 0o700)
168 bucket = digest[:2]
169 try:
170 os.mkdir(bucket, mode=0o700, dir_fd=root_fd)
171 except FileExistsError:
172 pass
173 bucket_fd = os.open(bucket, directory_flags, dir_fd=root_fd)
174 try:
175 os.fchmod(bucket_fd, 0o700)
176 name = digest
177 temporary = f".tmp-{uuid4().hex}"
178 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow
179 fd = os.open(temporary, flags, 0o600, dir_fd=bucket_fd)
180 with os.fdopen(fd, "wb") as stream:
181 stream.write(content)
182 stream.flush()
183 os.fsync(stream.fileno())
184 try:
185 os.link(
186 temporary,
187 name,
188 src_dir_fd=bucket_fd,
189 dst_dir_fd=bucket_fd,
190 follow_symlinks=False,
191 )
192 except FileExistsError:
193 existing_fd = os.open(name, os.O_RDONLY | nofollow, dir_fd=bucket_fd)
194 try:
195 existing_stat = os.fstat(existing_fd)
196 if not stat.S_ISREG(existing_stat.st_mode) or existing_stat.st_size != len(
197 content
198 ):
199 raise OSError("managed attachment content-address collision")
200 with os.fdopen(os.dup(existing_fd), "rb") as existing:
201 existing_digest = hashlib.sha256(
202 existing.read(MAX_ATTACHMENT_BYTES + 1)
203 ).hexdigest()
204 if existing_digest != digest:
205 raise OSError("managed attachment content-address collision")
206 finally:
207 os.close(existing_fd)
208 finally:
209 os.unlink(temporary, dir_fd=bucket_fd)
210 os.fsync(bucket_fd)
211 finally:
212 os.close(bucket_fd)
213 finally:
214 os.close(root_fd)
215 return (Path("attachments") / digest[:2] / digest).as_posix()
218def read_managed_content(
219 root: Path,
220 relative_path: str,
221 *,
222 expected_digest: str,
223 expected_size: int,
224) -> bytes:
225 """Read a registered content-addressed file without following untrusted paths."""
226 match = _CONTENT_ADDRESS.fullmatch(relative_path)
227 if ( 227 ↛ 235line 227 didn't jump to line 235 because the condition on line 227 was never true
228 match is None
229 or match.group(1) != expected_digest[:2]
230 or match.group(2) != expected_digest
231 or not re.fullmatch(r"[0-9a-f]{64}", expected_digest)
232 or type(expected_size) is not int
233 or not 0 <= expected_size <= MAX_ATTACHMENT_BYTES
234 ):
235 raise ValueError("attachment_evidence_unavailable")
236 directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
237 nofollow = getattr(os, "O_NOFOLLOW", 0)
238 root_fd = os.open(root, directory_flags)
239 try:
240 bucket_fd = os.open(match.group(1), directory_flags, dir_fd=root_fd)
241 try:
242 fd = os.open(match.group(2), os.O_RDONLY | nofollow, dir_fd=bucket_fd)
243 try:
244 info = os.fstat(fd)
245 if not stat.S_ISREG(info.st_mode) or info.st_size != expected_size: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true
246 raise ValueError("attachment_evidence_unavailable")
247 with os.fdopen(os.dup(fd), "rb") as stream:
248 content = stream.read(MAX_ATTACHMENT_BYTES + 1)
249 finally:
250 os.close(fd)
251 finally:
252 os.close(bucket_fd)
253 finally:
254 os.close(root_fd)
255 if len(content) != expected_size or hashlib.sha256(content).hexdigest() != expected_digest: 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true
256 raise ValueError("attachment_evidence_unavailable")
257 return content