Coverage for downtify/cookies.py: 90%

168 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Managed storage for the YouTube ``cookies.txt`` used by yt-dlp. 

2 

3Cookies are what get age-restricted (explicit) tracks and bot-challenged 

4downloads working, but until now the only way to supply them was the 

5``DOWNTIFY_COOKIES_FILE`` environment variable plus a bind mount — a 

6non-starter on Docker Desktop for Windows, and a frequent source of 

7"downloads just fail" reports (henriquesebastiao/downtify#280). 

8 

9This module keeps an uploaded ``cookies.txt`` inside the ``/data`` 

10volume, so it survives container updates the same way ``settings.json`` 

11and the Playlist Monitor database do. The environment variable still 

12wins when it's set: a deployment that mounts its own cookie file stays 

13in charge, and the web UI shows the upload controls as locked. 

14""" 

15 

16from __future__ import annotations 

17 

18import contextlib 

19import os 

20import shutil 

21import tempfile 

22import threading 

23from collections.abc import Iterator 

24from datetime import datetime, timezone 

25from http.cookiejar import Cookie 

26from pathlib import Path 

27from typing import Any, Optional 

28 

29from loguru import logger 

30from yt_dlp.cookies import YoutubeDLCookieJar 

31 

32#: Environment variable that pins a cookie file, bypassing this store. 

33ENV_VAR = 'DOWNTIFY_COOKIES_FILE' 

34 

35#: Cookie jars are a few KB at most; anything larger isn't a cookies.txt. 

36MAX_COOKIES_BYTES = 2 * 1024 * 1024 

37 

38#: Netscape cookie lines have exactly these 7 tab-separated fields. 

39_NETSCAPE_FIELD_COUNT = 7 

40 

41_USEFUL_DOMAINS = ('youtube.com', 'google.com') 

42 

43#: ``health`` before any check ran (or after the jar was removed). 

44UNKNOWN_HEALTH: dict[str, Any] = { 

45 'state': 'unknown', 

46 'checked_at': None, 

47 'message': None, 

48 'refreshed_at': None, 

49 'bot_check_at': None, 

50} 

51 

52 

53#: One cookie's identity in a jar: ``(domain, path, name)``. 

54CookieKey = tuple[str, str, str] 

55 

56#: Serializes every write of a shared cookies.txt in this process, so two 

57#: downloads finishing together can't interleave their merges. 

58_WRITE_LOCK = threading.Lock() 

59 

60 

61class InvalidCookiesFile(ValueError): 

62 """Raised when an upload isn't a usable Netscape cookies.txt.""" 

63 

64 

65def _cookie_lines(text: str) -> list[list[str]]: 

66 """Split *text* into its Netscape cookie records. 

67 

68 Comments and blank lines are dropped. ``#HttpOnly_`` is a real 

69 prefix browsers' export extensions write on otherwise normal 

70 records, so it's unwrapped rather than treated as a comment. 

71 """ 

72 

73 records: list[list[str]] = [] 

74 for raw_line in text.splitlines(): 

75 line = raw_line.strip() 

76 if not line: 

77 continue 

78 if line.startswith('#HttpOnly_'): 

79 line = line[len('#HttpOnly_') :] 

80 elif line.startswith('#'): 

81 continue 

82 fields = line.split('\t') 

83 if len(fields) == _NETSCAPE_FIELD_COUNT: 

84 records.append(fields) 

85 return records 

86 

87 

88def validate_cookies(content: bytes) -> list[str]: 

89 """Validate an uploaded cookie jar, returning non-fatal warnings. 

90 

91 Raises :class:`InvalidCookiesFile` when the upload can't work at all 

92 — that's far kinder than accepting it and letting every download 

93 fail later with an opaque yt-dlp error. 

94 """ 

95 

96 if not content.strip(): 

97 raise InvalidCookiesFile('The file is empty.') 

98 if len(content) > MAX_COOKIES_BYTES: 

99 raise InvalidCookiesFile( 

100 f'The file is larger than ' 

101 f'{MAX_COOKIES_BYTES // (1024 * 1024)} MB, so it is not a ' 

102 f'cookies.txt.' 

103 ) 

104 try: 

105 text = content.decode('utf-8') 

106 except UnicodeDecodeError as exc: 

107 raise InvalidCookiesFile( 

108 'The file is not UTF-8 text. Export it as a Netscape ' 

109 'cookies.txt, not as a binary or spreadsheet file.' 

110 ) from exc 

111 

112 records = _cookie_lines(text) 

113 if not records: 

114 raise InvalidCookiesFile( 

115 'No cookies found. The file must be in Netscape format — one ' 

116 'cookie per line, fields separated by tabs. Use a browser ' 

117 'extension such as "Get cookies.txt LOCALLY" to export it.' 

118 ) 

119 

120 warnings: list[str] = [] 

121 domains = {fields[0].lstrip('.').lower() for fields in records} 

122 if not any( 

123 domain.endswith(useful) 

124 for domain in domains 

125 for useful in _USEFUL_DOMAINS 

126 ): 

127 warnings.append( 

128 'No youtube.com or google.com cookies found in this file. ' 

129 'Export it from a tab logged into YouTube, otherwise it ' 

130 "won't help with age-restricted tracks." 

131 ) 

132 return warnings 

133 

134 

135def load_jar(path: Path | str) -> dict[CookieKey, Cookie]: 

136 """Every cookie in the Netscape file at *path*, keyed by identity. 

137 

138 An unreadable or malformed file reads as empty: callers use this to 

139 diff and merge, never to decide whether cookies are configured. 

140 """ 

141 

142 jar = YoutubeDLCookieJar(str(path)) 

143 try: 

144 jar.load() 

145 except (OSError, ValueError) as exc: 

146 logger.debug('Could not read cookie jar {}: {}', path, exc) 

147 return {} 

148 return {(c.domain, c.path, c.name): c for c in jar} 

149 

150 

151def changed_cookies( 

152 before: dict[CookieKey, Cookie], after: dict[CookieKey, Cookie] 

153) -> list[Cookie]: 

154 """Cookies of *after* that are new or carry a new value or expiry. 

155 

156 Cookies missing from *after* are deliberately not reported: a merge 

157 only ever adds or refreshes, so a client that dropped a cookie can't 

158 take it away from the shared file. 

159 """ 

160 

161 changed: list[Cookie] = [] 

162 for key, cookie in after.items(): 

163 old = before.get(key) 

164 if ( 

165 old is None 

166 or old.value != cookie.value 

167 or old.expires != cookie.expires 

168 ): 

169 changed.append(cookie) 

170 return changed 

171 

172 

173def merge_into(path: Path | str, cookies: list[Cookie]) -> int: 

174 """Write *cookies* into the shared jar at *path*, keeping the rest. 

175 

176 YouTube and Google rotate their session cookies (``__Secure-*PSIDTS``, 

177 ``SIDCC`` and friends) as they're used; a jar that never learns the 

178 new values is eventually refused ("cookies are no longer valid"). 

179 Only the changed cookies are applied, on top of the file as it is 

180 *now*, so a slow download can't roll back a newer value another one 

181 already saved. Returns how many cookies were written; 0 when the 

182 file is gone or can't be written (a read-only bind mount stays 

183 exactly as it was). 

184 """ 

185 

186 target = Path(path) 

187 if not cookies or not target.is_file(): 

188 return 0 

189 with _WRITE_LOCK: 

190 jar = YoutubeDLCookieJar(str(target)) 

191 try: 

192 jar.load() 

193 except (OSError, ValueError) as exc: 

194 logger.debug( 

195 'Cookie merge skipped, {} unreadable: {}', target, exc 

196 ) 

197 return 0 

198 for cookie in cookies: 

199 jar.set_cookie(cookie) 

200 if not _save_jar(jar, target): 

201 return 0 

202 logger.debug('Saved {} refreshed cookie(s) to {}', len(cookies), target) 

203 return len(cookies) 

204 

205 

206def _save_jar(jar: YoutubeDLCookieJar, target: Path) -> bool: 

207 """Write *jar* over *target*: atomically when the folder allows it, 

208 in place when *target* is a single-file bind mount (renaming over a 

209 mount point fails with ``EBUSY``). Callers hold ``_WRITE_LOCK``.""" 

210 

211 tmp_path = target.with_name(f'{target.name}.merge.tmp') 

212 try: 

213 jar.save(str(tmp_path)) 

214 with contextlib.suppress(OSError): 

215 tmp_path.chmod(0o600) 

216 tmp_path.replace(target) 

217 return True 

218 except OSError: 

219 with contextlib.suppress(OSError): 

220 tmp_path.unlink() 

221 try: 

222 jar.save(str(target)) 

223 return True 

224 except OSError as exc: 

225 logger.debug('Cookie merge skipped, {} not writable: {}', target, exc) 

226 return False 

227 

228 

229@contextlib.contextmanager 

230def private_cookiefile(ydl_opts: dict[str, Any]) -> Iterator[None]: 

231 """Point ``cookiefile`` at a per-run copy of the cookie jar. 

232 

233 yt-dlp rewrites ``cookiefile`` on exit (truncate then write), so 

234 parallel downloads sharing one cookies.txt clobber it and a 

235 concurrent reader hits a half-written file ("does not look like a 

236 Netscape format cookies file"). The copy keeps that write off the 

237 shared path; once yt-dlp is done, the cookies YouTube rotated during 

238 the run are merged back (:func:`merge_into`) so the shared jar stays 

239 signed in instead of going stale after a few hours. 

240 """ 

241 

242 shared = ydl_opts.get('cookiefile') 

243 if not shared or not os.path.isfile(shared): 

244 yield 

245 return 

246 fd, tmp = tempfile.mkstemp(prefix='cookies-', suffix='.txt') 

247 os.close(fd) 

248 before: dict[CookieKey, Cookie] = {} 

249 try: 

250 # Under the write lock: an in-place save (see _save_jar) must 

251 # not be copied half-written. 

252 with _WRITE_LOCK: 

253 shutil.copyfile(shared, tmp) 

254 before = load_jar(tmp) 

255 ydl_opts['cookiefile'] = tmp 

256 yield 

257 finally: 

258 ydl_opts['cookiefile'] = shared 

259 try: 

260 merge_into(shared, changed_cookies(before, load_jar(tmp))) 

261 except Exception: 

262 logger.opt(exception=True).debug('Cookie merge-back failed') 

263 with contextlib.suppress(OSError): 

264 os.remove(tmp) 

265 

266 

267class CookiesStore: 

268 """The uploaded ``cookies.txt``, plus the env var that overrides it.""" 

269 

270 def __init__(self, path: Path | str) -> None: 

271 self._path = Path(path) 

272 #: Last result of the keep-alive check (downtify/cookie_refresh.py), 

273 #: surfaced as ``health`` by :meth:`status`. 

274 self.health: dict[str, Any] = dict(UNKNOWN_HEALTH) 

275 

276 @property 

277 def path(self) -> Path: 

278 return self._path 

279 

280 @staticmethod 

281 def env_override() -> str: 

282 """``DOWNTIFY_COOKIES_FILE``, or ``''`` when unset.""" 

283 return os.getenv(ENV_VAR, '').strip() 

284 

285 def is_locked(self) -> bool: 

286 """Whether the env var owns cookie configuration (UI read-only).""" 

287 return bool(self.env_override()) 

288 

289 def active_path(self) -> Optional[Path]: 

290 """The cookie file yt-dlp should use, if any. 

291 

292 The environment variable wins so an existing deployment's bind 

293 mount keeps working exactly as before. 

294 """ 

295 env_path = self.env_override() 

296 if env_path: 

297 return Path(env_path) 

298 return self._path if self._path.is_file() else None 

299 

300 def status(self) -> dict[str, Any]: 

301 """Shape consumed by ``GET /api/cookies`` and the settings UI.""" 

302 return {**self._file_status(), 'health': dict(self.health)} 

303 

304 def _file_status(self) -> dict[str, Any]: 

305 env_path = self.env_override() 

306 if env_path: 

307 return { 

308 'configured': Path(env_path).is_file(), 

309 'source': 'env', 

310 'locked': True, 

311 'path': env_path, 

312 'size': None, 

313 'updated_at': None, 

314 } 

315 if self._path.is_file(): 

316 stat = self._path.stat() 

317 return { 

318 'configured': True, 

319 'source': 'upload', 

320 'locked': False, 

321 'path': str(self._path), 

322 'size': stat.st_size, 

323 'updated_at': datetime.fromtimestamp( 

324 stat.st_mtime, tz=timezone.utc 

325 ).isoformat(), 

326 } 

327 return { 

328 'configured': False, 

329 'source': None, 

330 'locked': False, 

331 'path': None, 

332 'size': None, 

333 'updated_at': None, 

334 } 

335 

336 def save(self, content: bytes) -> list[str]: 

337 """Validate and persist *content*, replacing any previous file. 

338 

339 Returns the non-fatal warnings from :func:`validate_cookies`. 

340 """ 

341 

342 warnings = validate_cookies(content) 

343 self._path.parent.mkdir(parents=True, exist_ok=True) 

344 # Write-then-replace so a failed write can't leave a truncated 

345 # jar behind that would break every subsequent download. 

346 tmp_path = self._path.with_name(f'{self._path.name}.tmp') 

347 tmp_path.write_bytes(content) 

348 # Session tokens: keep it out of reach of other users on the host. 

349 # Best-effort — filesystems without POSIX permissions (a Windows 

350 # bind mount, most notably) must not fail the upload itself. 

351 with contextlib.suppress(OSError): 

352 tmp_path.chmod(0o600) 

353 tmp_path.replace(self._path) 

354 self.health = dict(UNKNOWN_HEALTH) 

355 logger.info('Stored uploaded cookies file at {}', self._path) 

356 return warnings 

357 

358 def delete(self) -> bool: 

359 """Remove the uploaded file. ``False`` when there was none.""" 

360 if not self._path.is_file(): 

361 return False 

362 self._path.unlink() 

363 self.health = dict(UNKNOWN_HEALTH) 

364 logger.info('Removed uploaded cookies file at {}', self._path) 

365 return True