Coverage for downtify/cookies.py: 90%
168 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Managed storage for the YouTube ``cookies.txt`` used by yt-dlp.
3Cookies are what get age-restricted (explicit) tracks and bot-challenged
4downloads working, but until now the only way to supply them was the
5``DOWNTIFY_COOKIES_FILE`` environment variable plus a bind mount — a
6non-starter on Docker Desktop for Windows, and a frequent source of
7"downloads just fail" reports (henriquesebastiao/downtify#280).
9This module keeps an uploaded ``cookies.txt`` inside the ``/data``
10volume, so it survives container updates the same way ``settings.json``
11and the Playlist Monitor database do. The environment variable still
12wins when it's set: a deployment that mounts its own cookie file stays
13in charge, and the web UI shows the upload controls as locked.
14"""
16from __future__ import annotations
18import contextlib
19import os
20import shutil
21import tempfile
22import threading
23from collections.abc import Iterator
24from datetime import datetime, timezone
25from http.cookiejar import Cookie
26from pathlib import Path
27from typing import Any, Optional
29from loguru import logger
30from yt_dlp.cookies import YoutubeDLCookieJar
32#: Environment variable that pins a cookie file, bypassing this store.
33ENV_VAR = 'DOWNTIFY_COOKIES_FILE'
35#: Cookie jars are a few KB at most; anything larger isn't a cookies.txt.
36MAX_COOKIES_BYTES = 2 * 1024 * 1024
38#: Netscape cookie lines have exactly these 7 tab-separated fields.
39_NETSCAPE_FIELD_COUNT = 7
41_USEFUL_DOMAINS = ('youtube.com', 'google.com')
43#: ``health`` before any check ran (or after the jar was removed).
44UNKNOWN_HEALTH: dict[str, Any] = {
45 'state': 'unknown',
46 'checked_at': None,
47 'message': None,
48 'refreshed_at': None,
49 'bot_check_at': None,
50}
53#: One cookie's identity in a jar: ``(domain, path, name)``.
54CookieKey = tuple[str, str, str]
56#: Serializes every write of a shared cookies.txt in this process, so two
57#: downloads finishing together can't interleave their merges.
58_WRITE_LOCK = threading.Lock()
61class InvalidCookiesFile(ValueError):
62 """Raised when an upload isn't a usable Netscape cookies.txt."""
65def _cookie_lines(text: str) -> list[list[str]]:
66 """Split *text* into its Netscape cookie records.
68 Comments and blank lines are dropped. ``#HttpOnly_`` is a real
69 prefix browsers' export extensions write on otherwise normal
70 records, so it's unwrapped rather than treated as a comment.
71 """
73 records: list[list[str]] = []
74 for raw_line in text.splitlines():
75 line = raw_line.strip()
76 if not line:
77 continue
78 if line.startswith('#HttpOnly_'):
79 line = line[len('#HttpOnly_') :]
80 elif line.startswith('#'):
81 continue
82 fields = line.split('\t')
83 if len(fields) == _NETSCAPE_FIELD_COUNT:
84 records.append(fields)
85 return records
88def validate_cookies(content: bytes) -> list[str]:
89 """Validate an uploaded cookie jar, returning non-fatal warnings.
91 Raises :class:`InvalidCookiesFile` when the upload can't work at all
92 — that's far kinder than accepting it and letting every download
93 fail later with an opaque yt-dlp error.
94 """
96 if not content.strip():
97 raise InvalidCookiesFile('The file is empty.')
98 if len(content) > MAX_COOKIES_BYTES:
99 raise InvalidCookiesFile(
100 f'The file is larger than '
101 f'{MAX_COOKIES_BYTES // (1024 * 1024)} MB, so it is not a '
102 f'cookies.txt.'
103 )
104 try:
105 text = content.decode('utf-8')
106 except UnicodeDecodeError as exc:
107 raise InvalidCookiesFile(
108 'The file is not UTF-8 text. Export it as a Netscape '
109 'cookies.txt, not as a binary or spreadsheet file.'
110 ) from exc
112 records = _cookie_lines(text)
113 if not records:
114 raise InvalidCookiesFile(
115 'No cookies found. The file must be in Netscape format — one '
116 'cookie per line, fields separated by tabs. Use a browser '
117 'extension such as "Get cookies.txt LOCALLY" to export it.'
118 )
120 warnings: list[str] = []
121 domains = {fields[0].lstrip('.').lower() for fields in records}
122 if not any(
123 domain.endswith(useful)
124 for domain in domains
125 for useful in _USEFUL_DOMAINS
126 ):
127 warnings.append(
128 'No youtube.com or google.com cookies found in this file. '
129 'Export it from a tab logged into YouTube, otherwise it '
130 "won't help with age-restricted tracks."
131 )
132 return warnings
135def load_jar(path: Path | str) -> dict[CookieKey, Cookie]:
136 """Every cookie in the Netscape file at *path*, keyed by identity.
138 An unreadable or malformed file reads as empty: callers use this to
139 diff and merge, never to decide whether cookies are configured.
140 """
142 jar = YoutubeDLCookieJar(str(path))
143 try:
144 jar.load()
145 except (OSError, ValueError) as exc:
146 logger.debug('Could not read cookie jar {}: {}', path, exc)
147 return {}
148 return {(c.domain, c.path, c.name): c for c in jar}
151def changed_cookies(
152 before: dict[CookieKey, Cookie], after: dict[CookieKey, Cookie]
153) -> list[Cookie]:
154 """Cookies of *after* that are new or carry a new value or expiry.
156 Cookies missing from *after* are deliberately not reported: a merge
157 only ever adds or refreshes, so a client that dropped a cookie can't
158 take it away from the shared file.
159 """
161 changed: list[Cookie] = []
162 for key, cookie in after.items():
163 old = before.get(key)
164 if (
165 old is None
166 or old.value != cookie.value
167 or old.expires != cookie.expires
168 ):
169 changed.append(cookie)
170 return changed
173def merge_into(path: Path | str, cookies: list[Cookie]) -> int:
174 """Write *cookies* into the shared jar at *path*, keeping the rest.
176 YouTube and Google rotate their session cookies (``__Secure-*PSIDTS``,
177 ``SIDCC`` and friends) as they're used; a jar that never learns the
178 new values is eventually refused ("cookies are no longer valid").
179 Only the changed cookies are applied, on top of the file as it is
180 *now*, so a slow download can't roll back a newer value another one
181 already saved. Returns how many cookies were written; 0 when the
182 file is gone or can't be written (a read-only bind mount stays
183 exactly as it was).
184 """
186 target = Path(path)
187 if not cookies or not target.is_file():
188 return 0
189 with _WRITE_LOCK:
190 jar = YoutubeDLCookieJar(str(target))
191 try:
192 jar.load()
193 except (OSError, ValueError) as exc:
194 logger.debug(
195 'Cookie merge skipped, {} unreadable: {}', target, exc
196 )
197 return 0
198 for cookie in cookies:
199 jar.set_cookie(cookie)
200 if not _save_jar(jar, target):
201 return 0
202 logger.debug('Saved {} refreshed cookie(s) to {}', len(cookies), target)
203 return len(cookies)
206def _save_jar(jar: YoutubeDLCookieJar, target: Path) -> bool:
207 """Write *jar* over *target*: atomically when the folder allows it,
208 in place when *target* is a single-file bind mount (renaming over a
209 mount point fails with ``EBUSY``). Callers hold ``_WRITE_LOCK``."""
211 tmp_path = target.with_name(f'{target.name}.merge.tmp')
212 try:
213 jar.save(str(tmp_path))
214 with contextlib.suppress(OSError):
215 tmp_path.chmod(0o600)
216 tmp_path.replace(target)
217 return True
218 except OSError:
219 with contextlib.suppress(OSError):
220 tmp_path.unlink()
221 try:
222 jar.save(str(target))
223 return True
224 except OSError as exc:
225 logger.debug('Cookie merge skipped, {} not writable: {}', target, exc)
226 return False
229@contextlib.contextmanager
230def private_cookiefile(ydl_opts: dict[str, Any]) -> Iterator[None]:
231 """Point ``cookiefile`` at a per-run copy of the cookie jar.
233 yt-dlp rewrites ``cookiefile`` on exit (truncate then write), so
234 parallel downloads sharing one cookies.txt clobber it and a
235 concurrent reader hits a half-written file ("does not look like a
236 Netscape format cookies file"). The copy keeps that write off the
237 shared path; once yt-dlp is done, the cookies YouTube rotated during
238 the run are merged back (:func:`merge_into`) so the shared jar stays
239 signed in instead of going stale after a few hours.
240 """
242 shared = ydl_opts.get('cookiefile')
243 if not shared or not os.path.isfile(shared):
244 yield
245 return
246 fd, tmp = tempfile.mkstemp(prefix='cookies-', suffix='.txt')
247 os.close(fd)
248 before: dict[CookieKey, Cookie] = {}
249 try:
250 # Under the write lock: an in-place save (see _save_jar) must
251 # not be copied half-written.
252 with _WRITE_LOCK:
253 shutil.copyfile(shared, tmp)
254 before = load_jar(tmp)
255 ydl_opts['cookiefile'] = tmp
256 yield
257 finally:
258 ydl_opts['cookiefile'] = shared
259 try:
260 merge_into(shared, changed_cookies(before, load_jar(tmp)))
261 except Exception:
262 logger.opt(exception=True).debug('Cookie merge-back failed')
263 with contextlib.suppress(OSError):
264 os.remove(tmp)
267class CookiesStore:
268 """The uploaded ``cookies.txt``, plus the env var that overrides it."""
270 def __init__(self, path: Path | str) -> None:
271 self._path = Path(path)
272 #: Last result of the keep-alive check (downtify/cookie_refresh.py),
273 #: surfaced as ``health`` by :meth:`status`.
274 self.health: dict[str, Any] = dict(UNKNOWN_HEALTH)
276 @property
277 def path(self) -> Path:
278 return self._path
280 @staticmethod
281 def env_override() -> str:
282 """``DOWNTIFY_COOKIES_FILE``, or ``''`` when unset."""
283 return os.getenv(ENV_VAR, '').strip()
285 def is_locked(self) -> bool:
286 """Whether the env var owns cookie configuration (UI read-only)."""
287 return bool(self.env_override())
289 def active_path(self) -> Optional[Path]:
290 """The cookie file yt-dlp should use, if any.
292 The environment variable wins so an existing deployment's bind
293 mount keeps working exactly as before.
294 """
295 env_path = self.env_override()
296 if env_path:
297 return Path(env_path)
298 return self._path if self._path.is_file() else None
300 def status(self) -> dict[str, Any]:
301 """Shape consumed by ``GET /api/cookies`` and the settings UI."""
302 return {**self._file_status(), 'health': dict(self.health)}
304 def _file_status(self) -> dict[str, Any]:
305 env_path = self.env_override()
306 if env_path:
307 return {
308 'configured': Path(env_path).is_file(),
309 'source': 'env',
310 'locked': True,
311 'path': env_path,
312 'size': None,
313 'updated_at': None,
314 }
315 if self._path.is_file():
316 stat = self._path.stat()
317 return {
318 'configured': True,
319 'source': 'upload',
320 'locked': False,
321 'path': str(self._path),
322 'size': stat.st_size,
323 'updated_at': datetime.fromtimestamp(
324 stat.st_mtime, tz=timezone.utc
325 ).isoformat(),
326 }
327 return {
328 'configured': False,
329 'source': None,
330 'locked': False,
331 'path': None,
332 'size': None,
333 'updated_at': None,
334 }
336 def save(self, content: bytes) -> list[str]:
337 """Validate and persist *content*, replacing any previous file.
339 Returns the non-fatal warnings from :func:`validate_cookies`.
340 """
342 warnings = validate_cookies(content)
343 self._path.parent.mkdir(parents=True, exist_ok=True)
344 # Write-then-replace so a failed write can't leave a truncated
345 # jar behind that would break every subsequent download.
346 tmp_path = self._path.with_name(f'{self._path.name}.tmp')
347 tmp_path.write_bytes(content)
348 # Session tokens: keep it out of reach of other users on the host.
349 # Best-effort — filesystems without POSIX permissions (a Windows
350 # bind mount, most notably) must not fail the upload itself.
351 with contextlib.suppress(OSError):
352 tmp_path.chmod(0o600)
353 tmp_path.replace(self._path)
354 self.health = dict(UNKNOWN_HEALTH)
355 logger.info('Stored uploaded cookies file at {}', self._path)
356 return warnings
358 def delete(self) -> bool:
359 """Remove the uploaded file. ``False`` when there was none."""
360 if not self._path.is_file():
361 return False
362 self._path.unlink()
363 self.health = dict(UNKNOWN_HEALTH)
364 logger.info('Removed uploaded cookies file at {}', self._path)
365 return True