Coverage for downtify/stream.py: 79%
207 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Full-track streaming, served from this server.
3A track the library doesn't have yet still plays in full, in the
4built-in player, without a library download: the frontend resolves it
5to a YouTube video id and plays ``GET /api/stream/file``, which the
6:class:`StreamCache` below downloads once (via yt-dlp, with the same
7anti-bot posture as a real download) into ``/data/stream_cache`` and
8then serves with seeking support. This is what makes streaming work
9where the browser itself can't reach YouTube (or only slowly): the
10only YouTube traffic is server-side, and the second play of a track
11never touches YouTube at all.
13Two endpoints share this module: ``GET /api/stream`` hands out a
14fresh direct-audio URL for third-party clients, ``GET
15/api/stream/file`` serves the cached file to the web player.
17Failures surface as :class:`ValueError` (same convention as
18:mod:`downtify.deezer`), so the endpoints map them to ``503``.
19"""
21from __future__ import annotations
23import asyncio
24import contextlib
25import os
26import re
27import time
28from pathlib import Path
29from typing import Any, Optional
31import yt_dlp
32from loguru import logger
33from mutagen import File as _mutagen_file
35from .cookies import private_cookiefile
36from .downloader import (
37 MAX_TRACK_DURATION_SECONDS,
38 TrackTooLongError,
39 _yt_player_clients,
40 _YtdlpLogger,
41 describe_duration,
42 is_age_restricted_error,
43)
45#: What a YouTube video id looks like — anything else is rejected
46#: without touching the network.
47VIDEO_ID_RE = re.compile(r'^[A-Za-z0-9_-]{11}$')
49_TIMEOUT = 30
52def clean_video_id(video_id: str) -> str:
53 """*video_id* when it is shaped like a YouTube video id."""
55 candidate = str(video_id or '').strip()
56 if not VIDEO_ID_RE.fullmatch(candidate):
57 raise ValueError(f'Not a YouTube video id: {candidate[:32]!r}')
58 return candidate
61def _ydl_options() -> dict[str, Any]:
62 """yt-dlp options for reading (not downloading) one audio URL.
64 Mirrors the anti-bot posture of a real download (see
65 ``Downloader._ydl_options``): the TV/mobile player clients that
66 bypass "Sign in to confirm you're not a bot", the EJS challenge
67 solver, IPv4 when asked, and the same cookies.
68 """
70 opts: dict[str, Any] = {
71 'format': 'bestaudio[ext=m4a]/bestaudio/best',
72 'quiet': True,
73 'noprogress': True,
74 'logger': _YtdlpLogger(),
75 'noplaylist': True,
76 'nocheckcertificate': True,
77 'socket_timeout': _TIMEOUT,
78 'extractor_args': {'youtube': {'player_client': _yt_player_clients()}},
79 'remote_components': ['ejs:github'],
80 }
81 if os.getenv('DOWNTIFY_FORCE_IPV4', '').strip() in {'1', 'true', 'yes'}:
82 opts['source_address'] = '0.0.0.0'
83 return opts
86def stream_url_for_video(
87 video_id: str, *, cookies_file: str = ''
88) -> dict[str, Any]:
89 """The current direct-audio URL of the YouTube video *video_id*.
91 Returns ``{url, ext}`` — ``ext`` is ``m4a`` when YouTube served
92 AAC (plays in every browser) and whatever ``bestaudio`` was
93 otherwise. Raises :class:`ValueError` when the id is malformed,
94 age-gated without usable cookies, or YouTube can't be reached.
95 The URL itself is never logged: it carries a signature.
96 """
98 vid = clean_video_id(video_id)
99 opts = _ydl_options()
100 resolved = str(cookies_file or '').strip()
101 if resolved:
102 opts['cookiefile'] = resolved
103 cookies_browser = os.getenv('DOWNTIFY_COOKIES_FROM_BROWSER', '').strip()
104 if cookies_browser:
105 parts = cookies_browser.split(':', 1)
106 opts['cookiesfrombrowser'] = (
107 (parts[0],) if len(parts) == 1 else (parts[0], parts[1])
108 )
109 url = f'https://www.youtube.com/watch?v={vid}'
110 try:
111 with private_cookiefile(opts), yt_dlp.YoutubeDL(opts) as ydl:
112 info = ydl.extract_info(url, download=False)
113 except Exception as exc:
114 if is_age_restricted_error(str(exc)):
115 raise ValueError(
116 'This track is age-restricted on YouTube: upload a '
117 'YouTube cookies.txt in Settings > YouTube cookies '
118 'to stream it.'
119 ) from exc
120 raise ValueError(
121 f'YouTube did not answer: {type(exc).__name__}'
122 ) from exc
123 if not isinstance(info, dict) or not info.get('url'):
124 raise ValueError('YouTube answered with an unreadable reply')
125 direct = str(info['url'])
126 logger.info(
127 'Stream URL resolved for videoId={} ext={!r}', vid, info.get('ext')
128 )
129 return {'url': direct, 'ext': str(info.get('ext') or '')}
132def resolve_cookies_file(store: Optional[Any]) -> str:
133 """The cookies.txt streaming should use (UI upload wins, env next)."""
135 try:
136 active = store.active_path() if store is not None else None
137 except Exception:
138 active = None
139 if active:
140 return str(active)
141 return os.getenv('DOWNTIFY_COOKIES_FILE', '').strip()
144#: Default cap of the on-disk stream cache (see ``stream_cache_from_env``).
145DEFAULT_STREAM_CACHE_MB = 1024
146#: How many stream downloads may run at once (they are heavier than
147#: metadata lookups but lighter than full tagged downloads).
148DEFAULT_STREAM_CONCURRENCY = 2
150_MEDIA_TYPES = {
151 'm4a': 'audio/mp4',
152 'mp4': 'audio/mp4',
153 'aac': 'audio/aac',
154 'mp3': 'audio/mpeg',
155 'webm': 'audio/webm',
156 'opus': 'audio/ogg',
157 'ogg': 'audio/ogg',
158 'flac': 'audio/flac',
159 'wav': 'audio/wav',
160}
163def media_type_for(path: Any) -> str:
164 """The ``Content-Type`` a cached stream file is served with."""
166 ext = str(getattr(path, 'suffix', '') or '').lower().lstrip('.')
167 return _MEDIA_TYPES.get(ext, 'application/octet-stream')
170#: Detail for the ``400`` when a stream is refused for length (mirrors
171#: the download message in ``downtify.downloader``; no name here, only
172#: a video id reached this layer).
173STREAM_TOO_LONG_DETAIL = (
174 'This track runs over 10 minutes; tracks over 10 minutes are not streamed'
175)
178def too_long_detail(max_seconds: int) -> str:
179 """The refusal for the configured bound (``0`` is never refused)."""
181 bound = describe_duration(max_seconds)
182 return (
183 f'This track runs over {bound}; tracks over {bound} are not streamed'
184 )
187def _probed_seconds(path: Path) -> float:
188 """The audio length of a cached file in seconds, ``0`` when unknown.
190 Read off the downloaded file itself (no extra YouTube request), so
191 the duration cap holds no matter which client asked. Fail-open: an
192 unreadable file is served rather than breaking playback.
193 """
195 try:
196 audio = _mutagen_file(path)
197 if audio is not None and audio.info is not None:
198 return float(getattr(audio.info, 'length', 0) or 0)
199 except Exception:
200 logger.opt(exception=True).debug('Stream length probe failed')
201 return 0.0
204class _StreamJob:
205 """One cache download, shared by every request waiting for it."""
207 def __init__(self, task: asyncio.Task[Path]) -> None:
208 self.task = task
209 self.waiters = 0
210 self.fetch_started = False
213class StreamCache:
214 """On-disk cache of streamed audio (see the module docstring).
216 ``get`` downloads a video's audio once no matter how many requests
217 arrive together, serves the finished file afterwards, and prunes
218 the least recently used files past the size limit. Files are named
219 ``<video_id>.<ext>`` with ``.part`` left to yt-dlp while a download
220 is in flight (never served: only finished files match).
221 """
223 def __init__(
224 self,
225 cache_dir: Any,
226 *,
227 max_bytes: int = DEFAULT_STREAM_CACHE_MB * 1024**2,
228 max_concurrent: int = DEFAULT_STREAM_CONCURRENCY,
229 ) -> None:
230 self._dir = (
231 Path(cache_dir) if not isinstance(cache_dir, Path) else cache_dir
232 )
233 self._max_bytes = max(0, int(max_bytes))
234 self._semaphore = asyncio.Semaphore(max(1, int(max_concurrent)))
235 self._jobs: dict[str, _StreamJob] = {}
236 self.running = 0
238 def cached_path(self, video_id: str) -> Optional[Path]:
239 """The finished cached file for *video_id*, if one is stored."""
241 try:
242 vid = clean_video_id(video_id)
243 except ValueError:
244 return None
245 for path in sorted(self._dir.glob(f'{vid}.*')):
246 if path.name.endswith('.part') or not path.is_file():
247 continue
248 try:
249 stat = path.stat()
250 if stat.st_size <= 0:
251 continue
252 with contextlib.suppress(OSError):
253 # Track cache use without changing the HTTP file validator.
254 os.utime(path, ns=(time.time_ns(), stat.st_mtime_ns))
255 except OSError:
256 continue
257 return path
258 return None
260 def streamable_path(
261 self, video_id: str, max_seconds: Optional[int] = None
262 ) -> Optional[Path]:
263 """The finished cached file for *video_id*, enforcing the cap.
265 A cached file past the bound is deleted and raises
266 :class:`TrackTooLongError` instead of being served, so long
267 tracks never accumulate on disk. ``None`` keeps the default
268 bound, ``0`` serves everything.
269 """
271 limit = (
272 MAX_TRACK_DURATION_SECONDS if max_seconds is None else max_seconds
273 )
274 path = self.cached_path(video_id)
275 if path is None:
276 return None
277 if limit and _probed_seconds(path) > limit:
278 with contextlib.suppress(OSError):
279 path.unlink()
280 logger.warning(
281 'Stream cache refused an overlong file for videoId={}',
282 video_id,
283 )
284 raise TrackTooLongError(too_long_detail(limit))
285 return path
287 async def get(
288 self,
289 video_id: str,
290 *,
291 cookies_file: str = '',
292 max_seconds: Optional[int] = None,
293 ) -> Path:
294 """The cached audio file for *video_id*, downloading it first.
296 Concurrent callers share one download. A queued download is
297 cancelled once nobody waits for it; a running download finishes
298 into the cache, keeping its concurrency slot and remaining
299 available to new callers. Raises :class:`ValueError` when there
300 is nothing to serve.
301 """
303 vid = clean_video_id(video_id)
304 cached = self.streamable_path(vid, max_seconds)
305 if cached is not None:
306 return cached
307 job = self._jobs.get(vid)
308 if job is None or job.task.cancelling():
309 job = _StreamJob(
310 asyncio.ensure_future(
311 self._run(vid, cookies_file, max_seconds)
312 )
313 )
314 self._jobs[vid] = job
315 job.task.add_done_callback(lambda task: self._job_done(vid, task))
316 job.waiters += 1
317 try:
318 return await asyncio.shield(job.task)
319 finally:
320 job.waiters -= 1
321 if (
322 job.waiters <= 0
323 and not job.fetch_started
324 and not job.task.done()
325 ):
326 job.task.cancel()
328 def _job_done(self, video_id: str, task: asyncio.Task[Path]) -> None:
329 job = self._jobs.get(video_id)
330 if job is not None and job.task is task:
331 self._jobs.pop(video_id)
332 if not task.cancelled():
333 # A download may finish after its last caller was cancelled.
334 task.exception()
336 async def _run(
337 self, video_id: str, cookies_file: str, max_seconds: Optional[int]
338 ) -> Path:
339 self._dir.mkdir(parents=True, exist_ok=True)
340 async with self._semaphore:
341 cached = self.streamable_path(video_id, max_seconds)
342 if cached is not None:
343 return cached
344 # Cancelling to_thread cannot stop the blocking yt-dlp worker.
345 self._jobs[video_id].fetch_started = True
346 self.running += 1
347 try:
348 await asyncio.to_thread(self._fetch, video_id, cookies_file)
349 finally:
350 self.running -= 1
351 found = self.streamable_path(video_id, max_seconds)
352 if found is None:
353 raise ValueError(
354 'YouTube gave nothing to stream for this track'
355 )
356 await asyncio.to_thread(self.prune)
357 return found
359 def _fetch(self, video_id: str, cookies_file: str) -> None:
360 """Download *video_id*'s audio into the cache dir (blocking)."""
362 opts: dict[str, Any] = {
363 'format': 'bestaudio[ext=m4a]/bestaudio/best',
364 'outtmpl': str(self._dir / f'{video_id}.%(ext)s'),
365 'quiet': True,
366 'noprogress': True,
367 'logger': _YtdlpLogger(),
368 'noplaylist': True,
369 'nocheckcertificate': True,
370 'socket_timeout': _TIMEOUT,
371 'retries': 10,
372 'extractor_retries': 3,
373 'extractor_args': {
374 'youtube': {'player_client': _yt_player_clients()}
375 },
376 'remote_components': ['ejs:github'],
377 }
378 if os.getenv('DOWNTIFY_FORCE_IPV4', '').strip() in {
379 '1',
380 'true',
381 'yes',
382 }:
383 opts['source_address'] = '0.0.0.0'
384 resolved = str(cookies_file or '').strip()
385 if resolved:
386 opts['cookiefile'] = resolved
387 cookies_browser = os.getenv(
388 'DOWNTIFY_COOKIES_FROM_BROWSER', ''
389 ).strip()
390 if cookies_browser:
391 parts = cookies_browser.split(':', 1)
392 opts['cookiesfrombrowser'] = (
393 (parts[0],) if len(parts) == 1 else (parts[0], parts[1])
394 )
395 url = f'https://www.youtube.com/watch?v={video_id}'
396 try:
397 with private_cookiefile(opts), yt_dlp.YoutubeDL(opts) as ydl:
398 ydl.download([url])
399 except Exception as exc:
400 if is_age_restricted_error(str(exc)):
401 raise ValueError(
402 'This track is age-restricted on YouTube: upload a '
403 'YouTube cookies.txt in Settings > YouTube cookies '
404 'to stream it.'
405 ) from exc
406 raise ValueError(
407 f'YouTube did not answer: {type(exc).__name__}'
408 ) from exc
409 logger.info('Stream cached for videoId={}', video_id)
411 def prune(self) -> int:
412 """Delete the least recently used files past the size limit."""
414 try:
415 files = [
416 p
417 for p in self._dir.iterdir()
418 if p.is_file() and not p.name.endswith('.part')
419 ]
420 except OSError:
421 return 0
422 stats = []
423 for path in files:
424 try:
425 st = path.stat()
426 except OSError:
427 continue
428 if st.st_size <= 0:
429 with contextlib.suppress(OSError):
430 path.unlink()
431 continue
432 stats.append((st.st_atime_ns, st.st_size, path))
433 total = sum(size for _, size, _ in stats)
434 removed = 0
435 for _atime, size, path in sorted(stats):
436 if total <= self._max_bytes:
437 break
438 with contextlib.suppress(OSError):
439 path.unlink()
440 total -= size
441 removed += 1
442 return removed
445def stream_cache_from_env(data_dir: Any) -> StreamCache:
446 """A :class:`StreamCache` under ``<data_dir>/stream_cache``.
448 Limits come from ``DOWNTIFY_STREAM_CACHE_MB`` (default 1024) and
449 ``DOWNTIFY_STREAM_CONCURRENCY`` (default 2).
450 """
452 def _int(name: str, default: int) -> int:
453 try:
454 return max(0, int(os.getenv(name, '') or default))
455 except (TypeError, ValueError):
456 return default
458 base = Path(data_dir) if not isinstance(data_dir, Path) else data_dir
459 return StreamCache(
460 base / 'stream_cache',
461 max_bytes=_int('DOWNTIFY_STREAM_CACHE_MB', DEFAULT_STREAM_CACHE_MB)
462 * 1024**2,
463 max_concurrent=_int(
464 'DOWNTIFY_STREAM_CONCURRENCY', DEFAULT_STREAM_CONCURRENCY
465 )
466 or DEFAULT_STREAM_CONCURRENCY,
467 )