Coverage for downtify/stream.py: 79%

207 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Full-track streaming, served from this server. 

2 

3A track the library doesn't have yet still plays in full, in the 

4built-in player, without a library download: the frontend resolves it 

5to a YouTube video id and plays ``GET /api/stream/file``, which the 

6:class:`StreamCache` below downloads once (via yt-dlp, with the same 

7anti-bot posture as a real download) into ``/data/stream_cache`` and 

8then serves with seeking support. This is what makes streaming work 

9where the browser itself can't reach YouTube (or only slowly): the 

10only YouTube traffic is server-side, and the second play of a track 

11never touches YouTube at all. 

12 

13Two endpoints share this module: ``GET /api/stream`` hands out a 

14fresh direct-audio URL for third-party clients, ``GET 

15/api/stream/file`` serves the cached file to the web player. 

16 

17Failures surface as :class:`ValueError` (same convention as 

18:mod:`downtify.deezer`), so the endpoints map them to ``503``. 

19""" 

20 

21from __future__ import annotations 

22 

23import asyncio 

24import contextlib 

25import os 

26import re 

27import time 

28from pathlib import Path 

29from typing import Any, Optional 

30 

31import yt_dlp 

32from loguru import logger 

33from mutagen import File as _mutagen_file 

34 

35from .cookies import private_cookiefile 

36from .downloader import ( 

37 MAX_TRACK_DURATION_SECONDS, 

38 TrackTooLongError, 

39 _yt_player_clients, 

40 _YtdlpLogger, 

41 describe_duration, 

42 is_age_restricted_error, 

43) 

44 

45#: What a YouTube video id looks like — anything else is rejected 

46#: without touching the network. 

47VIDEO_ID_RE = re.compile(r'^[A-Za-z0-9_-]{11}$') 

48 

49_TIMEOUT = 30 

50 

51 

52def clean_video_id(video_id: str) -> str: 

53 """*video_id* when it is shaped like a YouTube video id.""" 

54 

55 candidate = str(video_id or '').strip() 

56 if not VIDEO_ID_RE.fullmatch(candidate): 

57 raise ValueError(f'Not a YouTube video id: {candidate[:32]!r}') 

58 return candidate 

59 

60 

61def _ydl_options() -> dict[str, Any]: 

62 """yt-dlp options for reading (not downloading) one audio URL. 

63 

64 Mirrors the anti-bot posture of a real download (see 

65 ``Downloader._ydl_options``): the TV/mobile player clients that 

66 bypass "Sign in to confirm you're not a bot", the EJS challenge 

67 solver, IPv4 when asked, and the same cookies. 

68 """ 

69 

70 opts: dict[str, Any] = { 

71 'format': 'bestaudio[ext=m4a]/bestaudio/best', 

72 'quiet': True, 

73 'noprogress': True, 

74 'logger': _YtdlpLogger(), 

75 'noplaylist': True, 

76 'nocheckcertificate': True, 

77 'socket_timeout': _TIMEOUT, 

78 'extractor_args': {'youtube': {'player_client': _yt_player_clients()}}, 

79 'remote_components': ['ejs:github'], 

80 } 

81 if os.getenv('DOWNTIFY_FORCE_IPV4', '').strip() in {'1', 'true', 'yes'}: 

82 opts['source_address'] = '0.0.0.0' 

83 return opts 

84 

85 

86def stream_url_for_video( 

87 video_id: str, *, cookies_file: str = '' 

88) -> dict[str, Any]: 

89 """The current direct-audio URL of the YouTube video *video_id*. 

90 

91 Returns ``{url, ext}`` — ``ext`` is ``m4a`` when YouTube served 

92 AAC (plays in every browser) and whatever ``bestaudio`` was 

93 otherwise. Raises :class:`ValueError` when the id is malformed, 

94 age-gated without usable cookies, or YouTube can't be reached. 

95 The URL itself is never logged: it carries a signature. 

96 """ 

97 

98 vid = clean_video_id(video_id) 

99 opts = _ydl_options() 

100 resolved = str(cookies_file or '').strip() 

101 if resolved: 

102 opts['cookiefile'] = resolved 

103 cookies_browser = os.getenv('DOWNTIFY_COOKIES_FROM_BROWSER', '').strip() 

104 if cookies_browser: 

105 parts = cookies_browser.split(':', 1) 

106 opts['cookiesfrombrowser'] = ( 

107 (parts[0],) if len(parts) == 1 else (parts[0], parts[1]) 

108 ) 

109 url = f'https://www.youtube.com/watch?v={vid}' 

110 try: 

111 with private_cookiefile(opts), yt_dlp.YoutubeDL(opts) as ydl: 

112 info = ydl.extract_info(url, download=False) 

113 except Exception as exc: 

114 if is_age_restricted_error(str(exc)): 

115 raise ValueError( 

116 'This track is age-restricted on YouTube: upload a ' 

117 'YouTube cookies.txt in Settings > YouTube cookies ' 

118 'to stream it.' 

119 ) from exc 

120 raise ValueError( 

121 f'YouTube did not answer: {type(exc).__name__}' 

122 ) from exc 

123 if not isinstance(info, dict) or not info.get('url'): 

124 raise ValueError('YouTube answered with an unreadable reply') 

125 direct = str(info['url']) 

126 logger.info( 

127 'Stream URL resolved for videoId={} ext={!r}', vid, info.get('ext') 

128 ) 

129 return {'url': direct, 'ext': str(info.get('ext') or '')} 

130 

131 

132def resolve_cookies_file(store: Optional[Any]) -> str: 

133 """The cookies.txt streaming should use (UI upload wins, env next).""" 

134 

135 try: 

136 active = store.active_path() if store is not None else None 

137 except Exception: 

138 active = None 

139 if active: 

140 return str(active) 

141 return os.getenv('DOWNTIFY_COOKIES_FILE', '').strip() 

142 

143 

144#: Default cap of the on-disk stream cache (see ``stream_cache_from_env``). 

145DEFAULT_STREAM_CACHE_MB = 1024 

146#: How many stream downloads may run at once (they are heavier than 

147#: metadata lookups but lighter than full tagged downloads). 

148DEFAULT_STREAM_CONCURRENCY = 2 

149 

150_MEDIA_TYPES = { 

151 'm4a': 'audio/mp4', 

152 'mp4': 'audio/mp4', 

153 'aac': 'audio/aac', 

154 'mp3': 'audio/mpeg', 

155 'webm': 'audio/webm', 

156 'opus': 'audio/ogg', 

157 'ogg': 'audio/ogg', 

158 'flac': 'audio/flac', 

159 'wav': 'audio/wav', 

160} 

161 

162 

163def media_type_for(path: Any) -> str: 

164 """The ``Content-Type`` a cached stream file is served with.""" 

165 

166 ext = str(getattr(path, 'suffix', '') or '').lower().lstrip('.') 

167 return _MEDIA_TYPES.get(ext, 'application/octet-stream') 

168 

169 

170#: Detail for the ``400`` when a stream is refused for length (mirrors 

171#: the download message in ``downtify.downloader``; no name here, only 

172#: a video id reached this layer). 

173STREAM_TOO_LONG_DETAIL = ( 

174 'This track runs over 10 minutes; tracks over 10 minutes are not streamed' 

175) 

176 

177 

178def too_long_detail(max_seconds: int) -> str: 

179 """The refusal for the configured bound (``0`` is never refused).""" 

180 

181 bound = describe_duration(max_seconds) 

182 return ( 

183 f'This track runs over {bound}; tracks over {bound} are not streamed' 

184 ) 

185 

186 

187def _probed_seconds(path: Path) -> float: 

188 """The audio length of a cached file in seconds, ``0`` when unknown. 

189 

190 Read off the downloaded file itself (no extra YouTube request), so 

191 the duration cap holds no matter which client asked. Fail-open: an 

192 unreadable file is served rather than breaking playback. 

193 """ 

194 

195 try: 

196 audio = _mutagen_file(path) 

197 if audio is not None and audio.info is not None: 

198 return float(getattr(audio.info, 'length', 0) or 0) 

199 except Exception: 

200 logger.opt(exception=True).debug('Stream length probe failed') 

201 return 0.0 

202 

203 

204class _StreamJob: 

205 """One cache download, shared by every request waiting for it.""" 

206 

207 def __init__(self, task: asyncio.Task[Path]) -> None: 

208 self.task = task 

209 self.waiters = 0 

210 self.fetch_started = False 

211 

212 

213class StreamCache: 

214 """On-disk cache of streamed audio (see the module docstring). 

215 

216 ``get`` downloads a video's audio once no matter how many requests 

217 arrive together, serves the finished file afterwards, and prunes 

218 the least recently used files past the size limit. Files are named 

219 ``<video_id>.<ext>`` with ``.part`` left to yt-dlp while a download 

220 is in flight (never served: only finished files match). 

221 """ 

222 

223 def __init__( 

224 self, 

225 cache_dir: Any, 

226 *, 

227 max_bytes: int = DEFAULT_STREAM_CACHE_MB * 1024**2, 

228 max_concurrent: int = DEFAULT_STREAM_CONCURRENCY, 

229 ) -> None: 

230 self._dir = ( 

231 Path(cache_dir) if not isinstance(cache_dir, Path) else cache_dir 

232 ) 

233 self._max_bytes = max(0, int(max_bytes)) 

234 self._semaphore = asyncio.Semaphore(max(1, int(max_concurrent))) 

235 self._jobs: dict[str, _StreamJob] = {} 

236 self.running = 0 

237 

238 def cached_path(self, video_id: str) -> Optional[Path]: 

239 """The finished cached file for *video_id*, if one is stored.""" 

240 

241 try: 

242 vid = clean_video_id(video_id) 

243 except ValueError: 

244 return None 

245 for path in sorted(self._dir.glob(f'{vid}.*')): 

246 if path.name.endswith('.part') or not path.is_file(): 

247 continue 

248 try: 

249 stat = path.stat() 

250 if stat.st_size <= 0: 

251 continue 

252 with contextlib.suppress(OSError): 

253 # Track cache use without changing the HTTP file validator. 

254 os.utime(path, ns=(time.time_ns(), stat.st_mtime_ns)) 

255 except OSError: 

256 continue 

257 return path 

258 return None 

259 

260 def streamable_path( 

261 self, video_id: str, max_seconds: Optional[int] = None 

262 ) -> Optional[Path]: 

263 """The finished cached file for *video_id*, enforcing the cap. 

264 

265 A cached file past the bound is deleted and raises 

266 :class:`TrackTooLongError` instead of being served, so long 

267 tracks never accumulate on disk. ``None`` keeps the default 

268 bound, ``0`` serves everything. 

269 """ 

270 

271 limit = ( 

272 MAX_TRACK_DURATION_SECONDS if max_seconds is None else max_seconds 

273 ) 

274 path = self.cached_path(video_id) 

275 if path is None: 

276 return None 

277 if limit and _probed_seconds(path) > limit: 

278 with contextlib.suppress(OSError): 

279 path.unlink() 

280 logger.warning( 

281 'Stream cache refused an overlong file for videoId={}', 

282 video_id, 

283 ) 

284 raise TrackTooLongError(too_long_detail(limit)) 

285 return path 

286 

287 async def get( 

288 self, 

289 video_id: str, 

290 *, 

291 cookies_file: str = '', 

292 max_seconds: Optional[int] = None, 

293 ) -> Path: 

294 """The cached audio file for *video_id*, downloading it first. 

295 

296 Concurrent callers share one download. A queued download is 

297 cancelled once nobody waits for it; a running download finishes 

298 into the cache, keeping its concurrency slot and remaining 

299 available to new callers. Raises :class:`ValueError` when there 

300 is nothing to serve. 

301 """ 

302 

303 vid = clean_video_id(video_id) 

304 cached = self.streamable_path(vid, max_seconds) 

305 if cached is not None: 

306 return cached 

307 job = self._jobs.get(vid) 

308 if job is None or job.task.cancelling(): 

309 job = _StreamJob( 

310 asyncio.ensure_future( 

311 self._run(vid, cookies_file, max_seconds) 

312 ) 

313 ) 

314 self._jobs[vid] = job 

315 job.task.add_done_callback(lambda task: self._job_done(vid, task)) 

316 job.waiters += 1 

317 try: 

318 return await asyncio.shield(job.task) 

319 finally: 

320 job.waiters -= 1 

321 if ( 

322 job.waiters <= 0 

323 and not job.fetch_started 

324 and not job.task.done() 

325 ): 

326 job.task.cancel() 

327 

328 def _job_done(self, video_id: str, task: asyncio.Task[Path]) -> None: 

329 job = self._jobs.get(video_id) 

330 if job is not None and job.task is task: 

331 self._jobs.pop(video_id) 

332 if not task.cancelled(): 

333 # A download may finish after its last caller was cancelled. 

334 task.exception() 

335 

336 async def _run( 

337 self, video_id: str, cookies_file: str, max_seconds: Optional[int] 

338 ) -> Path: 

339 self._dir.mkdir(parents=True, exist_ok=True) 

340 async with self._semaphore: 

341 cached = self.streamable_path(video_id, max_seconds) 

342 if cached is not None: 

343 return cached 

344 # Cancelling to_thread cannot stop the blocking yt-dlp worker. 

345 self._jobs[video_id].fetch_started = True 

346 self.running += 1 

347 try: 

348 await asyncio.to_thread(self._fetch, video_id, cookies_file) 

349 finally: 

350 self.running -= 1 

351 found = self.streamable_path(video_id, max_seconds) 

352 if found is None: 

353 raise ValueError( 

354 'YouTube gave nothing to stream for this track' 

355 ) 

356 await asyncio.to_thread(self.prune) 

357 return found 

358 

359 def _fetch(self, video_id: str, cookies_file: str) -> None: 

360 """Download *video_id*'s audio into the cache dir (blocking).""" 

361 

362 opts: dict[str, Any] = { 

363 'format': 'bestaudio[ext=m4a]/bestaudio/best', 

364 'outtmpl': str(self._dir / f'{video_id}.%(ext)s'), 

365 'quiet': True, 

366 'noprogress': True, 

367 'logger': _YtdlpLogger(), 

368 'noplaylist': True, 

369 'nocheckcertificate': True, 

370 'socket_timeout': _TIMEOUT, 

371 'retries': 10, 

372 'extractor_retries': 3, 

373 'extractor_args': { 

374 'youtube': {'player_client': _yt_player_clients()} 

375 }, 

376 'remote_components': ['ejs:github'], 

377 } 

378 if os.getenv('DOWNTIFY_FORCE_IPV4', '').strip() in { 

379 '1', 

380 'true', 

381 'yes', 

382 }: 

383 opts['source_address'] = '0.0.0.0' 

384 resolved = str(cookies_file or '').strip() 

385 if resolved: 

386 opts['cookiefile'] = resolved 

387 cookies_browser = os.getenv( 

388 'DOWNTIFY_COOKIES_FROM_BROWSER', '' 

389 ).strip() 

390 if cookies_browser: 

391 parts = cookies_browser.split(':', 1) 

392 opts['cookiesfrombrowser'] = ( 

393 (parts[0],) if len(parts) == 1 else (parts[0], parts[1]) 

394 ) 

395 url = f'https://www.youtube.com/watch?v={video_id}' 

396 try: 

397 with private_cookiefile(opts), yt_dlp.YoutubeDL(opts) as ydl: 

398 ydl.download([url]) 

399 except Exception as exc: 

400 if is_age_restricted_error(str(exc)): 

401 raise ValueError( 

402 'This track is age-restricted on YouTube: upload a ' 

403 'YouTube cookies.txt in Settings > YouTube cookies ' 

404 'to stream it.' 

405 ) from exc 

406 raise ValueError( 

407 f'YouTube did not answer: {type(exc).__name__}' 

408 ) from exc 

409 logger.info('Stream cached for videoId={}', video_id) 

410 

411 def prune(self) -> int: 

412 """Delete the least recently used files past the size limit.""" 

413 

414 try: 

415 files = [ 

416 p 

417 for p in self._dir.iterdir() 

418 if p.is_file() and not p.name.endswith('.part') 

419 ] 

420 except OSError: 

421 return 0 

422 stats = [] 

423 for path in files: 

424 try: 

425 st = path.stat() 

426 except OSError: 

427 continue 

428 if st.st_size <= 0: 

429 with contextlib.suppress(OSError): 

430 path.unlink() 

431 continue 

432 stats.append((st.st_atime_ns, st.st_size, path)) 

433 total = sum(size for _, size, _ in stats) 

434 removed = 0 

435 for _atime, size, path in sorted(stats): 

436 if total <= self._max_bytes: 

437 break 

438 with contextlib.suppress(OSError): 

439 path.unlink() 

440 total -= size 

441 removed += 1 

442 return removed 

443 

444 

445def stream_cache_from_env(data_dir: Any) -> StreamCache: 

446 """A :class:`StreamCache` under ``<data_dir>/stream_cache``. 

447 

448 Limits come from ``DOWNTIFY_STREAM_CACHE_MB`` (default 1024) and 

449 ``DOWNTIFY_STREAM_CONCURRENCY`` (default 2). 

450 """ 

451 

452 def _int(name: str, default: int) -> int: 

453 try: 

454 return max(0, int(os.getenv(name, '') or default)) 

455 except (TypeError, ValueError): 

456 return default 

457 

458 base = Path(data_dir) if not isinstance(data_dir, Path) else data_dir 

459 return StreamCache( 

460 base / 'stream_cache', 

461 max_bytes=_int('DOWNTIFY_STREAM_CACHE_MB', DEFAULT_STREAM_CACHE_MB) 

462 * 1024**2, 

463 max_concurrent=_int( 

464 'DOWNTIFY_STREAM_CONCURRENCY', DEFAULT_STREAM_CONCURRENCY 

465 ) 

466 or DEFAULT_STREAM_CONCURRENCY, 

467 )