Coverage for downtify/transcode.py: 91%
157 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Transcoding for streaming over a slow or metered connection.
3A phone on mobile data asks for a smaller copy of a track (e.g. Opus at
4160 kbps) instead of the original FLAC. Two ways to serve that:
6* **Stream ffmpeg's output as it's produced.** First byte in well under a
7 second, but the length isn't known up front and the client can't seek
8 (no HTTP Range) until the whole thing has been produced - and seeking
9 is the first thing a player does when it resumes a track.
10* **Transcode to a cached file, then serve that file.** The request waits
11 for ffmpeg - a few seconds for a song, since ffmpeg runs 50-100x faster
12 than real time - and from then on it's an ordinary file: exact
13 ``Content-Length``, full Range support, and every later play of the same
14 track at the same quality is instant.
16Downtify does the second: seeking must work, and songs are short. The
17cache lives in ``<data>/transcode_cache``, is bounded by size (least
18recently used files go first) and keyed by the source file's path, size
19and modification time plus the format and bitrate, so a re-tagged or
20replaced file is never served stale. At most
21:data:`DEFAULT_MAX_CONCURRENT` ffmpeg processes run at once; two requests
22for the same copy share one run; and when every request waiting for a run
23has gone away (the client disconnected), the run is stopped and its
24partial file deleted.
26A track is never transcoded when the original already fits what was asked
27for - a lossy file at or below the requested bitrate is served as is, the
28same rule the downloader follows when it doesn't re-encode (see
29``docs/features/download-settings.md``).
30"""
32from __future__ import annotations
34import asyncio
35import contextlib
36import hashlib
37import os
38import shutil
39import time
40from dataclasses import dataclass
41from pathlib import Path
42from typing import Any, Optional
44from loguru import logger
46#: Bitrates (kbps) a client may ask for.
47BITRATES = (96, 128, 160, 192, 256, 320)
48#: ffmpeg processes allowed at once, by default.
49DEFAULT_MAX_CONCURRENT = 2
50#: Size the cache is kept under, by default.
51DEFAULT_CACHE_BYTES = 2 * 1024**3
52#: Longest a single transcode may take.
53TRANSCODE_TIMEOUT = 300.0
55#: Codecs that are already lossy: served as is when small enough.
56_LOSSY = frozenset({'mp3', 'aac', 'opus', 'vorbis'})
59@dataclass(frozen=True)
60class Format:
61 name: str
62 codec: str
63 extension: str
64 media_type: str
65 args: tuple[str, ...]
68FORMATS: dict[str, Format] = {
69 'opus': Format(
70 'opus',
71 'opus',
72 '.opus',
73 'audio/ogg',
74 ('-c:a', 'libopus', '-vbr', 'on', '-f', 'ogg'),
75 ),
76 'aac': Format(
77 'aac',
78 'aac',
79 '.m4a',
80 'audio/mp4',
81 ('-c:a', 'aac', '-movflags', '+faststart', '-f', 'mp4'),
82 ),
83 'mp3': Format(
84 'mp3',
85 'mp3',
86 '.mp3',
87 'audio/mpeg',
88 ('-c:a', 'libmp3lame', '-f', 'mp3'),
89 ),
90}
93def normalize_bitrate(value: Any) -> int:
94 """The nearest allowed bitrate at or above *value* (kbps); the
95 highest one for anything above it; ``0`` for something unreadable."""
97 try:
98 wanted = int(value)
99 except (TypeError, ValueError):
100 return 0
101 if wanted <= 0:
102 return 0
103 for bitrate in BITRATES:
104 if bitrate >= wanted:
105 return bitrate
106 return BITRATES[-1]
109def serve_original(codec: str, bitrate_bps: int, bitrate_kbps: int) -> bool:
110 """Whether a file already fits a request for *bitrate_kbps*: it's
111 lossy and no bigger than asked for (a small margin allows for VBR).
112 Lossless and unknown files are always transcoded."""
114 if codec not in _LOSSY or bitrate_bps <= 0:
115 return False
116 return bitrate_bps <= bitrate_kbps * 1000 * 1.1
119def ffmpeg_args(
120 ffmpeg: str, source: Path, target: Path, fmt: Format, bitrate_kbps: int
121) -> list[str]:
122 """The ffmpeg command line for one transcode: audio only (cover art
123 and other streams dropped), tags kept, stereo at most."""
125 return [
126 ffmpeg,
127 '-nostdin',
128 '-hide_banner',
129 '-loglevel',
130 'error',
131 '-y',
132 '-i',
133 str(source),
134 '-map',
135 '0:a:0',
136 '-map_metadata',
137 '0',
138 '-vn',
139 '-ac',
140 '2',
141 *fmt.args,
142 '-b:a',
143 f'{bitrate_kbps}k',
144 str(target),
145 ]
148def cache_key(source: Path, fmt: str, bitrate_kbps: int) -> str:
149 st = source.stat()
150 raw = f'{source.resolve()}\n{st.st_size}\n{st.st_mtime_ns}\n{fmt}\n'
151 raw += str(bitrate_kbps)
152 return hashlib.sha256(raw.encode()).hexdigest()
155class TranscodeError(RuntimeError):
156 pass
159class _Job:
160 """One ffmpeg run, shared by every request waiting for its output."""
162 def __init__(self, task: asyncio.Task[Path]) -> None:
163 self.task = task
164 self.waiters = 0
167class Transcoder:
168 """Cached, bounded transcoding (see the module docstring)."""
170 def __init__(
171 self,
172 cache_dir: Path,
173 *,
174 max_bytes: int = DEFAULT_CACHE_BYTES,
175 max_concurrent: int = DEFAULT_MAX_CONCURRENT,
176 ffmpeg: Optional[str] = None,
177 ) -> None:
178 self._dir = Path(cache_dir)
179 self._max_bytes = max(0, int(max_bytes))
180 self._ffmpeg = ffmpeg if ffmpeg is not None else shutil.which('ffmpeg')
181 self._semaphore = asyncio.Semaphore(max(1, int(max_concurrent)))
182 self._jobs: dict[str, _Job] = {}
183 self.running = 0
185 @property
186 def available(self) -> bool:
187 return bool(self._ffmpeg)
189 def capability(self) -> dict[str, Any]:
190 """What ``GET /api/server/info`` reports under ``transcoding``."""
192 return {
193 'available': self.available,
194 'formats': sorted(FORMATS) if self.available else [],
195 'bitrates': list(BITRATES) if self.available else [],
196 }
198 def cached_path(
199 self, source: Path, fmt: str, bitrate_kbps: int
200 ) -> Optional[Path]:
201 """The finished copy when it's in the cache (and mark it used)."""
203 target = self._target(source, fmt, bitrate_kbps)
204 if not target.is_file():
205 return None
206 # Keep mtime stable: FileResponse uses it to generate the ETag for
207 # conditional range requests. Access time tracks cache recency instead.
208 with contextlib.suppress(OSError):
209 st = target.stat()
210 os.utime(target, ns=(time.time_ns(), st.st_mtime_ns))
211 return target
213 def _target(self, source: Path, fmt: str, bitrate_kbps: int) -> Path:
214 key = cache_key(source, fmt, bitrate_kbps)
215 return self._dir / f'{key}{FORMATS[fmt].extension}'
217 async def get(self, source: Path, fmt: str, bitrate_kbps: int) -> Path:
218 """The transcoded copy of *source*, made now if it isn't cached.
220 Cancelling the call (the client went away) stops the ffmpeg run
221 once nobody else is waiting for it.
222 """
224 if not self.available:
225 raise TranscodeError('ffmpeg is not available')
226 if fmt not in FORMATS or bitrate_kbps not in BITRATES:
227 raise ValueError('Unsupported format or bitrate')
228 cached = self.cached_path(source, fmt, bitrate_kbps)
229 if cached is not None:
230 return cached
231 target = self._target(source, fmt, bitrate_kbps)
232 key = target.name
233 job = self._jobs.get(key)
234 if job is None:
235 job = _Job(
236 asyncio.ensure_future(
237 self._run(source, target, FORMATS[fmt], bitrate_kbps)
238 )
239 )
240 self._jobs[key] = job
241 job.task.add_done_callback(lambda _t: self._jobs.pop(key, None))
242 job.waiters += 1
243 try:
244 return await asyncio.shield(job.task)
245 except asyncio.CancelledError:
246 job.waiters -= 1
247 if job.waiters <= 0 and not job.task.done():
248 job.task.cancel()
249 raise
251 async def _run(
252 self, source: Path, target: Path, fmt: Format, bitrate_kbps: int
253 ) -> Path:
254 self._dir.mkdir(parents=True, exist_ok=True)
255 partial = target.with_name(target.name + '.part')
256 async with self._semaphore:
257 if target.is_file():
258 return target
259 args = ffmpeg_args(
260 str(self._ffmpeg), source, partial, fmt, bitrate_kbps
261 )
262 self.running += 1
263 process = await asyncio.create_subprocess_exec(
264 *args,
265 stdout=asyncio.subprocess.DEVNULL,
266 stderr=asyncio.subprocess.PIPE,
267 )
268 try:
269 _out, err = await asyncio.wait_for(
270 process.communicate(), TRANSCODE_TIMEOUT
271 )
272 except BaseException:
273 with contextlib.suppress(ProcessLookupError):
274 process.kill()
275 with contextlib.suppress(Exception):
276 await process.wait()
277 partial.unlink(missing_ok=True)
278 raise
279 finally:
280 self.running -= 1
281 if process.returncode != 0 or not partial.is_file():
282 partial.unlink(missing_ok=True)
283 message = (err or b'').decode(errors='replace').strip()
284 logger.warning(
285 'Transcode failed for {}: {}', source.name, message[-300:]
286 )
287 raise TranscodeError('Transcoding failed')
288 partial.replace(target)
289 logger.info(
290 'Transcoded {} to {} {} kbps', source.name, fmt.name, bitrate_kbps
291 )
292 await asyncio.to_thread(self.prune)
293 return target
295 def prune(self) -> int:
296 """Delete the least recently used copies until the cache is under
297 its size limit. Returns how many were deleted."""
299 try:
300 files = [
301 p
302 for p in self._dir.iterdir()
303 if p.is_file() and not p.name.endswith('.part')
304 ]
305 except OSError:
306 return 0
307 stats = []
308 for path in files:
309 try:
310 st = path.stat()
311 except OSError:
312 continue
313 stats.append((st.st_atime_ns, st.st_size, path))
314 total = sum(size for _m, size, _p in stats)
315 removed = 0
316 for _atime, size, path in sorted(stats):
317 if total <= self._max_bytes:
318 break
319 with contextlib.suppress(OSError):
320 path.unlink()
321 total -= size
322 removed += 1
323 return removed
326def transcoder_from_env(data_dir: Path) -> Transcoder:
327 """A :class:`Transcoder` with the limits from the environment:
328 ``DOWNTIFY_TRANSCODE_CACHE_MB`` and ``DOWNTIFY_TRANSCODE_CONCURRENCY``."""
330 def number(name: str, default: int) -> int:
331 try:
332 return max(0, int(os.getenv(name, '') or default))
333 except ValueError:
334 logger.warning('{} is not a number; using {}', name, default)
335 return default
337 return Transcoder(
338 Path(data_dir) / 'transcode_cache',
339 max_bytes=number(
340 'DOWNTIFY_TRANSCODE_CACHE_MB', DEFAULT_CACHE_BYTES // 1024**2
341 )
342 * 1024**2,
343 max_concurrent=number(
344 'DOWNTIFY_TRANSCODE_CONCURRENCY', DEFAULT_MAX_CONCURRENT
345 ),
346 )