Coverage for downtify/transcode.py: 91%

157 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Transcoding for streaming over a slow or metered connection. 

2 

3A phone on mobile data asks for a smaller copy of a track (e.g. Opus at 

4160 kbps) instead of the original FLAC. Two ways to serve that: 

5 

6* **Stream ffmpeg's output as it's produced.** First byte in well under a 

7 second, but the length isn't known up front and the client can't seek 

8 (no HTTP Range) until the whole thing has been produced - and seeking 

9 is the first thing a player does when it resumes a track. 

10* **Transcode to a cached file, then serve that file.** The request waits 

11 for ffmpeg - a few seconds for a song, since ffmpeg runs 50-100x faster 

12 than real time - and from then on it's an ordinary file: exact 

13 ``Content-Length``, full Range support, and every later play of the same 

14 track at the same quality is instant. 

15 

16Downtify does the second: seeking must work, and songs are short. The 

17cache lives in ``<data>/transcode_cache``, is bounded by size (least 

18recently used files go first) and keyed by the source file's path, size 

19and modification time plus the format and bitrate, so a re-tagged or 

20replaced file is never served stale. At most 

21:data:`DEFAULT_MAX_CONCURRENT` ffmpeg processes run at once; two requests 

22for the same copy share one run; and when every request waiting for a run 

23has gone away (the client disconnected), the run is stopped and its 

24partial file deleted. 

25 

26A track is never transcoded when the original already fits what was asked 

27for - a lossy file at or below the requested bitrate is served as is, the 

28same rule the downloader follows when it doesn't re-encode (see 

29``docs/features/download-settings.md``). 

30""" 

31 

32from __future__ import annotations 

33 

34import asyncio 

35import contextlib 

36import hashlib 

37import os 

38import shutil 

39import time 

40from dataclasses import dataclass 

41from pathlib import Path 

42from typing import Any, Optional 

43 

44from loguru import logger 

45 

46#: Bitrates (kbps) a client may ask for. 

47BITRATES = (96, 128, 160, 192, 256, 320) 

48#: ffmpeg processes allowed at once, by default. 

49DEFAULT_MAX_CONCURRENT = 2 

50#: Size the cache is kept under, by default. 

51DEFAULT_CACHE_BYTES = 2 * 1024**3 

52#: Longest a single transcode may take. 

53TRANSCODE_TIMEOUT = 300.0 

54 

55#: Codecs that are already lossy: served as is when small enough. 

56_LOSSY = frozenset({'mp3', 'aac', 'opus', 'vorbis'}) 

57 

58 

59@dataclass(frozen=True) 

60class Format: 

61 name: str 

62 codec: str 

63 extension: str 

64 media_type: str 

65 args: tuple[str, ...] 

66 

67 

68FORMATS: dict[str, Format] = { 

69 'opus': Format( 

70 'opus', 

71 'opus', 

72 '.opus', 

73 'audio/ogg', 

74 ('-c:a', 'libopus', '-vbr', 'on', '-f', 'ogg'), 

75 ), 

76 'aac': Format( 

77 'aac', 

78 'aac', 

79 '.m4a', 

80 'audio/mp4', 

81 ('-c:a', 'aac', '-movflags', '+faststart', '-f', 'mp4'), 

82 ), 

83 'mp3': Format( 

84 'mp3', 

85 'mp3', 

86 '.mp3', 

87 'audio/mpeg', 

88 ('-c:a', 'libmp3lame', '-f', 'mp3'), 

89 ), 

90} 

91 

92 

93def normalize_bitrate(value: Any) -> int: 

94 """The nearest allowed bitrate at or above *value* (kbps); the 

95 highest one for anything above it; ``0`` for something unreadable.""" 

96 

97 try: 

98 wanted = int(value) 

99 except (TypeError, ValueError): 

100 return 0 

101 if wanted <= 0: 

102 return 0 

103 for bitrate in BITRATES: 

104 if bitrate >= wanted: 

105 return bitrate 

106 return BITRATES[-1] 

107 

108 

109def serve_original(codec: str, bitrate_bps: int, bitrate_kbps: int) -> bool: 

110 """Whether a file already fits a request for *bitrate_kbps*: it's 

111 lossy and no bigger than asked for (a small margin allows for VBR). 

112 Lossless and unknown files are always transcoded.""" 

113 

114 if codec not in _LOSSY or bitrate_bps <= 0: 

115 return False 

116 return bitrate_bps <= bitrate_kbps * 1000 * 1.1 

117 

118 

119def ffmpeg_args( 

120 ffmpeg: str, source: Path, target: Path, fmt: Format, bitrate_kbps: int 

121) -> list[str]: 

122 """The ffmpeg command line for one transcode: audio only (cover art 

123 and other streams dropped), tags kept, stereo at most.""" 

124 

125 return [ 

126 ffmpeg, 

127 '-nostdin', 

128 '-hide_banner', 

129 '-loglevel', 

130 'error', 

131 '-y', 

132 '-i', 

133 str(source), 

134 '-map', 

135 '0:a:0', 

136 '-map_metadata', 

137 '0', 

138 '-vn', 

139 '-ac', 

140 '2', 

141 *fmt.args, 

142 '-b:a', 

143 f'{bitrate_kbps}k', 

144 str(target), 

145 ] 

146 

147 

148def cache_key(source: Path, fmt: str, bitrate_kbps: int) -> str: 

149 st = source.stat() 

150 raw = f'{source.resolve()}\n{st.st_size}\n{st.st_mtime_ns}\n{fmt}\n' 

151 raw += str(bitrate_kbps) 

152 return hashlib.sha256(raw.encode()).hexdigest() 

153 

154 

155class TranscodeError(RuntimeError): 

156 pass 

157 

158 

159class _Job: 

160 """One ffmpeg run, shared by every request waiting for its output.""" 

161 

162 def __init__(self, task: asyncio.Task[Path]) -> None: 

163 self.task = task 

164 self.waiters = 0 

165 

166 

167class Transcoder: 

168 """Cached, bounded transcoding (see the module docstring).""" 

169 

170 def __init__( 

171 self, 

172 cache_dir: Path, 

173 *, 

174 max_bytes: int = DEFAULT_CACHE_BYTES, 

175 max_concurrent: int = DEFAULT_MAX_CONCURRENT, 

176 ffmpeg: Optional[str] = None, 

177 ) -> None: 

178 self._dir = Path(cache_dir) 

179 self._max_bytes = max(0, int(max_bytes)) 

180 self._ffmpeg = ffmpeg if ffmpeg is not None else shutil.which('ffmpeg') 

181 self._semaphore = asyncio.Semaphore(max(1, int(max_concurrent))) 

182 self._jobs: dict[str, _Job] = {} 

183 self.running = 0 

184 

185 @property 

186 def available(self) -> bool: 

187 return bool(self._ffmpeg) 

188 

189 def capability(self) -> dict[str, Any]: 

190 """What ``GET /api/server/info`` reports under ``transcoding``.""" 

191 

192 return { 

193 'available': self.available, 

194 'formats': sorted(FORMATS) if self.available else [], 

195 'bitrates': list(BITRATES) if self.available else [], 

196 } 

197 

198 def cached_path( 

199 self, source: Path, fmt: str, bitrate_kbps: int 

200 ) -> Optional[Path]: 

201 """The finished copy when it's in the cache (and mark it used).""" 

202 

203 target = self._target(source, fmt, bitrate_kbps) 

204 if not target.is_file(): 

205 return None 

206 # Keep mtime stable: FileResponse uses it to generate the ETag for 

207 # conditional range requests. Access time tracks cache recency instead. 

208 with contextlib.suppress(OSError): 

209 st = target.stat() 

210 os.utime(target, ns=(time.time_ns(), st.st_mtime_ns)) 

211 return target 

212 

213 def _target(self, source: Path, fmt: str, bitrate_kbps: int) -> Path: 

214 key = cache_key(source, fmt, bitrate_kbps) 

215 return self._dir / f'{key}{FORMATS[fmt].extension}' 

216 

217 async def get(self, source: Path, fmt: str, bitrate_kbps: int) -> Path: 

218 """The transcoded copy of *source*, made now if it isn't cached. 

219 

220 Cancelling the call (the client went away) stops the ffmpeg run 

221 once nobody else is waiting for it. 

222 """ 

223 

224 if not self.available: 

225 raise TranscodeError('ffmpeg is not available') 

226 if fmt not in FORMATS or bitrate_kbps not in BITRATES: 

227 raise ValueError('Unsupported format or bitrate') 

228 cached = self.cached_path(source, fmt, bitrate_kbps) 

229 if cached is not None: 

230 return cached 

231 target = self._target(source, fmt, bitrate_kbps) 

232 key = target.name 

233 job = self._jobs.get(key) 

234 if job is None: 

235 job = _Job( 

236 asyncio.ensure_future( 

237 self._run(source, target, FORMATS[fmt], bitrate_kbps) 

238 ) 

239 ) 

240 self._jobs[key] = job 

241 job.task.add_done_callback(lambda _t: self._jobs.pop(key, None)) 

242 job.waiters += 1 

243 try: 

244 return await asyncio.shield(job.task) 

245 except asyncio.CancelledError: 

246 job.waiters -= 1 

247 if job.waiters <= 0 and not job.task.done(): 

248 job.task.cancel() 

249 raise 

250 

251 async def _run( 

252 self, source: Path, target: Path, fmt: Format, bitrate_kbps: int 

253 ) -> Path: 

254 self._dir.mkdir(parents=True, exist_ok=True) 

255 partial = target.with_name(target.name + '.part') 

256 async with self._semaphore: 

257 if target.is_file(): 

258 return target 

259 args = ffmpeg_args( 

260 str(self._ffmpeg), source, partial, fmt, bitrate_kbps 

261 ) 

262 self.running += 1 

263 process = await asyncio.create_subprocess_exec( 

264 *args, 

265 stdout=asyncio.subprocess.DEVNULL, 

266 stderr=asyncio.subprocess.PIPE, 

267 ) 

268 try: 

269 _out, err = await asyncio.wait_for( 

270 process.communicate(), TRANSCODE_TIMEOUT 

271 ) 

272 except BaseException: 

273 with contextlib.suppress(ProcessLookupError): 

274 process.kill() 

275 with contextlib.suppress(Exception): 

276 await process.wait() 

277 partial.unlink(missing_ok=True) 

278 raise 

279 finally: 

280 self.running -= 1 

281 if process.returncode != 0 or not partial.is_file(): 

282 partial.unlink(missing_ok=True) 

283 message = (err or b'').decode(errors='replace').strip() 

284 logger.warning( 

285 'Transcode failed for {}: {}', source.name, message[-300:] 

286 ) 

287 raise TranscodeError('Transcoding failed') 

288 partial.replace(target) 

289 logger.info( 

290 'Transcoded {} to {} {} kbps', source.name, fmt.name, bitrate_kbps 

291 ) 

292 await asyncio.to_thread(self.prune) 

293 return target 

294 

295 def prune(self) -> int: 

296 """Delete the least recently used copies until the cache is under 

297 its size limit. Returns how many were deleted.""" 

298 

299 try: 

300 files = [ 

301 p 

302 for p in self._dir.iterdir() 

303 if p.is_file() and not p.name.endswith('.part') 

304 ] 

305 except OSError: 

306 return 0 

307 stats = [] 

308 for path in files: 

309 try: 

310 st = path.stat() 

311 except OSError: 

312 continue 

313 stats.append((st.st_atime_ns, st.st_size, path)) 

314 total = sum(size for _m, size, _p in stats) 

315 removed = 0 

316 for _atime, size, path in sorted(stats): 

317 if total <= self._max_bytes: 

318 break 

319 with contextlib.suppress(OSError): 

320 path.unlink() 

321 total -= size 

322 removed += 1 

323 return removed 

324 

325 

326def transcoder_from_env(data_dir: Path) -> Transcoder: 

327 """A :class:`Transcoder` with the limits from the environment: 

328 ``DOWNTIFY_TRANSCODE_CACHE_MB`` and ``DOWNTIFY_TRANSCODE_CONCURRENCY``.""" 

329 

330 def number(name: str, default: int) -> int: 

331 try: 

332 return max(0, int(os.getenv(name, '') or default)) 

333 except ValueError: 

334 logger.warning('{} is not a number; using {}', name, default) 

335 return default 

336 

337 return Transcoder( 

338 Path(data_dir) / 'transcode_cache', 

339 max_bytes=number( 

340 'DOWNTIFY_TRANSCODE_CACHE_MB', DEFAULT_CACHE_BYTES // 1024**2 

341 ) 

342 * 1024**2, 

343 max_concurrent=number( 

344 'DOWNTIFY_TRANSCODE_CONCURRENCY', DEFAULT_MAX_CONCURRENT 

345 ), 

346 )