Coverage for downtify/loudness.py: 74%

203 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Read-only loudness measurement and a durable, CPU-bounded analysis queue.""" 

2 

3from __future__ import annotations 

4 

5import hashlib 

6import json 

7import math 

8import os 

9import re 

10import shutil 

11import subprocess 

12import threading 

13from pathlib import Path 

14from typing import Any, Callable 

15 

16from .db.config import Dialect 

17from .db.core import Database, as_database 

18from .db.errors import DatabaseError 

19from .library_catalog import LibraryContext, resolve_library_file 

20 

21ALGORITHM_VERSION = 1 

22TARGET_LUFS = -18.0 

23MAX_GAIN_DB = 12.0 

24PEAK_CEILING_DBTP = -1.0 

25_DEMUXERS = { 

26 '.mp3': 'mp3', 

27 '.flac': 'flac', 

28 '.wav': 'wav', 

29 '.ogg': 'ogg', 

30 '.opus': 'ogg', 

31 '.m4a': 'mov', 

32 '.m4b': 'mov', 

33 '.mp4': 'mov', 

34 '.aac': 'aac', 

35 '.aif': 'aiff', 

36 '.aiff': 'aiff', 

37 '.wma': 'asf', 

38 '.mka': 'matroska', 

39 '.alac': 'mov', 

40} 

41 

42 

43def normalization_gain(integrated: float, true_peak: float) -> float: 

44 """A constant track gain, capped by both amplification and true peak.""" 

45 if not math.isfinite(integrated) or not math.isfinite(true_peak): 

46 return 0.0 

47 if not -120 <= integrated <= 20 or not -160 <= true_peak <= 20: 

48 raise ValueError('Invalid loudness measurement') 

49 return min( 

50 MAX_GAIN_DB, TARGET_LUFS - integrated, PEAK_CEILING_DBTP - true_peak 

51 ) 

52 

53 

54def file_signature(path: Path) -> str: 

55 stat = path.stat() 

56 identity = ( 

57 stat.st_size, 

58 stat.st_mtime_ns, 

59 stat.st_dev, 

60 stat.st_ino, 

61 ALGORITHM_VERSION, 

62 ) 

63 return hashlib.sha256(repr(identity).encode()).hexdigest() 

64 

65 

66def file_etag(stat: os.stat_result) -> str: 

67 """The ETag Starlette's ``FileResponse`` sends for a file with ``stat``. 

68 

69 The apps download the original through ``FileResponse`` and compare 

70 this to the header they got, to know the measurement is of their copy. 

71 """ 

72 etag = hashlib.md5( 

73 f'{stat.st_mtime}-{stat.st_size}'.encode(), usedforsecurity=False 

74 ).hexdigest() 

75 return f'"{etag}"' 

76 

77 

78def analysis_workers() -> int: 

79 """Respect the container quota and leave a fixed upper bound for library IO.""" 

80 capacity = float(os.cpu_count() or 1) 

81 try: 

82 capacity = min(capacity, len(os.sched_getaffinity(0))) 

83 except (AttributeError, OSError): 

84 pass 

85 try: 

86 quota, period = ( 

87 Path('/sys/fs/cgroup/cpu.max').read_text(encoding='utf-8').split() 

88 ) 

89 if quota != 'max': 

90 capacity = min(capacity, float(quota) / float(period)) 

91 except (OSError, ValueError, ZeroDivisionError): 

92 try: 

93 quota = float( 

94 Path('/sys/fs/cgroup/cpu/cpu.cfs_quota_us').read_text( 

95 encoding='utf-8' 

96 ) 

97 ) 

98 period = float( 

99 Path('/sys/fs/cgroup/cpu/cpu.cfs_period_us').read_text( 

100 encoding='utf-8' 

101 ) 

102 ) 

103 if quota > 0: 

104 capacity = min(capacity, quota / period) 

105 except (OSError, ValueError, ZeroDivisionError): 

106 pass 

107 return max(1, min(2, int(capacity))) 

108 

109 

110class LoudnessAnalyzer: 

111 """No analysis starts on construction or after a restart.""" 

112 

113 def __init__( 

114 self, 

115 db_path: str | Path | Database, 

116 context: Callable[[], LibraryContext], 

117 *, 

118 workers: int | None = None, 

119 ) -> None: 

120 self.db = as_database(db_path, 'library') 

121 if isinstance(db_path, Database): 

122 self.path: str | None = str(self.db.path) if self.db.path else None 

123 else: 

124 self.path = str(db_path) 

125 self.context = context 

126 self._lock = threading.RLock() 

127 self._pause = threading.Event() 

128 self._closed = threading.Event() 

129 self._thread: threading.Thread | None = None 

130 self._processes: set[subprocess.Popen] = set() 

131 self._workers = max( 

132 1, min(2, workers if workers is not None else analysis_workers()) 

133 ) 

134 self._init_db() 

135 

136 def _init_db(self) -> None: 

137 # On PostgreSQL the library migrations already created these tables. 

138 with self.db.connect() as conn: 

139 if self.db.dialect is Dialect.SQLITE: 

140 conn.execute("""CREATE TABLE IF NOT EXISTS loudness_measurements ( 

141 file TEXT PRIMARY KEY, signature TEXT NOT NULL, profile TEXT NOT NULL)""") 

142 conn.execute("""CREATE TABLE IF NOT EXISTS loudness_jobs ( 

143 file TEXT PRIMARY KEY, position INTEGER NOT NULL, status TEXT NOT NULL)""") 

144 conn.execute( 

145 'CREATE INDEX IF NOT EXISTS loudness_jobs_pending ON loudness_jobs(status, position)' 

146 ) 

147 # A job left running by a crash starts over on every backend. 

148 conn.execute( 

149 "UPDATE loudness_jobs SET status='pending' WHERE status='running'" 

150 ) 

151 

152 def info(self, file: str, full_path: Path) -> dict[str, Any] | None: 

153 try: 

154 signature = file_signature(full_path) 

155 except OSError: 

156 return None 

157 with self.db.connect() as conn: 

158 row = conn.execute( 

159 'SELECT profile FROM loudness_measurements WHERE file=? AND signature=?', 

160 (file, signature), 

161 ).fetchone() 

162 return json.loads(row[0]) if row else None 

163 

164 def status(self) -> dict[str, Any]: 

165 with self.db.connect() as conn: 

166 counts = dict( 

167 conn.execute( 

168 'SELECT status, COUNT(*) FROM loudness_jobs GROUP BY status' 

169 ) 

170 ) 

171 with self._lock: 

172 active = self._thread is not None and self._thread.is_alive() 

173 return { 

174 'active': active, 

175 'pause_requested': self._pause.is_set(), 

176 'paused': not active and bool(counts.get('pending', 0)), 

177 'total': sum(counts.values()), 

178 'done': counts.get('done', 0), 

179 'failed': counts.get('failed', 0), 

180 'pending': counts.get('pending', 0), 

181 'running': counts.get('running', 0), 

182 'workers': self._workers, 

183 } 

184 

185 def start(self, files: list[str]) -> None: 

186 with self._lock: 

187 self._check_idle() 

188 with self.db.connect() as conn: 

189 conn.execute('DELETE FROM loudness_jobs') 

190 conn.executemany( 

191 'INSERT INTO loudness_jobs (file, position, status) ' 

192 "VALUES (?, ?, 'pending') ON CONFLICT DO NOTHING", 

193 [(file, index) for index, file in enumerate(files)], 

194 ) 

195 self.resume() 

196 

197 def _check_idle(self) -> None: 

198 if self._closed.is_set(): 

199 raise ValueError('Analyzer is closed') 

200 if self._thread is not None and self._thread.is_alive(): 

201 raise ValueError('An analysis is already running') 

202 

203 def resume(self) -> None: 

204 with self._lock: 

205 self._check_idle() 

206 self._pause.clear() 

207 self._thread = threading.Thread( 

208 target=self._run, name='loudness-analysis', daemon=True 

209 ) 

210 self._thread.start() 

211 

212 def pause(self) -> None: 

213 # Finish the current file, preserving the rest of the queue. 

214 with self._lock: 

215 self._pause.set() 

216 

217 def close(self) -> None: 

218 self._closed.set() 

219 self._pause.set() 

220 with self._lock: 

221 thread = self._thread 

222 for process in self._processes: 

223 if process.poll() is None: 

224 process.terminate() 

225 if thread is not None: 

226 thread.join(timeout=5) 

227 

228 def _measure(self, path: Path) -> dict[str, Any]: 

229 executable = shutil.which('ffmpeg') 

230 demuxer = _DEMUXERS.get(path.suffix.lower()) 

231 if not executable or not demuxer: 

232 raise ValueError('FFmpeg or the audio format is unavailable') 

233 command = [ 

234 executable, 

235 '-hide_banner', 

236 '-nostdin', 

237 '-nostats', 

238 '-protocol_whitelist', 

239 'file,pipe', 

240 '-threads', 

241 '1', 

242 '-f', 

243 demuxer, 

244 '-i', 

245 str(path), 

246 '-map', 

247 '0:a:0', 

248 '-vn', 

249 '-filter_threads', 

250 '1', 

251 '-af', 

252 # Keep the existing true-peak sampling rate, without normalizing 

253 # an unused output. Padding only this discarded analysis stream 

254 # flushes the final partial block, including its last sample peak. 

255 'aresample=192000,apad=pad_dur=0.1,' 

256 'ebur128=peak=sample:metadata=1,' 

257 'ametadata=print:key=lavfi.r128.I,' 

258 'ametadata=print:key=lavfi.r128.sample_peak', 

259 '-f', 

260 'null', 

261 '-', 

262 ] 

263 with self._lock: 

264 if self._closed.is_set(): 

265 raise ValueError('Analyzer is closed') 

266 process = subprocess.Popen( 

267 command, 

268 stdin=subprocess.DEVNULL, 

269 stdout=subprocess.DEVNULL, 

270 stderr=subprocess.PIPE, 

271 text=True, 

272 encoding='utf-8', 

273 errors='replace', 

274 ) 

275 self._processes.add(process) 

276 if hasattr(os, 'setpriority'): 

277 try: 

278 os.setpriority(os.PRIO_PROCESS, process.pid, 10) 

279 except OSError: 

280 pass 

281 try: 

282 _, stderr = process.communicate(timeout=1800) 

283 if process.returncode: 

284 raise ValueError('Audio could not be analyzed') 

285 levels = re.findall(r'lavfi\.r128\.I=([^\r\n]+)', stderr) 

286 peaks = re.findall(r'lavfi\.r128\.sample_peak=([^\r\n]+)', stderr) 

287 summary = stderr.rsplit('Summary:', 1)[-1] 

288 final_peaks = re.findall(r'Peak:\s+([-+\w.]+)\s+dBFS', summary) 

289 if not levels or not peaks or not final_peaks: 

290 raise ValueError('Audio measurement unavailable') 

291 integrated = float(levels[-1]) 

292 amplitude = max(float(value) for value in peaks) 

293 peak = max( 

294 20 * math.log10(amplitude) if amplitude > 0 else float('-inf'), 

295 # The summary rounds to 0.1 dB: take its conservative bound. 

296 max(float(value) for value in final_peaks) + 0.05, 

297 ) 

298 if integrated <= -70: 

299 integrated = float('-inf') 

300 if not math.isfinite(integrated) or not math.isfinite(peak): 

301 return { 

302 'state': 'silent', 

303 'gain_db': 0.0, 

304 'algorithm_version': ALGORITHM_VERSION, 

305 } 

306 return { 

307 'state': 'ready', 

308 'gain_db': normalization_gain(integrated, peak), 

309 'integrated_lufs': integrated, 

310 'true_peak_dbtp': peak, 

311 'algorithm_version': ALGORITHM_VERSION, 

312 } 

313 except subprocess.TimeoutExpired: 

314 process.kill() 

315 process.communicate() 

316 raise ValueError('Analysis timed out') from None 

317 finally: 

318 with self._lock: 

319 self._processes.discard(process) 

320 

321 def _run(self) -> None: 

322 if self._workers == 1: 

323 self._drain() 

324 return 

325 threads = [ 

326 threading.Thread( 

327 target=self._drain, 

328 name=f'loudness-worker-{index}', 

329 daemon=True, 

330 ) 

331 for index in range(self._workers) 

332 ] 

333 for thread in threads: 

334 thread.start() 

335 for thread in threads: 

336 thread.join() 

337 

338 def _drain(self) -> None: 

339 while not self._pause.is_set() and not self._closed.is_set(): 

340 with self._lock: 

341 if self._pause.is_set() or self._closed.is_set(): 

342 return 

343 with self.db.connect() as conn: 

344 row = conn.execute( 

345 "SELECT file FROM loudness_jobs WHERE status='pending' ORDER BY position LIMIT 1" 

346 ).fetchone() 

347 if row is None: 

348 return 

349 file = row[0] 

350 conn.execute( 

351 "UPDATE loudness_jobs SET status='running' WHERE file=?", 

352 (file,), 

353 ) 

354 result = 'failed' 

355 try: 

356 full = resolve_library_file(file, self.context()) 

357 if full is None: 

358 raise ValueError('File unavailable') 

359 before = file_signature(full) 

360 profile = self.info(file, full) 

361 if profile is None: 

362 profile = self._measure(full) 

363 if self._closed.is_set(): 

364 raise ValueError('Analyzer is closed') 

365 if file_signature(full) != before: 

366 raise ValueError('File changed during analysis') 

367 stat = full.stat() 

368 profile['fingerprint'] = before 

369 profile['source_size'] = stat.st_size 

370 profile['source_etag'] = file_etag(stat) 

371 with self.db.connect() as conn: 

372 conn.execute( 

373 'INSERT INTO loudness_measurements VALUES (?, ?, ?) ' 

374 'ON CONFLICT(file) DO UPDATE SET signature=excluded.signature, profile=excluded.profile', 

375 ( 

376 file, 

377 before, 

378 json.dumps(profile, allow_nan=False), 

379 ), 

380 ) 

381 result = 'done' 

382 except (OSError, ValueError, KeyError, IndexError, DatabaseError): 

383 # Do not export FFmpeg stderr: tags and filenames can be private. 

384 if self._closed.is_set(): 

385 result = 'pending' 

386 finally: 

387 with self.db.connect() as conn: 

388 conn.execute( 

389 'UPDATE loudness_jobs SET status=? WHERE file=?', 

390 (result, file), 

391 )