Coverage for downtify/loudness.py: 74%
203 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Read-only loudness measurement and a durable, CPU-bounded analysis queue."""
3from __future__ import annotations
5import hashlib
6import json
7import math
8import os
9import re
10import shutil
11import subprocess
12import threading
13from pathlib import Path
14from typing import Any, Callable
16from .db.config import Dialect
17from .db.core import Database, as_database
18from .db.errors import DatabaseError
19from .library_catalog import LibraryContext, resolve_library_file
21ALGORITHM_VERSION = 1
22TARGET_LUFS = -18.0
23MAX_GAIN_DB = 12.0
24PEAK_CEILING_DBTP = -1.0
25_DEMUXERS = {
26 '.mp3': 'mp3',
27 '.flac': 'flac',
28 '.wav': 'wav',
29 '.ogg': 'ogg',
30 '.opus': 'ogg',
31 '.m4a': 'mov',
32 '.m4b': 'mov',
33 '.mp4': 'mov',
34 '.aac': 'aac',
35 '.aif': 'aiff',
36 '.aiff': 'aiff',
37 '.wma': 'asf',
38 '.mka': 'matroska',
39 '.alac': 'mov',
40}
43def normalization_gain(integrated: float, true_peak: float) -> float:
44 """A constant track gain, capped by both amplification and true peak."""
45 if not math.isfinite(integrated) or not math.isfinite(true_peak):
46 return 0.0
47 if not -120 <= integrated <= 20 or not -160 <= true_peak <= 20:
48 raise ValueError('Invalid loudness measurement')
49 return min(
50 MAX_GAIN_DB, TARGET_LUFS - integrated, PEAK_CEILING_DBTP - true_peak
51 )
54def file_signature(path: Path) -> str:
55 stat = path.stat()
56 identity = (
57 stat.st_size,
58 stat.st_mtime_ns,
59 stat.st_dev,
60 stat.st_ino,
61 ALGORITHM_VERSION,
62 )
63 return hashlib.sha256(repr(identity).encode()).hexdigest()
66def file_etag(stat: os.stat_result) -> str:
67 """The ETag Starlette's ``FileResponse`` sends for a file with ``stat``.
69 The apps download the original through ``FileResponse`` and compare
70 this to the header they got, to know the measurement is of their copy.
71 """
72 etag = hashlib.md5(
73 f'{stat.st_mtime}-{stat.st_size}'.encode(), usedforsecurity=False
74 ).hexdigest()
75 return f'"{etag}"'
78def analysis_workers() -> int:
79 """Respect the container quota and leave a fixed upper bound for library IO."""
80 capacity = float(os.cpu_count() or 1)
81 try:
82 capacity = min(capacity, len(os.sched_getaffinity(0)))
83 except (AttributeError, OSError):
84 pass
85 try:
86 quota, period = (
87 Path('/sys/fs/cgroup/cpu.max').read_text(encoding='utf-8').split()
88 )
89 if quota != 'max':
90 capacity = min(capacity, float(quota) / float(period))
91 except (OSError, ValueError, ZeroDivisionError):
92 try:
93 quota = float(
94 Path('/sys/fs/cgroup/cpu/cpu.cfs_quota_us').read_text(
95 encoding='utf-8'
96 )
97 )
98 period = float(
99 Path('/sys/fs/cgroup/cpu/cpu.cfs_period_us').read_text(
100 encoding='utf-8'
101 )
102 )
103 if quota > 0:
104 capacity = min(capacity, quota / period)
105 except (OSError, ValueError, ZeroDivisionError):
106 pass
107 return max(1, min(2, int(capacity)))
110class LoudnessAnalyzer:
111 """No analysis starts on construction or after a restart."""
113 def __init__(
114 self,
115 db_path: str | Path | Database,
116 context: Callable[[], LibraryContext],
117 *,
118 workers: int | None = None,
119 ) -> None:
120 self.db = as_database(db_path, 'library')
121 if isinstance(db_path, Database):
122 self.path: str | None = str(self.db.path) if self.db.path else None
123 else:
124 self.path = str(db_path)
125 self.context = context
126 self._lock = threading.RLock()
127 self._pause = threading.Event()
128 self._closed = threading.Event()
129 self._thread: threading.Thread | None = None
130 self._processes: set[subprocess.Popen] = set()
131 self._workers = max(
132 1, min(2, workers if workers is not None else analysis_workers())
133 )
134 self._init_db()
136 def _init_db(self) -> None:
137 # On PostgreSQL the library migrations already created these tables.
138 with self.db.connect() as conn:
139 if self.db.dialect is Dialect.SQLITE:
140 conn.execute("""CREATE TABLE IF NOT EXISTS loudness_measurements (
141 file TEXT PRIMARY KEY, signature TEXT NOT NULL, profile TEXT NOT NULL)""")
142 conn.execute("""CREATE TABLE IF NOT EXISTS loudness_jobs (
143 file TEXT PRIMARY KEY, position INTEGER NOT NULL, status TEXT NOT NULL)""")
144 conn.execute(
145 'CREATE INDEX IF NOT EXISTS loudness_jobs_pending ON loudness_jobs(status, position)'
146 )
147 # A job left running by a crash starts over on every backend.
148 conn.execute(
149 "UPDATE loudness_jobs SET status='pending' WHERE status='running'"
150 )
152 def info(self, file: str, full_path: Path) -> dict[str, Any] | None:
153 try:
154 signature = file_signature(full_path)
155 except OSError:
156 return None
157 with self.db.connect() as conn:
158 row = conn.execute(
159 'SELECT profile FROM loudness_measurements WHERE file=? AND signature=?',
160 (file, signature),
161 ).fetchone()
162 return json.loads(row[0]) if row else None
164 def status(self) -> dict[str, Any]:
165 with self.db.connect() as conn:
166 counts = dict(
167 conn.execute(
168 'SELECT status, COUNT(*) FROM loudness_jobs GROUP BY status'
169 )
170 )
171 with self._lock:
172 active = self._thread is not None and self._thread.is_alive()
173 return {
174 'active': active,
175 'pause_requested': self._pause.is_set(),
176 'paused': not active and bool(counts.get('pending', 0)),
177 'total': sum(counts.values()),
178 'done': counts.get('done', 0),
179 'failed': counts.get('failed', 0),
180 'pending': counts.get('pending', 0),
181 'running': counts.get('running', 0),
182 'workers': self._workers,
183 }
185 def start(self, files: list[str]) -> None:
186 with self._lock:
187 self._check_idle()
188 with self.db.connect() as conn:
189 conn.execute('DELETE FROM loudness_jobs')
190 conn.executemany(
191 'INSERT INTO loudness_jobs (file, position, status) '
192 "VALUES (?, ?, 'pending') ON CONFLICT DO NOTHING",
193 [(file, index) for index, file in enumerate(files)],
194 )
195 self.resume()
197 def _check_idle(self) -> None:
198 if self._closed.is_set():
199 raise ValueError('Analyzer is closed')
200 if self._thread is not None and self._thread.is_alive():
201 raise ValueError('An analysis is already running')
203 def resume(self) -> None:
204 with self._lock:
205 self._check_idle()
206 self._pause.clear()
207 self._thread = threading.Thread(
208 target=self._run, name='loudness-analysis', daemon=True
209 )
210 self._thread.start()
212 def pause(self) -> None:
213 # Finish the current file, preserving the rest of the queue.
214 with self._lock:
215 self._pause.set()
217 def close(self) -> None:
218 self._closed.set()
219 self._pause.set()
220 with self._lock:
221 thread = self._thread
222 for process in self._processes:
223 if process.poll() is None:
224 process.terminate()
225 if thread is not None:
226 thread.join(timeout=5)
228 def _measure(self, path: Path) -> dict[str, Any]:
229 executable = shutil.which('ffmpeg')
230 demuxer = _DEMUXERS.get(path.suffix.lower())
231 if not executable or not demuxer:
232 raise ValueError('FFmpeg or the audio format is unavailable')
233 command = [
234 executable,
235 '-hide_banner',
236 '-nostdin',
237 '-nostats',
238 '-protocol_whitelist',
239 'file,pipe',
240 '-threads',
241 '1',
242 '-f',
243 demuxer,
244 '-i',
245 str(path),
246 '-map',
247 '0:a:0',
248 '-vn',
249 '-filter_threads',
250 '1',
251 '-af',
252 # Keep the existing true-peak sampling rate, without normalizing
253 # an unused output. Padding only this discarded analysis stream
254 # flushes the final partial block, including its last sample peak.
255 'aresample=192000,apad=pad_dur=0.1,'
256 'ebur128=peak=sample:metadata=1,'
257 'ametadata=print:key=lavfi.r128.I,'
258 'ametadata=print:key=lavfi.r128.sample_peak',
259 '-f',
260 'null',
261 '-',
262 ]
263 with self._lock:
264 if self._closed.is_set():
265 raise ValueError('Analyzer is closed')
266 process = subprocess.Popen(
267 command,
268 stdin=subprocess.DEVNULL,
269 stdout=subprocess.DEVNULL,
270 stderr=subprocess.PIPE,
271 text=True,
272 encoding='utf-8',
273 errors='replace',
274 )
275 self._processes.add(process)
276 if hasattr(os, 'setpriority'):
277 try:
278 os.setpriority(os.PRIO_PROCESS, process.pid, 10)
279 except OSError:
280 pass
281 try:
282 _, stderr = process.communicate(timeout=1800)
283 if process.returncode:
284 raise ValueError('Audio could not be analyzed')
285 levels = re.findall(r'lavfi\.r128\.I=([^\r\n]+)', stderr)
286 peaks = re.findall(r'lavfi\.r128\.sample_peak=([^\r\n]+)', stderr)
287 summary = stderr.rsplit('Summary:', 1)[-1]
288 final_peaks = re.findall(r'Peak:\s+([-+\w.]+)\s+dBFS', summary)
289 if not levels or not peaks or not final_peaks:
290 raise ValueError('Audio measurement unavailable')
291 integrated = float(levels[-1])
292 amplitude = max(float(value) for value in peaks)
293 peak = max(
294 20 * math.log10(amplitude) if amplitude > 0 else float('-inf'),
295 # The summary rounds to 0.1 dB: take its conservative bound.
296 max(float(value) for value in final_peaks) + 0.05,
297 )
298 if integrated <= -70:
299 integrated = float('-inf')
300 if not math.isfinite(integrated) or not math.isfinite(peak):
301 return {
302 'state': 'silent',
303 'gain_db': 0.0,
304 'algorithm_version': ALGORITHM_VERSION,
305 }
306 return {
307 'state': 'ready',
308 'gain_db': normalization_gain(integrated, peak),
309 'integrated_lufs': integrated,
310 'true_peak_dbtp': peak,
311 'algorithm_version': ALGORITHM_VERSION,
312 }
313 except subprocess.TimeoutExpired:
314 process.kill()
315 process.communicate()
316 raise ValueError('Analysis timed out') from None
317 finally:
318 with self._lock:
319 self._processes.discard(process)
321 def _run(self) -> None:
322 if self._workers == 1:
323 self._drain()
324 return
325 threads = [
326 threading.Thread(
327 target=self._drain,
328 name=f'loudness-worker-{index}',
329 daemon=True,
330 )
331 for index in range(self._workers)
332 ]
333 for thread in threads:
334 thread.start()
335 for thread in threads:
336 thread.join()
338 def _drain(self) -> None:
339 while not self._pause.is_set() and not self._closed.is_set():
340 with self._lock:
341 if self._pause.is_set() or self._closed.is_set():
342 return
343 with self.db.connect() as conn:
344 row = conn.execute(
345 "SELECT file FROM loudness_jobs WHERE status='pending' ORDER BY position LIMIT 1"
346 ).fetchone()
347 if row is None:
348 return
349 file = row[0]
350 conn.execute(
351 "UPDATE loudness_jobs SET status='running' WHERE file=?",
352 (file,),
353 )
354 result = 'failed'
355 try:
356 full = resolve_library_file(file, self.context())
357 if full is None:
358 raise ValueError('File unavailable')
359 before = file_signature(full)
360 profile = self.info(file, full)
361 if profile is None:
362 profile = self._measure(full)
363 if self._closed.is_set():
364 raise ValueError('Analyzer is closed')
365 if file_signature(full) != before:
366 raise ValueError('File changed during analysis')
367 stat = full.stat()
368 profile['fingerprint'] = before
369 profile['source_size'] = stat.st_size
370 profile['source_etag'] = file_etag(stat)
371 with self.db.connect() as conn:
372 conn.execute(
373 'INSERT INTO loudness_measurements VALUES (?, ?, ?) '
374 'ON CONFLICT(file) DO UPDATE SET signature=excluded.signature, profile=excluded.profile',
375 (
376 file,
377 before,
378 json.dumps(profile, allow_nan=False),
379 ),
380 )
381 result = 'done'
382 except (OSError, ValueError, KeyError, IndexError, DatabaseError):
383 # Do not export FFmpeg stderr: tags and filenames can be private.
384 if self._closed.is_set():
385 result = 'pending'
386 finally:
387 with self.db.connect() as conn:
388 conn.execute(
389 'UPDATE loudness_jobs SET status=? WHERE file=?',
390 (result, file),
391 )