Coverage for downtify/download_sources.py: 84%
122 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Where each downloaded file's audio came from, and the sources that
2turned out wrong for a song (issue #582).
4Two things a library file carries no mark of:
6* **Which upload it is.** When a YouTube / YouTube Music / SoundCloud
7 download finishes, the video id (or SoundCloud track URL) is kept here,
8 by library path, with the song's identity (:func:`track_keys`) and a
9 signature of the audio as written (:func:`audio_signature`: codec,
10 sample rate, channels and length - tag rewrites leave it alone).
11* **Which uploads are wrong for a song.** When an admin replaces a
12 track's audio (Replace audio, or by swapping the file on disk), the
13 upload it came from is *rejected* for that song: the matcher never picks
14 it for the song again, from any playlist, artist watch or download
15 (``providers.excluding_sources``). Otherwise the next download of the
16 song, from another playlist, finds the same wrong video.
18A file swapped on disk outside Downtify (a CD rip or a Soulseek download
19saved under the same name) is told apart by its audio signature: when it
20no longer matches what Downtify wrote, :meth:`DownloadSources.replaced_on_disk`
21says so, and the downloader keeps the file as if it had been replaced
22through Replace audio (``downtify/hand_replaced.py``), even with
23**Download again if the file exists** on.
24"""
26from __future__ import annotations
28import json
29import re
30import time
31import unicodedata
32from pathlib import Path
33from typing import Any, Iterable, Optional
35from loguru import logger
36from mutagen import File as MutagenFile
38from .db.config import Dialect
39from .db.core import Database, as_database
41#: Lengths (seconds) closer than this are the same audio: what a tag
42#: rewrite can shift in a VBR estimate, far below any other recording.
43LENGTH_TOLERANCE = 0.5
45_SPACES_RE = re.compile(r'\s+')
48def _key(file: str) -> str:
49 return str(file or '').replace('\\', '/').strip('/')
52def _fold(text: Any) -> str:
53 value = unicodedata.normalize('NFKC', str(text or '')).casefold()
54 return _SPACES_RE.sub(' ', value).strip()
57def track_keys(song: dict[str, Any]) -> list[str]:
58 """The keys a song is known by: its Spotify id, its ISRC, and its
59 lead artist + title. A rejection is kept under every one, so it holds
60 whichever of them the next download of the song carries."""
62 keys: list[str] = []
63 spotify_id = str(song.get('spotify_id') or '').strip()
64 if not spotify_id and song.get('source') in {None, '', 'spotify'}:
65 spotify_id = str(song.get('song_id') or '').strip()
66 if re.fullmatch(r'[0-9A-Za-z]{22}', spotify_id):
67 keys.append(f'spotify:{spotify_id}')
68 isrc = str(song.get('isrc') or '').strip().upper()
69 if isrc:
70 keys.append(f'isrc:{isrc}')
71 artists = song.get('artists') or []
72 if isinstance(artists, str):
73 artists = [artists]
74 artist = _fold(artists[0] if artists else song.get('artist'))
75 title = _fold(song.get('name') or song.get('title'))
76 if artist and title:
77 keys.append(f'name:{artist}|{title}')
78 return keys
81def audio_signature(path: Path) -> Optional[dict[str, Any]]:
82 """What the audio of *path* is, independent of its tags: ``{codec,
83 sample_rate, channels, length}``; ``None`` when it can't be read."""
85 try:
86 audio = MutagenFile(str(path))
87 except Exception:
88 return None
89 info = getattr(audio, 'info', None)
90 if audio is None or info is None:
91 return None
92 return {
93 'codec': type(info).__name__,
94 'sample_rate': int(getattr(info, 'sample_rate', 0) or 0),
95 'channels': int(getattr(info, 'channels', 0) or 0),
96 'length': round(float(getattr(info, 'length', 0) or 0), 3),
97 }
100def same_audio(a: dict[str, Any], b: dict[str, Any]) -> bool:
101 """Whether two :func:`audio_signature` results are the same audio."""
103 return (
104 a.get('codec') == b.get('codec')
105 and a.get('sample_rate') == b.get('sample_rate')
106 and a.get('channels') == b.get('channels')
107 and abs(float(a.get('length') or 0) - float(b.get('length') or 0))
108 < LENGTH_TOLERANCE
109 )
112class DownloadSources:
113 """The source of each downloaded file and the sources rejected per
114 song, in the library database."""
116 def __init__(self, path_or_db: str | Path | Database) -> None:
117 self.db = as_database(path_or_db, 'library')
118 # The Postgres schema is created by migrations; only SQLite needs DDL.
119 if self.db.dialect is Dialect.SQLITE:
120 with self.db.connect() as conn:
121 conn.execute("""
122 CREATE TABLE IF NOT EXISTS file_download_sources (
123 file TEXT PRIMARY KEY,
124 source TEXT NOT NULL DEFAULT '',
125 track_keys TEXT NOT NULL DEFAULT '[]',
126 signature TEXT NOT NULL DEFAULT '',
127 recorded_at REAL NOT NULL
128 )
129 """)
130 conn.execute("""
131 CREATE TABLE IF NOT EXISTS rejected_sources (
132 track_key TEXT NOT NULL,
133 source TEXT NOT NULL,
134 rejected_at REAL NOT NULL,
135 PRIMARY KEY (track_key, source)
136 )
137 """)
139 # -- per file -------------------------------------------------------
141 def record(
142 self,
143 file: str,
144 full_path: Optional[Path],
145 source: str,
146 keys: Iterable[str],
147 ) -> None:
148 """*file* was just written from *source* (a video id, a SoundCloud
149 URL, or ``''`` for an uploaded file) for the song known by *keys*."""
151 signature = audio_signature(full_path) if full_path else None
152 with self.db.connect() as conn:
153 conn.execute(
154 """INSERT INTO file_download_sources
155 (file, source, track_keys, signature, recorded_at)
156 VALUES (?, ?, ?, ?, ?)
157 ON CONFLICT(file) DO UPDATE SET
158 source = excluded.source,
159 track_keys = excluded.track_keys,
160 signature = excluded.signature,
161 recorded_at = excluded.recorded_at""",
162 (
163 _key(file),
164 str(source or '').strip(),
165 json.dumps(sorted(set(keys))),
166 json.dumps(signature) if signature else '',
167 time.time(),
168 ),
169 )
171 def get(self, file: str) -> Optional[dict[str, Any]]:
172 """``{file, source, track_keys, signature, recorded_at}`` or
173 ``None`` when nothing was recorded for *file*."""
175 with self.db.connect() as conn:
176 row = conn.execute(
177 'SELECT file, source, track_keys, signature, recorded_at '
178 'FROM file_download_sources WHERE file = ?',
179 (_key(file),),
180 ).fetchone()
181 if row is None:
182 return None
183 found = dict(row)
184 try:
185 found['track_keys'] = list(json.loads(found['track_keys']))
186 except (TypeError, ValueError):
187 found['track_keys'] = []
188 try:
189 found['signature'] = (
190 json.loads(found['signature']) if found['signature'] else None
191 )
192 except (TypeError, ValueError):
193 found['signature'] = None
194 return found
196 def replaced_on_disk(self, file: str, full_path: Path) -> bool:
197 """Whether the audio at *full_path* is no longer what Downtify
198 wrote to *file*: someone swapped the file outside Downtify. A file
199 Downtify never recorded, or can't read, says no."""
201 row = self.get(file)
202 if row is None or not row['signature'] or not full_path.is_file():
203 return False
204 current = audio_signature(full_path)
205 if current is None:
206 return False
207 return not same_audio(row['signature'], current)
209 # -- rejections -----------------------------------------------------
211 def reject_file_source(
212 self,
213 file: str,
214 extra_keys: Iterable[str] = (),
215 keep: str = '',
216 why: str = 'its audio was replaced',
217 ) -> Optional[str]:
218 """The upload *file* came from was wrong: never pick it again for
219 its song (known by the recorded keys plus *extra_keys*). Returns
220 the rejected source, or ``None`` when none is known (or it is
221 *keep*, the source that replaces it)."""
223 row = self.get(file)
224 source = str((row or {}).get('source') or '').strip()
225 if not source or source == str(keep or '').strip():
226 return None
227 keys = {*((row or {}).get('track_keys') or []), *extra_keys}
228 self.reject(source, keys)
229 logger.info('Source {} rejected for {} ({})', source, file, why)
230 return source
232 def reject(self, source: str, keys: Iterable[str]) -> None:
233 source = str(source or '').strip()
234 rows = [(key, source, time.time()) for key in set(keys) if key]
235 if not source or not rows:
236 return
237 with self.db.connect() as conn:
238 for row in rows:
239 conn.execute(
240 """INSERT INTO rejected_sources
241 (track_key, source, rejected_at) VALUES (?, ?, ?)
242 ON CONFLICT(track_key, source) DO UPDATE SET
243 rejected_at = excluded.rejected_at""",
244 row,
245 )
247 def allow(self, source: str, keys: Iterable[str]) -> None:
248 """Take back *source*'s rejection for the song known by *keys*
249 (an admin picked it again by hand)."""
251 source = str(source or '').strip()
252 keys = [key for key in set(keys) if key]
253 if not source or not keys:
254 return
255 marks = ', '.join('?' for _ in keys)
256 with self.db.connect() as conn:
257 conn.execute(
258 'DELETE FROM rejected_sources WHERE source = ? '
259 f'AND track_key IN ({marks})',
260 (source, *keys),
261 )
263 def rejected_for(self, song: dict[str, Any]) -> set[str]:
264 """Every source rejected for *song* under any of its keys."""
266 keys = track_keys(song)
267 if not keys:
268 return set()
269 marks = ', '.join('?' for _ in keys)
270 with self.db.connect() as conn:
271 rows = conn.execute(
272 'SELECT DISTINCT source FROM rejected_sources '
273 f'WHERE track_key IN ({marks})',
274 tuple(keys),
275 ).fetchall()
276 return {str(row['source']) for row in rows}
278 def forget_rejections(self, keys: Iterable[str]) -> int:
279 """Allow every rejected source of the song known by *keys* again."""
281 keys = [key for key in set(keys) if key]
282 if not keys:
283 return 0
284 marks = ', '.join('?' for _ in keys)
285 with self.db.connect() as conn:
286 cursor = conn.execute(
287 f'DELETE FROM rejected_sources WHERE track_key IN ({marks})',
288 tuple(keys),
289 )
290 return cursor.rowcount