Coverage for downtify/download_sources.py: 84%

122 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Where each downloaded file's audio came from, and the sources that 

2turned out wrong for a song (issue #582). 

3 

4Two things a library file carries no mark of: 

5 

6* **Which upload it is.** When a YouTube / YouTube Music / SoundCloud 

7 download finishes, the video id (or SoundCloud track URL) is kept here, 

8 by library path, with the song's identity (:func:`track_keys`) and a 

9 signature of the audio as written (:func:`audio_signature`: codec, 

10 sample rate, channels and length - tag rewrites leave it alone). 

11* **Which uploads are wrong for a song.** When an admin replaces a 

12 track's audio (Replace audio, or by swapping the file on disk), the 

13 upload it came from is *rejected* for that song: the matcher never picks 

14 it for the song again, from any playlist, artist watch or download 

15 (``providers.excluding_sources``). Otherwise the next download of the 

16 song, from another playlist, finds the same wrong video. 

17 

18A file swapped on disk outside Downtify (a CD rip or a Soulseek download 

19saved under the same name) is told apart by its audio signature: when it 

20no longer matches what Downtify wrote, :meth:`DownloadSources.replaced_on_disk` 

21says so, and the downloader keeps the file as if it had been replaced 

22through Replace audio (``downtify/hand_replaced.py``), even with 

23**Download again if the file exists** on. 

24""" 

25 

26from __future__ import annotations 

27 

28import json 

29import re 

30import time 

31import unicodedata 

32from pathlib import Path 

33from typing import Any, Iterable, Optional 

34 

35from loguru import logger 

36from mutagen import File as MutagenFile 

37 

38from .db.config import Dialect 

39from .db.core import Database, as_database 

40 

41#: Lengths (seconds) closer than this are the same audio: what a tag 

42#: rewrite can shift in a VBR estimate, far below any other recording. 

43LENGTH_TOLERANCE = 0.5 

44 

45_SPACES_RE = re.compile(r'\s+') 

46 

47 

48def _key(file: str) -> str: 

49 return str(file or '').replace('\\', '/').strip('/') 

50 

51 

52def _fold(text: Any) -> str: 

53 value = unicodedata.normalize('NFKC', str(text or '')).casefold() 

54 return _SPACES_RE.sub(' ', value).strip() 

55 

56 

57def track_keys(song: dict[str, Any]) -> list[str]: 

58 """The keys a song is known by: its Spotify id, its ISRC, and its 

59 lead artist + title. A rejection is kept under every one, so it holds 

60 whichever of them the next download of the song carries.""" 

61 

62 keys: list[str] = [] 

63 spotify_id = str(song.get('spotify_id') or '').strip() 

64 if not spotify_id and song.get('source') in {None, '', 'spotify'}: 

65 spotify_id = str(song.get('song_id') or '').strip() 

66 if re.fullmatch(r'[0-9A-Za-z]{22}', spotify_id): 

67 keys.append(f'spotify:{spotify_id}') 

68 isrc = str(song.get('isrc') or '').strip().upper() 

69 if isrc: 

70 keys.append(f'isrc:{isrc}') 

71 artists = song.get('artists') or [] 

72 if isinstance(artists, str): 

73 artists = [artists] 

74 artist = _fold(artists[0] if artists else song.get('artist')) 

75 title = _fold(song.get('name') or song.get('title')) 

76 if artist and title: 

77 keys.append(f'name:{artist}|{title}') 

78 return keys 

79 

80 

81def audio_signature(path: Path) -> Optional[dict[str, Any]]: 

82 """What the audio of *path* is, independent of its tags: ``{codec, 

83 sample_rate, channels, length}``; ``None`` when it can't be read.""" 

84 

85 try: 

86 audio = MutagenFile(str(path)) 

87 except Exception: 

88 return None 

89 info = getattr(audio, 'info', None) 

90 if audio is None or info is None: 

91 return None 

92 return { 

93 'codec': type(info).__name__, 

94 'sample_rate': int(getattr(info, 'sample_rate', 0) or 0), 

95 'channels': int(getattr(info, 'channels', 0) or 0), 

96 'length': round(float(getattr(info, 'length', 0) or 0), 3), 

97 } 

98 

99 

100def same_audio(a: dict[str, Any], b: dict[str, Any]) -> bool: 

101 """Whether two :func:`audio_signature` results are the same audio.""" 

102 

103 return ( 

104 a.get('codec') == b.get('codec') 

105 and a.get('sample_rate') == b.get('sample_rate') 

106 and a.get('channels') == b.get('channels') 

107 and abs(float(a.get('length') or 0) - float(b.get('length') or 0)) 

108 < LENGTH_TOLERANCE 

109 ) 

110 

111 

112class DownloadSources: 

113 """The source of each downloaded file and the sources rejected per 

114 song, in the library database.""" 

115 

116 def __init__(self, path_or_db: str | Path | Database) -> None: 

117 self.db = as_database(path_or_db, 'library') 

118 # The Postgres schema is created by migrations; only SQLite needs DDL. 

119 if self.db.dialect is Dialect.SQLITE: 

120 with self.db.connect() as conn: 

121 conn.execute(""" 

122 CREATE TABLE IF NOT EXISTS file_download_sources ( 

123 file TEXT PRIMARY KEY, 

124 source TEXT NOT NULL DEFAULT '', 

125 track_keys TEXT NOT NULL DEFAULT '[]', 

126 signature TEXT NOT NULL DEFAULT '', 

127 recorded_at REAL NOT NULL 

128 ) 

129 """) 

130 conn.execute(""" 

131 CREATE TABLE IF NOT EXISTS rejected_sources ( 

132 track_key TEXT NOT NULL, 

133 source TEXT NOT NULL, 

134 rejected_at REAL NOT NULL, 

135 PRIMARY KEY (track_key, source) 

136 ) 

137 """) 

138 

139 # -- per file ------------------------------------------------------- 

140 

141 def record( 

142 self, 

143 file: str, 

144 full_path: Optional[Path], 

145 source: str, 

146 keys: Iterable[str], 

147 ) -> None: 

148 """*file* was just written from *source* (a video id, a SoundCloud 

149 URL, or ``''`` for an uploaded file) for the song known by *keys*.""" 

150 

151 signature = audio_signature(full_path) if full_path else None 

152 with self.db.connect() as conn: 

153 conn.execute( 

154 """INSERT INTO file_download_sources 

155 (file, source, track_keys, signature, recorded_at) 

156 VALUES (?, ?, ?, ?, ?) 

157 ON CONFLICT(file) DO UPDATE SET 

158 source = excluded.source, 

159 track_keys = excluded.track_keys, 

160 signature = excluded.signature, 

161 recorded_at = excluded.recorded_at""", 

162 ( 

163 _key(file), 

164 str(source or '').strip(), 

165 json.dumps(sorted(set(keys))), 

166 json.dumps(signature) if signature else '', 

167 time.time(), 

168 ), 

169 ) 

170 

171 def get(self, file: str) -> Optional[dict[str, Any]]: 

172 """``{file, source, track_keys, signature, recorded_at}`` or 

173 ``None`` when nothing was recorded for *file*.""" 

174 

175 with self.db.connect() as conn: 

176 row = conn.execute( 

177 'SELECT file, source, track_keys, signature, recorded_at ' 

178 'FROM file_download_sources WHERE file = ?', 

179 (_key(file),), 

180 ).fetchone() 

181 if row is None: 

182 return None 

183 found = dict(row) 

184 try: 

185 found['track_keys'] = list(json.loads(found['track_keys'])) 

186 except (TypeError, ValueError): 

187 found['track_keys'] = [] 

188 try: 

189 found['signature'] = ( 

190 json.loads(found['signature']) if found['signature'] else None 

191 ) 

192 except (TypeError, ValueError): 

193 found['signature'] = None 

194 return found 

195 

196 def replaced_on_disk(self, file: str, full_path: Path) -> bool: 

197 """Whether the audio at *full_path* is no longer what Downtify 

198 wrote to *file*: someone swapped the file outside Downtify. A file 

199 Downtify never recorded, or can't read, says no.""" 

200 

201 row = self.get(file) 

202 if row is None or not row['signature'] or not full_path.is_file(): 

203 return False 

204 current = audio_signature(full_path) 

205 if current is None: 

206 return False 

207 return not same_audio(row['signature'], current) 

208 

209 # -- rejections ----------------------------------------------------- 

210 

211 def reject_file_source( 

212 self, 

213 file: str, 

214 extra_keys: Iterable[str] = (), 

215 keep: str = '', 

216 why: str = 'its audio was replaced', 

217 ) -> Optional[str]: 

218 """The upload *file* came from was wrong: never pick it again for 

219 its song (known by the recorded keys plus *extra_keys*). Returns 

220 the rejected source, or ``None`` when none is known (or it is 

221 *keep*, the source that replaces it).""" 

222 

223 row = self.get(file) 

224 source = str((row or {}).get('source') or '').strip() 

225 if not source or source == str(keep or '').strip(): 

226 return None 

227 keys = {*((row or {}).get('track_keys') or []), *extra_keys} 

228 self.reject(source, keys) 

229 logger.info('Source {} rejected for {} ({})', source, file, why) 

230 return source 

231 

232 def reject(self, source: str, keys: Iterable[str]) -> None: 

233 source = str(source or '').strip() 

234 rows = [(key, source, time.time()) for key in set(keys) if key] 

235 if not source or not rows: 

236 return 

237 with self.db.connect() as conn: 

238 for row in rows: 

239 conn.execute( 

240 """INSERT INTO rejected_sources 

241 (track_key, source, rejected_at) VALUES (?, ?, ?) 

242 ON CONFLICT(track_key, source) DO UPDATE SET 

243 rejected_at = excluded.rejected_at""", 

244 row, 

245 ) 

246 

247 def allow(self, source: str, keys: Iterable[str]) -> None: 

248 """Take back *source*'s rejection for the song known by *keys* 

249 (an admin picked it again by hand).""" 

250 

251 source = str(source or '').strip() 

252 keys = [key for key in set(keys) if key] 

253 if not source or not keys: 

254 return 

255 marks = ', '.join('?' for _ in keys) 

256 with self.db.connect() as conn: 

257 conn.execute( 

258 'DELETE FROM rejected_sources WHERE source = ? ' 

259 f'AND track_key IN ({marks})', 

260 (source, *keys), 

261 ) 

262 

263 def rejected_for(self, song: dict[str, Any]) -> set[str]: 

264 """Every source rejected for *song* under any of its keys.""" 

265 

266 keys = track_keys(song) 

267 if not keys: 

268 return set() 

269 marks = ', '.join('?' for _ in keys) 

270 with self.db.connect() as conn: 

271 rows = conn.execute( 

272 'SELECT DISTINCT source FROM rejected_sources ' 

273 f'WHERE track_key IN ({marks})', 

274 tuple(keys), 

275 ).fetchall() 

276 return {str(row['source']) for row in rows} 

277 

278 def forget_rejections(self, keys: Iterable[str]) -> int: 

279 """Allow every rejected source of the song known by *keys* again.""" 

280 

281 keys = [key for key in set(keys) if key] 

282 if not keys: 

283 return 0 

284 marks = ', '.join('?' for _ in keys) 

285 with self.db.connect() as conn: 

286 cursor = conn.execute( 

287 f'DELETE FROM rejected_sources WHERE track_key IN ({marks})', 

288 tuple(keys), 

289 ) 

290 return cursor.rowcount