Coverage for downtify/duplicate_merge.py: 93%

181 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Merging duplicate songs into one file (the Doctor's duplicate review). 

2 

3Deleting one copy of a duplicate isn't enough: the playlists, likes and 

4Playlist Monitor records that pointed at it now point at nothing, so the 

5song looks missing and a watch downloads it again (issue #582). Merging 

6keeps one copy and moves **every reference** to the others onto it 

7first - the M3U files (manual playlists too), the playlist catalog, the 

8track index, likes, owners, hidden-song rules, the mobile apps' offline 

9copies (an alias, as a replaced song's), Playlist Monitor's downloaded 

10tracks and artist releases, and the watch history - and only then 

11removes the other copies. Nothing is "missing" afterwards, and the song 

12stays in every playlist it was in. 

13 

14It is the same publication Replace audio does 

15(:class:`downtify.deemix_upgrade_store.AudioReplacementStore`), with a 

16receipt per removed copy: M3U files first, then the library tables in 

17one transaction, then a check that none still names a removed copy. 

18 

19"Keep both" (:class:`KeptApart`) is the other answer: the copies are 

20different recordings that only look alike (an edit, a live take), and 

21the Doctor stops listing them. 

22""" 

23 

24from __future__ import annotations 

25 

26import contextlib 

27from dataclasses import dataclass, field 

28from datetime import datetime, timezone 

29from itertools import combinations 

30from pathlib import Path 

31from typing import Any, Optional 

32 

33from loguru import logger 

34 

35from . import m3u 

36from .db.config import Dialect 

37from .db.core import Connection, Database, as_database 

38from .deemix_upgrade_playlists import digest, rewrite_playlists 

39from .deemix_upgrade_store import AudioReplacementStore 

40from .library_cache_keys import file_content_key 

41from .library_catalog import LibraryContext, resolve_library_file 

42from .library_metadata import library_entry_for_file 

43from .manual_playlists import drop_copies_of 

44from .monitor import repoint_downloaded_file 

45 

46#: ``run_id`` of a merge's receipts (a library upgrade run is 1 or more). 

47MERGE_RUN = 0 

48#: A receipt left by an earlier merge or replacement of the same path, 

49#: done with, may be replaced by a new one. 

50_DONE_STATES = ('removed', 'complete', 'retained') 

51 

52 

53class MergeError(ValueError): 

54 """A merge refused before anything changed; ``code`` for the UI.""" 

55 

56 def __init__(self, code: str, file: str = '') -> None: 

57 super().__init__(code) 

58 self.code = code 

59 self.file = file 

60 

61 

62def _norm(value: Any) -> str: 

63 return str(value or '').strip().replace('\\', '/') 

64 

65 

66def _now() -> str: 

67 return datetime.now(timezone.utc).isoformat() 

68 

69 

70# --------------------------------------------------------------------------- 

71# Keep both 

72# --------------------------------------------------------------------------- 

73 

74 

75#: How a playlist (its M3U) is named among the pairs kept apart. 

76PLAYLIST_PREFIX = 'm3u:' 

77 

78 

79class KeptApart: 

80 """Library files an admin said are different songs, by pairs. 

81 

82 A group of look-alikes stays out of the Doctor's list while every 

83 pair in it is recorded, so a third copy turning up later brings the 

84 group back (with the new file to decide on). 

85 """ 

86 

87 def __init__(self, path_or_db: str | Path | Database) -> None: 

88 self.db = as_database(path_or_db, 'doctor_dupes') 

89 self._path = self.db.path 

90 if self.db.dialect is Dialect.SQLITE: 

91 # Postgres already has the table from the migration. 

92 with self._connect() as conn: 

93 conn.execute(""" 

94 CREATE TABLE IF NOT EXISTS kept_apart ( 

95 a TEXT NOT NULL, 

96 b TEXT NOT NULL, 

97 kept_at TEXT NOT NULL, 

98 PRIMARY KEY (a, b) 

99 ) 

100 """) 

101 

102 def _connect(self) -> contextlib.AbstractContextManager[Connection]: 

103 return self.db.connect() 

104 

105 @staticmethod 

106 def _pairs(files: list[str]) -> list[tuple[str, str]]: 

107 names = sorted({_norm(f) for f in files if _norm(f)}) 

108 return list(combinations(names, 2)) 

109 

110 def keep(self, files: list[str]) -> int: 

111 """Record *files* as different songs; returns the new pairs.""" 

112 

113 pairs = self._pairs(files) 

114 if not pairs: 

115 return 0 

116 now = _now() 

117 with self._connect() as conn: 

118 cursor = conn.executemany( 

119 'INSERT INTO kept_apart VALUES (?, ?, ?) ' 

120 'ON CONFLICT DO NOTHING', 

121 [(a, b, now) for a, b in pairs], 

122 ) 

123 return cursor.rowcount 

124 

125 def pairs(self) -> set[tuple[str, str]]: 

126 with self._connect() as conn: 

127 return { 

128 (str(a), str(b)) 

129 for a, b in conn.execute( 

130 'SELECT a, b FROM kept_apart' 

131 ).fetchall() 

132 } 

133 

134 def settled( 

135 self, files: list[str], pairs: Optional[set[tuple[str, str]]] = None 

136 ) -> bool: 

137 """Whether every pair of *files* was kept apart.""" 

138 

139 known = self.pairs() if pairs is None else pairs 

140 wanted = self._pairs(files) 

141 return bool(wanted) and all(pair in known for pair in wanted) 

142 

143 def forget(self, *, playlists: bool = False) -> int: 

144 """Forget every decision about songs (or, with *playlists*, about 

145 playlists - their M3Us are kept as ``m3u:<path>``): all those 

146 groups are listed again.""" 

147 

148 op = 'LIKE' if playlists else 'NOT LIKE' 

149 with self._connect() as conn: 

150 return conn.execute( 

151 f'DELETE FROM kept_apart WHERE a {op} ?', 

152 (f'{PLAYLIST_PREFIX}%',), 

153 ).rowcount 

154 

155 def forget_files(self, files: list[str]) -> None: 

156 """Drop the decisions about files that are gone (merged).""" 

157 

158 names = [_norm(f) for f in files if _norm(f)] 

159 if not names: 

160 return 

161 marks = ','.join('?' * len(names)) 

162 with self._connect() as conn: 

163 conn.execute( 

164 f'DELETE FROM kept_apart WHERE a IN ({marks}) ' 

165 f'OR b IN ({marks})', 

166 names + names, 

167 ) 

168 

169 

170# --------------------------------------------------------------------------- 

171# Merge 

172# --------------------------------------------------------------------------- 

173 

174 

175@dataclass 

176class MergeStores: 

177 """What a merge rewrites besides the library database tables.""" 

178 

179 ctx: LibraryContext 

180 replacements: AudioReplacementStore 

181 monitor_db: Any = None 

182 library_visibility: Any = None 

183 playlist_hidden: Any = None 

184 file_providers: Any = None 

185 playlist_catalog: Any = None 

186 

187 

188@dataclass 

189class MergePlan: 

190 """A checked merge: *keep* stays, every file of *extras* goes.""" 

191 

192 keep: str 

193 keep_path: Path 

194 extras: list[tuple[str, Path]] 

195 receipts: list[dict[str, Any]] = field(default_factory=list) 

196 playlists: set[str] = field(default_factory=set) 

197 

198 

199def plan_merge(keep: str, extras: list[str], ctx: LibraryContext) -> MergePlan: 

200 """Check that *keep* and every one of *extras* are library files 

201 (distinct ones). Raises :class:`MergeError`; changes nothing.""" 

202 

203 keep = _norm(keep) 

204 names = list(dict.fromkeys(_norm(e) for e in extras if _norm(e))) 

205 if not keep or not names or keep in names: 

206 raise MergeError('dedupe_invalid') 

207 keep_path = resolve_library_file(keep, ctx) 

208 if keep_path is None or not keep_path.is_file(): 

209 raise MergeError('dedupe_missing', keep) 

210 found: list[tuple[str, Path]] = [] 

211 for name in names: 

212 path = resolve_library_file(name, ctx) 

213 if path is None or not path.is_file(): 

214 raise MergeError('dedupe_missing', name) 

215 if path.resolve() == keep_path.resolve(): 

216 raise MergeError('dedupe_invalid', name) 

217 found.append((name, path)) 

218 return MergePlan(keep=keep, keep_path=keep_path, extras=found) 

219 

220 

221def _receipts( 

222 plan: MergePlan, store: AudioReplacementStore 

223) -> list[dict[str, Any]]: 

224 keep_key = file_content_key(plan.keep_path) or '' 

225 keep_sha = digest(plan.keep_path) 

226 metadata = library_entry_for_file(plan.keep, plan.keep_path) 

227 metadata['file'] = plan.keep 

228 receipts = [] 

229 for name, path in plan.extras: 

230 earlier = store.get(name) 

231 if earlier is not None and earlier['state'] not in _DONE_STATES: 

232 # A Replace audio run is still working on this file. 

233 raise MergeError('dedupe_replacement_pending', name) 

234 receipts.append({ 

235 'old_file': name, 

236 'run_id': MERGE_RUN, 

237 'new_file': plan.keep, 

238 'old_key': file_content_key(path) or '', 

239 'new_key': keep_key, 

240 'old_sha': digest(path), 

241 'new_sha': keep_sha, 

242 'new_metadata': metadata, 

243 'catalog': {'merged_into': plan.keep}, 

244 'delete_original': 1, 

245 }) 

246 return receipts 

247 

248 

249def relink_duplicates(plan: MergePlan, stores: MergeStores) -> MergePlan: 

250 """Point everything that names one of *plan*'s extras at its keeper. 

251 

252 The extras stay on disk: the caller removes them (to the music trash 

253 or for good) and then calls :func:`finish_merge`. Raises when a 

254 reference could not be moved; a rerun picks up where it stopped. 

255 """ 

256 

257 ctx = stores.ctx 

258 store = stores.replacements 

259 plan.receipts = _receipts(plan, store) 

260 extras = [name for name, _path in plan.extras] 

261 if stores.playlist_catalog is not None: 

262 for names in stores.playlist_catalog.playlists_by_filenames( 

263 extras 

264 ).values(): 

265 plan.playlists.update(names) 

266 

267 # A manual playlist holding both copies keeps the one it already 

268 # has, rather than listing the kept file twice. 

269 plan.playlists.update(drop_copies_of(ctx, plan.keep, extras)) 

270 roots = [ctx.download_dir, *ctx.extra_dirs] 

271 if ctx.slskd_dir is not None: 

272 roots.append(ctx.slskd_dir) 

273 duration = float(plan.receipts[0]['new_metadata'].get('duration') or 0) 

274 target = plan.keep_path.resolve() 

275 rewrite_playlists( 

276 roots, 

277 ctx.download_dir, 

278 {path.resolve(): (target, duration) for _name, path in plan.extras}, 

279 stopped=lambda: False, 

280 ) 

281 plan.playlists.update(_m3u_names_with(ctx, plan.keep)) 

282 

283 for receipt in plan.receipts: 

284 # The copy kept stays as visible as it was: hiding the extra 

285 # copy (to tidy the library) mustn't hide the song. 

286 if ( 

287 stores.library_visibility is not None 

288 and receipt['old_key'] 

289 and receipt['old_key'] != receipt['new_key'] 

290 ): 

291 stores.library_visibility.drop_track(receipt['old_key']) 

292 _save_receipt(store, receipt) 

293 store.relink(plan.receipts) 

294 store.assert_relinked(plan.receipts) 

295 

296 for receipt in plan.receipts: 

297 old_key, new_key = receipt['old_key'], receipt['new_key'] 

298 if stores.playlist_hidden is not None and old_key != new_key: 

299 stores.playlist_hidden.rekey(old_key, new_key, plan.keep) 

300 if stores.monitor_db is not None: 

301 repoint_downloaded_file( 

302 stores.monitor_db, receipt['old_file'], plan.keep 

303 ) 

304 logger.info( 

305 'Doctor: {} copy/copies of {} merged into it (playlists: {})', 

306 len(plan.receipts), 

307 plan.keep, 

308 ', '.join(sorted(plan.playlists)) or 'none', 

309 ) 

310 return plan 

311 

312 

313def _save_receipt( 

314 store: AudioReplacementStore, receipt: dict[str, Any] 

315) -> None: 

316 earlier = store.get(receipt['old_file']) 

317 if earlier is not None: 

318 with store.connect() as db: 

319 db.execute( 

320 'DELETE FROM library_audio_replacements WHERE old_file = ?', 

321 (receipt['old_file'],), 

322 ) 

323 store.save_prepared(receipt) 

324 

325 

326def _m3u_names_with(ctx: LibraryContext, stored: str) -> set[str]: 

327 names = set() 

328 for path in m3u.iter_m3u_files(ctx.download_dir): 

329 entries = m3u.read_m3u_entries( 

330 path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

331 ) 

332 if any(_norm(entry) == stored for entry, _extinf in entries): 

333 names.add(path.stem) 

334 return names 

335 

336 

337def finish_merge( 

338 plan: MergePlan, stores: MergeStores, removed: list[str] 

339) -> None: 

340 """Record the extras that are gone, and forget what only they had 

341 (an extra's provider record; not one its keeper shares a key with: 

342 a file's key is its name and size, which exact copies share).""" 

343 

344 gone = {_norm(name) for name in removed} 

345 for receipt in plan.receipts: 

346 if receipt['old_file'] not in gone: 

347 continue 

348 stores.replacements.mark( 

349 receipt['old_file'], 

350 'removed', 

351 'Duplicate merged into ' + plan.keep, 

352 deleted=True, 

353 ) 

354 old_key = receipt['old_key'] 

355 if ( 

356 stores.file_providers is not None 

357 and old_key 

358 and old_key != receipt['new_key'] 

359 ): 

360 stores.file_providers.drop(old_key) 

361 

362 

363def leftover_references(plan: MergePlan, stores: MergeStores) -> list[str]: 

364 """Where a merged extra is still named (empty after a good merge).""" 

365 

366 found = [] 

367 try: 

368 stores.replacements.assert_relinked(plan.receipts) 

369 except ValueError: 

370 found.append('library') 

371 if stores.monitor_db is not None: 

372 names = {name for name, _path in plan.extras} 

373 if any( 

374 _norm(row.get('filename')) in names 

375 for row in stores.monitor_db.list_all_downloaded_tracks() 

376 ): 

377 found.append('monitor') 

378 for name, _path in plan.extras: 

379 if _m3u_names_with(stores.ctx, name): 

380 found.append('m3u') 

381 break 

382 return found