Coverage for downtify/duplicate_merge.py: 93%
181 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Merging duplicate songs into one file (the Doctor's duplicate review).
3Deleting one copy of a duplicate isn't enough: the playlists, likes and
4Playlist Monitor records that pointed at it now point at nothing, so the
5song looks missing and a watch downloads it again (issue #582). Merging
6keeps one copy and moves **every reference** to the others onto it
7first - the M3U files (manual playlists too), the playlist catalog, the
8track index, likes, owners, hidden-song rules, the mobile apps' offline
9copies (an alias, as a replaced song's), Playlist Monitor's downloaded
10tracks and artist releases, and the watch history - and only then
11removes the other copies. Nothing is "missing" afterwards, and the song
12stays in every playlist it was in.
14It is the same publication Replace audio does
15(:class:`downtify.deemix_upgrade_store.AudioReplacementStore`), with a
16receipt per removed copy: M3U files first, then the library tables in
17one transaction, then a check that none still names a removed copy.
19"Keep both" (:class:`KeptApart`) is the other answer: the copies are
20different recordings that only look alike (an edit, a live take), and
21the Doctor stops listing them.
22"""
24from __future__ import annotations
26import contextlib
27from dataclasses import dataclass, field
28from datetime import datetime, timezone
29from itertools import combinations
30from pathlib import Path
31from typing import Any, Optional
33from loguru import logger
35from . import m3u
36from .db.config import Dialect
37from .db.core import Connection, Database, as_database
38from .deemix_upgrade_playlists import digest, rewrite_playlists
39from .deemix_upgrade_store import AudioReplacementStore
40from .library_cache_keys import file_content_key
41from .library_catalog import LibraryContext, resolve_library_file
42from .library_metadata import library_entry_for_file
43from .manual_playlists import drop_copies_of
44from .monitor import repoint_downloaded_file
46#: ``run_id`` of a merge's receipts (a library upgrade run is 1 or more).
47MERGE_RUN = 0
48#: A receipt left by an earlier merge or replacement of the same path,
49#: done with, may be replaced by a new one.
50_DONE_STATES = ('removed', 'complete', 'retained')
53class MergeError(ValueError):
54 """A merge refused before anything changed; ``code`` for the UI."""
56 def __init__(self, code: str, file: str = '') -> None:
57 super().__init__(code)
58 self.code = code
59 self.file = file
62def _norm(value: Any) -> str:
63 return str(value or '').strip().replace('\\', '/')
66def _now() -> str:
67 return datetime.now(timezone.utc).isoformat()
70# ---------------------------------------------------------------------------
71# Keep both
72# ---------------------------------------------------------------------------
75#: How a playlist (its M3U) is named among the pairs kept apart.
76PLAYLIST_PREFIX = 'm3u:'
79class KeptApart:
80 """Library files an admin said are different songs, by pairs.
82 A group of look-alikes stays out of the Doctor's list while every
83 pair in it is recorded, so a third copy turning up later brings the
84 group back (with the new file to decide on).
85 """
87 def __init__(self, path_or_db: str | Path | Database) -> None:
88 self.db = as_database(path_or_db, 'doctor_dupes')
89 self._path = self.db.path
90 if self.db.dialect is Dialect.SQLITE:
91 # Postgres already has the table from the migration.
92 with self._connect() as conn:
93 conn.execute("""
94 CREATE TABLE IF NOT EXISTS kept_apart (
95 a TEXT NOT NULL,
96 b TEXT NOT NULL,
97 kept_at TEXT NOT NULL,
98 PRIMARY KEY (a, b)
99 )
100 """)
102 def _connect(self) -> contextlib.AbstractContextManager[Connection]:
103 return self.db.connect()
105 @staticmethod
106 def _pairs(files: list[str]) -> list[tuple[str, str]]:
107 names = sorted({_norm(f) for f in files if _norm(f)})
108 return list(combinations(names, 2))
110 def keep(self, files: list[str]) -> int:
111 """Record *files* as different songs; returns the new pairs."""
113 pairs = self._pairs(files)
114 if not pairs:
115 return 0
116 now = _now()
117 with self._connect() as conn:
118 cursor = conn.executemany(
119 'INSERT INTO kept_apart VALUES (?, ?, ?) '
120 'ON CONFLICT DO NOTHING',
121 [(a, b, now) for a, b in pairs],
122 )
123 return cursor.rowcount
125 def pairs(self) -> set[tuple[str, str]]:
126 with self._connect() as conn:
127 return {
128 (str(a), str(b))
129 for a, b in conn.execute(
130 'SELECT a, b FROM kept_apart'
131 ).fetchall()
132 }
134 def settled(
135 self, files: list[str], pairs: Optional[set[tuple[str, str]]] = None
136 ) -> bool:
137 """Whether every pair of *files* was kept apart."""
139 known = self.pairs() if pairs is None else pairs
140 wanted = self._pairs(files)
141 return bool(wanted) and all(pair in known for pair in wanted)
143 def forget(self, *, playlists: bool = False) -> int:
144 """Forget every decision about songs (or, with *playlists*, about
145 playlists - their M3Us are kept as ``m3u:<path>``): all those
146 groups are listed again."""
148 op = 'LIKE' if playlists else 'NOT LIKE'
149 with self._connect() as conn:
150 return conn.execute(
151 f'DELETE FROM kept_apart WHERE a {op} ?',
152 (f'{PLAYLIST_PREFIX}%',),
153 ).rowcount
155 def forget_files(self, files: list[str]) -> None:
156 """Drop the decisions about files that are gone (merged)."""
158 names = [_norm(f) for f in files if _norm(f)]
159 if not names:
160 return
161 marks = ','.join('?' * len(names))
162 with self._connect() as conn:
163 conn.execute(
164 f'DELETE FROM kept_apart WHERE a IN ({marks}) '
165 f'OR b IN ({marks})',
166 names + names,
167 )
170# ---------------------------------------------------------------------------
171# Merge
172# ---------------------------------------------------------------------------
175@dataclass
176class MergeStores:
177 """What a merge rewrites besides the library database tables."""
179 ctx: LibraryContext
180 replacements: AudioReplacementStore
181 monitor_db: Any = None
182 library_visibility: Any = None
183 playlist_hidden: Any = None
184 file_providers: Any = None
185 playlist_catalog: Any = None
188@dataclass
189class MergePlan:
190 """A checked merge: *keep* stays, every file of *extras* goes."""
192 keep: str
193 keep_path: Path
194 extras: list[tuple[str, Path]]
195 receipts: list[dict[str, Any]] = field(default_factory=list)
196 playlists: set[str] = field(default_factory=set)
199def plan_merge(keep: str, extras: list[str], ctx: LibraryContext) -> MergePlan:
200 """Check that *keep* and every one of *extras* are library files
201 (distinct ones). Raises :class:`MergeError`; changes nothing."""
203 keep = _norm(keep)
204 names = list(dict.fromkeys(_norm(e) for e in extras if _norm(e)))
205 if not keep or not names or keep in names:
206 raise MergeError('dedupe_invalid')
207 keep_path = resolve_library_file(keep, ctx)
208 if keep_path is None or not keep_path.is_file():
209 raise MergeError('dedupe_missing', keep)
210 found: list[tuple[str, Path]] = []
211 for name in names:
212 path = resolve_library_file(name, ctx)
213 if path is None or not path.is_file():
214 raise MergeError('dedupe_missing', name)
215 if path.resolve() == keep_path.resolve():
216 raise MergeError('dedupe_invalid', name)
217 found.append((name, path))
218 return MergePlan(keep=keep, keep_path=keep_path, extras=found)
221def _receipts(
222 plan: MergePlan, store: AudioReplacementStore
223) -> list[dict[str, Any]]:
224 keep_key = file_content_key(plan.keep_path) or ''
225 keep_sha = digest(plan.keep_path)
226 metadata = library_entry_for_file(plan.keep, plan.keep_path)
227 metadata['file'] = plan.keep
228 receipts = []
229 for name, path in plan.extras:
230 earlier = store.get(name)
231 if earlier is not None and earlier['state'] not in _DONE_STATES:
232 # A Replace audio run is still working on this file.
233 raise MergeError('dedupe_replacement_pending', name)
234 receipts.append({
235 'old_file': name,
236 'run_id': MERGE_RUN,
237 'new_file': plan.keep,
238 'old_key': file_content_key(path) or '',
239 'new_key': keep_key,
240 'old_sha': digest(path),
241 'new_sha': keep_sha,
242 'new_metadata': metadata,
243 'catalog': {'merged_into': plan.keep},
244 'delete_original': 1,
245 })
246 return receipts
249def relink_duplicates(plan: MergePlan, stores: MergeStores) -> MergePlan:
250 """Point everything that names one of *plan*'s extras at its keeper.
252 The extras stay on disk: the caller removes them (to the music trash
253 or for good) and then calls :func:`finish_merge`. Raises when a
254 reference could not be moved; a rerun picks up where it stopped.
255 """
257 ctx = stores.ctx
258 store = stores.replacements
259 plan.receipts = _receipts(plan, store)
260 extras = [name for name, _path in plan.extras]
261 if stores.playlist_catalog is not None:
262 for names in stores.playlist_catalog.playlists_by_filenames(
263 extras
264 ).values():
265 plan.playlists.update(names)
267 # A manual playlist holding both copies keeps the one it already
268 # has, rather than listing the kept file twice.
269 plan.playlists.update(drop_copies_of(ctx, plan.keep, extras))
270 roots = [ctx.download_dir, *ctx.extra_dirs]
271 if ctx.slskd_dir is not None:
272 roots.append(ctx.slskd_dir)
273 duration = float(plan.receipts[0]['new_metadata'].get('duration') or 0)
274 target = plan.keep_path.resolve()
275 rewrite_playlists(
276 roots,
277 ctx.download_dir,
278 {path.resolve(): (target, duration) for _name, path in plan.extras},
279 stopped=lambda: False,
280 )
281 plan.playlists.update(_m3u_names_with(ctx, plan.keep))
283 for receipt in plan.receipts:
284 # The copy kept stays as visible as it was: hiding the extra
285 # copy (to tidy the library) mustn't hide the song.
286 if (
287 stores.library_visibility is not None
288 and receipt['old_key']
289 and receipt['old_key'] != receipt['new_key']
290 ):
291 stores.library_visibility.drop_track(receipt['old_key'])
292 _save_receipt(store, receipt)
293 store.relink(plan.receipts)
294 store.assert_relinked(plan.receipts)
296 for receipt in plan.receipts:
297 old_key, new_key = receipt['old_key'], receipt['new_key']
298 if stores.playlist_hidden is not None and old_key != new_key:
299 stores.playlist_hidden.rekey(old_key, new_key, plan.keep)
300 if stores.monitor_db is not None:
301 repoint_downloaded_file(
302 stores.monitor_db, receipt['old_file'], plan.keep
303 )
304 logger.info(
305 'Doctor: {} copy/copies of {} merged into it (playlists: {})',
306 len(plan.receipts),
307 plan.keep,
308 ', '.join(sorted(plan.playlists)) or 'none',
309 )
310 return plan
313def _save_receipt(
314 store: AudioReplacementStore, receipt: dict[str, Any]
315) -> None:
316 earlier = store.get(receipt['old_file'])
317 if earlier is not None:
318 with store.connect() as db:
319 db.execute(
320 'DELETE FROM library_audio_replacements WHERE old_file = ?',
321 (receipt['old_file'],),
322 )
323 store.save_prepared(receipt)
326def _m3u_names_with(ctx: LibraryContext, stored: str) -> set[str]:
327 names = set()
328 for path in m3u.iter_m3u_files(ctx.download_dir):
329 entries = m3u.read_m3u_entries(
330 path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
331 )
332 if any(_norm(entry) == stored for entry, _extinf in entries):
333 names.add(path.stem)
334 return names
337def finish_merge(
338 plan: MergePlan, stores: MergeStores, removed: list[str]
339) -> None:
340 """Record the extras that are gone, and forget what only they had
341 (an extra's provider record; not one its keeper shares a key with:
342 a file's key is its name and size, which exact copies share)."""
344 gone = {_norm(name) for name in removed}
345 for receipt in plan.receipts:
346 if receipt['old_file'] not in gone:
347 continue
348 stores.replacements.mark(
349 receipt['old_file'],
350 'removed',
351 'Duplicate merged into ' + plan.keep,
352 deleted=True,
353 )
354 old_key = receipt['old_key']
355 if (
356 stores.file_providers is not None
357 and old_key
358 and old_key != receipt['new_key']
359 ):
360 stores.file_providers.drop(old_key)
363def leftover_references(plan: MergePlan, stores: MergeStores) -> list[str]:
364 """Where a merged extra is still named (empty after a good merge)."""
366 found = []
367 try:
368 stores.replacements.assert_relinked(plan.receipts)
369 except ValueError:
370 found.append('library')
371 if stores.monitor_db is not None:
372 names = {name for name, _path in plan.extras}
373 if any(
374 _norm(row.get('filename')) in names
375 for row in stores.monitor_db.list_all_downloaded_tracks()
376 ):
377 found.append('monitor')
378 for name, _path in plan.extras:
379 if _m3u_names_with(stores.ctx, name):
380 found.append('m3u')
381 break
382 return found