Coverage for downtify/doctor.py: 91%
443 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Doctor: find and repair what older Downtify versions left behind.
3New versions fix a problem from then on - slskd transfers are now
4removed after import, playlist covers sit next to the M3U, Navidrome
5playlists are synced - but files and playlists written by an older
6version keep the old shape. The Doctor looks for that leftover state
7and, where it can prove a repair is safe, offers to fix it:
9* **slskd leftovers** (:func:`working_copy_rows`, :func:`verify_copies`) -
10 transfers an older version left in slskd's folder after copying them
11 into the library. A leftover is only ever offered for deletion once a
12 library file holding bit-identical audio is found
13 (:mod:`downtify.doctor_audio`).
14* **Duplicate playlists** (:func:`m3u_duplicate_groups`) - two M3U files
15 for one playlist, e.g. one left at the old ``Playlists/`` location.
16 Merging keeps every song of both (only when they mostly overlap), the
17 cover from whichever has one, and one file.
18* **Navidrome** (:func:`navidrome_playlist_findings`) - what clients
19 actually see: an M3U Navidrome imported *and* the playlist Downtify
20 created through the API, imported playlists whose M3U is gone (Navidrome
21 never removes those), and imported playlists missing songs Navidrome
22 couldn't find.
23* **Duplicate songs** (:func:`navidrome_song_duplicates`, plus
24 :func:`downtify.library_usage.duplicates` for Downtify's own view).
25* **Same song, new Spotify id** (:func:`relinked_tracks`) - a monitored
26 playlist whose track got a new Spotify id is downloaded again.
28Everything here is pure or takes its I/O as arguments; the routes
29(:mod:`downtify.doctor_routes`) gather the inputs and run the repairs.
30"""
32from __future__ import annotations
34import os
35import shutil
36from collections import defaultdict
37from collections.abc import Callable, Collection, Iterable, Sequence
38from dataclasses import dataclass, field
39from pathlib import Path, PurePosixPath
40from typing import Any, Optional
42from .library_catalog import song_key
43from .library_paths import SLSKD_LIBRARY_PREFIX
44from .library_usage import split_working_copies
45from .m3u import build_m3u_content, read_m3u_entries
46from .playlist_identity import imported_from, made_by_downtify, playlist_key
48#: A leftover changed this recently may still be written by slskd.
49RECENT_SECONDS = 15 * 60
50#: Two library files of one song whose lengths differ by more than this
51#: (seconds) are not the same recording.
52DURATION_TOLERANCE = 2.0
53#: Library files hashed per leftover before giving up on it.
54MAX_CANDIDATES = 8
55#: Share of the smaller playlist that must be in the larger one before
56#: two playlists are merged.
57MIN_OVERLAP = 0.8
59#: The Doctor's sections. The health check looks at all of them but
60#: ``slskd``, which has its own (slower) check.
61SECTIONS = ('slskd', 'playlists', 'navidrome', 'duplicates', 'relinked')
62HEALTH_SECTIONS = frozenset(SECTIONS) - {'slskd'}
63#: The sections each check finds again.
64CHECKS = {'health': HEALTH_SECTIONS, 'slskd': frozenset({'slskd'})}
65#: The sections whose findings each repair can change. Merging duplicate
66#: songs rewrites M3Us and removes library files a verified slskd
67#: leftover may match; deleting slskd leftovers changes the songs the
68#: duplicate and Navidrome checks counted; merging playlists changes the
69#: M3Us the slskd check counts as references. Merging duplicates leaves
70#: its own list usable - the server takes the merged copies out of it.
71REPAIR_IMPACT = {
72 'slskd_clean': frozenset({'slskd', 'duplicates', 'navidrome', 'relinked'}),
73 'merge': frozenset({'slskd', 'playlists', 'navidrome', 'duplicates'}),
74 'playlists': frozenset({'slskd', 'playlists', 'navidrome', 'duplicates'}),
75 'navidrome': frozenset({'navidrome'}),
76 'dedupe': frozenset({'slskd', 'playlists', 'navidrome', 'relinked'}),
77}
80def stale_after(stale: Collection[str], kind: str) -> set[str]:
81 """The sections out of date once a *kind* job has finished: a check
82 brings its own sections up to date, a repair puts the ones it can
83 change out of date until they are checked again."""
85 now = set(stale) - CHECKS.get(kind, frozenset())
86 return now | REPAIR_IMPACT.get(kind, frozenset())
89STATUS_VERIFIED = 'verified'
90STATUS_DIFFERENT = 'different'
91STATUS_UNMATCHED = 'unmatched'
92STATUS_UNREADABLE = 'unreadable'
93STATUS_RECENT = 'recent'
94STATUS_MISSING = 'missing'
97# ---------------------------------------------------------------------------
98# slskd leftovers
99# ---------------------------------------------------------------------------
102def norm(path: Any) -> str:
103 return str(path or '').strip().replace('\\', '/')
106def working_copy_rows(
107 entries: Iterable[dict[str, Any]],
108 *,
109 sources: dict[str, int],
110 slskd_in_place: bool,
111 referenced: Collection[str],
112) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
113 """``(library rows, leftover rows)`` - the same split the storage
114 usage page makes (:func:`downtify.library_usage.split_working_copies`),
115 with each leftover's listing row (tags, length) kept for matching.
117 A recorded transfer the listing doesn't know comes back as a bare
118 ``{'file', 'size'}`` row.
119 """
121 rows = list(entries)
122 library, working = split_working_copies(
123 rows,
124 sources=sources,
125 slskd_in_place=slskd_in_place,
126 referenced=referenced,
127 )
128 by_file = {norm(r.get('file')): r for r in rows}
129 leftovers = [
130 by_file.get(name) or {'file': name, 'size': size}
131 for name, size in sorted(working.items())
132 ]
133 return library, leftovers
136def _duration(row: dict[str, Any]) -> float:
137 try:
138 return float(row.get('duration') or 0)
139 except (TypeError, ValueError):
140 return 0.0
143def _codec(row: dict[str, Any]) -> str:
144 codec = str(row.get('codec') or '').strip().lower()
145 if codec:
146 return codec
147 return PurePosixPath(norm(row.get('file'))).suffix.lstrip('.').lower()
150class CandidateIndex:
151 """Library rows reachable by song (artist/title) and by length."""
153 def __init__(self, library: Iterable[dict[str, Any]]) -> None:
154 self.by_song: dict[str, list[dict[str, Any]]] = defaultdict(list)
155 self.by_second: dict[int, list[dict[str, Any]]] = defaultdict(list)
156 for row in library:
157 name = norm(row.get('file'))
158 if not name or name.startswith(SLSKD_LIBRARY_PREFIX):
159 continue
160 key = song_key(
161 str(row.get('artist') or ''), str(row.get('title') or '')
162 )
163 if key != '|':
164 self.by_song[key].append(row)
165 length = _duration(row)
166 if length:
167 self.by_second[round(length)].append(row)
169 def candidates(self, leftover: dict[str, Any]) -> list[dict[str, Any]]:
170 """Library rows that could hold *leftover*'s recording, the most
171 likely first: same song and length, then same length and format.
172 """
174 length = _duration(leftover)
175 codec = _codec(leftover)
176 found: list[dict[str, Any]] = []
177 seen: set[str] = set()
179 def add(row: dict[str, Any]) -> None:
180 name = norm(row.get('file'))
181 if name not in seen:
182 seen.add(name)
183 found.append(row)
185 key = song_key(
186 str(leftover.get('artist') or ''),
187 str(leftover.get('title') or ''),
188 )
189 for row in self.by_song.get(key, []) if key != '|' else []:
190 other = _duration(row)
191 if not length or not other or abs(other - length) <= 2:
192 add(row)
193 if length:
194 second = round(length)
195 for near in (second, second - 1, second + 1):
196 for row in self.by_second.get(near, []):
197 if (
198 abs(_duration(row) - length) <= 1
199 and _codec(row) == codec
200 ):
201 add(row)
202 return found[:MAX_CANDIDATES]
205@dataclass
206class CopyResult:
207 file: str
208 bytes: int
209 status: str
210 match: str = ''
211 title: str = ''
212 artist: str = ''
213 mtime_ns: int = 0
215 def as_dict(self) -> dict[str, Any]:
216 return {
217 'file': self.file,
218 'bytes': self.bytes,
219 'status': self.status,
220 'match': self.match,
221 'title': self.title,
222 'artist': self.artist,
223 }
226def verify_copies(
227 leftovers: Sequence[dict[str, Any]],
228 library: Iterable[dict[str, Any]],
229 *,
230 locate: Callable[[str], Optional[Path]],
231 digest: Callable[[Path], Optional[str]],
232 now: float,
233 progress: Optional[Callable[[int, int], None]] = None,
234 cancelled: Callable[[], bool] = lambda: False,
235) -> list[CopyResult]:
236 """Check each leftover against the library.
238 ``verified`` means a library file outside slskd's folder holds
239 bit-identical audio (same :func:`downtify.doctor_audio.audio_md5`) -
240 the only status a cleanup acts on. Everything else is kept:
241 ``different`` (look-alikes, none identical), ``unmatched`` (no
242 library file looks like it), ``unreadable`` (ffmpeg couldn't read
243 it), ``recent`` (changed in the last :data:`RECENT_SECONDS`) and
244 ``missing`` (gone since the listing).
245 """
247 index = CandidateIndex(library)
248 results: list[CopyResult] = []
249 total = len(leftovers)
250 for done, row in enumerate(leftovers):
251 if cancelled():
252 break
253 if progress is not None:
254 progress(done, total)
255 name = norm(row.get('file'))
256 size = int(row.get('size') or 0)
257 base = CopyResult(
258 name,
259 size,
260 STATUS_UNMATCHED,
261 title=str(row.get('title') or ''),
262 artist=str(row.get('artist') or ''),
263 )
264 path = locate(name)
265 if path is None:
266 base.status = STATUS_MISSING
267 results.append(base)
268 continue
269 try:
270 stat = path.stat()
271 except OSError:
272 base.status = STATUS_MISSING
273 results.append(base)
274 continue
275 base.bytes = stat.st_size
276 base.mtime_ns = stat.st_mtime_ns
277 if now - stat.st_mtime < RECENT_SECONDS:
278 base.status = STATUS_RECENT
279 results.append(base)
280 continue
281 candidates = index.candidates(row)
282 if not candidates:
283 results.append(base)
284 continue
285 own = digest(path)
286 if own is None:
287 base.status = STATUS_UNREADABLE
288 results.append(base)
289 continue
290 base.status = STATUS_DIFFERENT
291 for candidate in candidates:
292 other = locate(norm(candidate.get('file')))
293 if other is None or other == path:
294 continue
295 if digest(other) == own:
296 base.status = STATUS_VERIFIED
297 base.match = norm(candidate.get('file'))
298 break
299 results.append(base)
300 if progress is not None:
301 progress(len(results), total)
302 return results
305def summarize_copies(results: Iterable[CopyResult]) -> dict[str, Any]:
306 """``{status: {files, bytes}}`` for every status."""
308 out: dict[str, dict[str, int]] = {}
309 for result in results:
310 bucket = out.setdefault(result.status, {'files': 0, 'bytes': 0})
311 bucket['files'] += 1
312 bucket['bytes'] += result.bytes
313 return out
316#: Files of each kind the slskd folder breakdown lists by name.
317FOLDER_EXAMPLES = 25
318#: Kinds of file in slskd's folder (:func:`slskd_folder_breakdown`).
319FOLDER_LIBRARY = 'library'
320FOLDER_LEFTOVER = 'leftover'
321FOLDER_UNLISTED = 'unlisted_audio'
322FOLDER_OTHER = 'other'
323FOLDER_KINDS = (FOLDER_LIBRARY, FOLDER_LEFTOVER, FOLDER_UNLISTED, FOLDER_OTHER)
326def slskd_folder_breakdown(
327 files: Iterable[tuple[str, int]],
328 *,
329 library: Collection[str],
330 leftovers: Collection[str],
331 audio_extensions: Collection[str],
332 examples: int = FOLDER_EXAMPLES,
333) -> dict[str, Any]:
334 """What slskd's folder holds, next to what Downtify counts.
336 *files* are ``(stored path, size)`` for every file on disk under
337 slskd's folder (``slskd/...``). Each is a library song (in
338 *library*), a leftover the slskd check looks at (in *leftovers*),
339 audio Downtify's listing doesn't know (never imported, or kept after
340 a failed import), or another kind of file (covers, lyrics, slskd's
341 own files). Library and leftovers are what the storage page counts;
342 the other two explain why the folder looks bigger on disk.
343 """
345 audio = {e.lower().lstrip('.') for e in audio_extensions}
346 lib = {norm(f) for f in library}
347 left = {norm(f) for f in leftovers}
348 totals = {kind: {'files': 0, 'bytes': 0} for kind in FOLDER_KINDS}
349 by_ext: dict[str, dict[str, int]] = defaultdict(
350 lambda: {'files': 0, 'bytes': 0}
351 )
352 listed: dict[str, list[tuple[int, str]]] = {
353 FOLDER_UNLISTED: [],
354 FOLDER_OTHER: [],
355 }
356 for raw, raw_size in files:
357 name = norm(raw)
358 size = max(0, int(raw_size or 0))
359 ext = PurePosixPath(name).suffix.lstrip('.').lower()
360 if name in lib:
361 kind = FOLDER_LIBRARY
362 elif name in left:
363 kind = FOLDER_LEFTOVER
364 elif ext in audio:
365 kind = FOLDER_UNLISTED
366 else:
367 kind = FOLDER_OTHER
368 bucket = by_ext[ext or '']
369 bucket['files'] += 1
370 bucket['bytes'] += size
371 totals[kind]['files'] += 1
372 totals[kind]['bytes'] += size
373 if kind in listed:
374 listed[kind].append((size, name))
375 return {
376 'total': {
377 'files': sum(t['files'] for t in totals.values()),
378 'bytes': sum(t['bytes'] for t in totals.values()),
379 },
380 'kinds': totals,
381 'other_by_extension': sorted(
382 ({'extension': ext, **row} for ext, row in by_ext.items()),
383 key=lambda row: (-row['bytes'], row['extension']),
384 )[:10],
385 'examples': {
386 kind: [
387 {'file': name, 'bytes': size}
388 for size, name in sorted(rows, key=lambda r: (-r[0], r[1]))[
389 :examples
390 ]
391 ]
392 for kind, rows in listed.items()
393 },
394 }
397def prune_empty_dirs(start: Path, stop: Path) -> int:
398 """Remove *start* and its empty parents, never *stop* or above."""
400 removed = 0
401 stop = stop.resolve()
402 current = start.resolve()
403 while current != stop and stop in current.parents:
404 try:
405 current.rmdir()
406 except OSError:
407 break
408 removed += 1
409 current = current.parent
410 return removed
413# ---------------------------------------------------------------------------
414# Playlists
415# ---------------------------------------------------------------------------
418@dataclass
419class M3UInfo:
420 """One M3U file in Downtify's folder."""
422 path: str # relative to the downloads folder, POSIX
423 name: str
424 manual: bool
425 entries: list[str] # library paths of the songs that exist
426 lines: int # every track line, found or not
427 cover: bool
428 expected: bool # where Downtify writes this playlist today
430 def as_dict(self) -> dict[str, Any]:
431 return {
432 'path': self.path,
433 'name': self.name,
434 'manual': self.manual,
435 'songs': len(self.entries),
436 'lines': self.lines,
437 'cover': self.cover,
438 'expected': self.expected,
439 }
442def overlap(small: Collection[str], large: Collection[str]) -> float:
443 """Share of *small* that is also in *large* (1.0 for an empty one)."""
445 if not small:
446 return 1.0
447 big = set(large)
448 return sum(1 for item in small if item in big) / len(small)
451@dataclass
452class PlaylistGroup:
453 """M3U files that hold one playlist."""
455 key: str
456 members: list[M3UInfo]
457 reason: str # 'name' or 'songs'
458 keeper: M3UInfo = field(init=False)
459 largest: M3UInfo = field(init=False)
461 def __post_init__(self) -> None:
462 self.largest = max(
463 self.members, key=lambda m: (len(m.entries), m.cover)
464 )
465 expected = [m for m in self.members if m.expected]
466 self.keeper = expected[0] if expected else self.largest
468 def merged(self) -> list[str]:
469 """Every song of every member, the largest member's order first."""
471 order = [
472 self.largest,
473 *[m for m in self.members if m is not self.largest],
474 ]
475 seen: set[str] = set()
476 out: list[str] = []
477 for member in order:
478 for entry in member.entries:
479 if entry not in seen:
480 seen.add(entry)
481 out.append(entry)
482 return out
484 def min_overlap(self) -> float:
485 """The worst overlap of a member with the largest one."""
487 return min(
488 (
489 overlap(m.entries, self.largest.entries)
490 for m in self.members
491 if m is not self.largest
492 ),
493 default=1.0,
494 )
496 def safe(self) -> bool:
497 return self.min_overlap() >= MIN_OVERLAP
499 def as_dict(self) -> dict[str, Any]:
500 cover_from = (
501 self.keeper.path
502 if self.keeper.cover
503 else next((m.path for m in self.members if m.cover), '')
504 )
505 return {
506 'id': self.key,
507 'reason': self.reason,
508 'members': [m.as_dict() for m in self.members],
509 'keeper': self.keeper.path,
510 'songs_after': len(self.merged()),
511 'cover_from': cover_from,
512 'overlap': round(self.min_overlap(), 3),
513 'safe': self.safe(),
514 }
517def name_key(name: str) -> str:
518 """:func:`playlist_key` without spaces: a name and the file name
519 written for it compare equal however their punctuation was dropped
520 (``DAMNN...its already 3am🥵`` and ``DAMNNits already 3am``,
521 ``Rajju patel's playlist`` and ``Rajju patels playlist``)."""
523 return playlist_key(name).replace(' ', '')
526def same_songs(a: Collection[str], b: Collection[str]) -> bool:
527 """Whether two playlists hold nearly the same songs: at least
528 :data:`MIN_OVERLAP` of *each* one is in the other, and at least 5.
530 Both ways, not just the smaller one: a 50-song mix is almost all
531 inside a 7,000-song liked-songs backup, but they aren't one
532 playlist.
533 """
535 first, second = set(a), set(b)
536 if min(len(first), len(second)) < 5:
537 return False
538 shared = len(first & second)
539 return shared >= MIN_OVERLAP * len(first) and shared >= MIN_OVERLAP * len(
540 second
541 )
544def m3u_duplicate_groups(infos: Sequence[M3UInfo]) -> list[PlaylistGroup]:
545 """M3U files that look like one playlist.
547 Same name (:func:`name_key`) groups them; so does holding nearly
548 the same songs under different names (:func:`same_songs`) - a
549 playlist renamed on Spotify. A playlist joins a songs group only
550 when it holds nearly the same songs as *every* playlist already in
551 it, so one huge playlist can't chain unrelated ones together.
552 Playlists you created in the Library are left out: they are yours,
553 not a sync's.
554 """
556 auto = [m for m in infos if not m.manual and m.entries]
557 by_name: dict[str, list[M3UInfo]] = defaultdict(list)
558 for info in auto:
559 by_name[name_key(info.name)].append(info)
560 groups: list[PlaylistGroup] = []
561 placed: set[str] = set()
562 for members in by_name.values():
563 if len(members) > 1:
564 groups.append(_group(members, 'name'))
565 placed.update(m.path for m in members)
566 rest = sorted(
567 (m for m in auto if m.path not in placed),
568 key=lambda m: (-len(m.entries), m.path),
569 )
570 for i, first in enumerate(rest):
571 if first.path in placed:
572 continue
573 members = [first]
574 for second in rest[i + 1 :]:
575 if second.path in placed:
576 continue
577 if all(same_songs(m.entries, second.entries) for m in members):
578 members.append(second)
579 if len(members) > 1:
580 groups.append(_group(members, 'songs'))
581 placed.update(m.path for m in members)
582 groups.sort(key=lambda g: g.key)
583 return groups
586def _group(members: list[M3UInfo], reason: str) -> PlaylistGroup:
587 members = sorted(members, key=lambda m: m.path)
588 return PlaylistGroup('m3u:' + members[0].path, members, reason)
591def parse_extinf(extinf: str) -> dict[str, Any]:
592 """``{duration, artist, title}`` from an ``#EXTINF`` line."""
594 text = str(extinf or '').strip()
595 if not text.startswith('#EXTINF'):
596 return {}
597 head, _sep, label = text.partition(',')
598 try:
599 duration = int(head.split(':', 1)[1])
600 except (IndexError, ValueError):
601 duration = -1
602 artist, sep, title = label.partition(' - ')
603 if not sep:
604 artist, title = '', label
605 return {
606 'duration': duration if duration >= 0 else None,
607 'artist': artist.strip(),
608 'title': title.strip(),
609 }
612def merge_m3u_files(
613 members: Sequence[Path],
614 keeper: Path,
615 *,
616 download_dir: Path,
617 slskd_dir: Optional[Path],
618 extra_dirs: Sequence[Path],
619 backup_dir: Path,
620) -> dict[str, Any]:
621 """Merge *members* into *keeper*: every song once, largest first.
623 Every member and its cover are copied to *backup_dir* (keeping their
624 path under the downloads folder) before anything changes. The keeper
625 is written atomically, gets the first cover found if it has none,
626 and the other members and their covers are removed.
627 """
629 entries_by_member = {
630 member: read_m3u_entries(member, download_dir, slskd_dir, extra_dirs)
631 for member in members
632 }
633 order = sorted(members, key=lambda m: -len(entries_by_member[m]))
634 seen: set[str] = set()
635 merged: list[dict[str, Any]] = []
636 for member in order:
637 for stored, extinf in entries_by_member[member]:
638 if stored in seen:
639 continue
640 seen.add(stored)
641 merged.append({'filename': stored, **parse_extinf(extinf)})
643 root = Path(download_dir)
644 for member in members:
645 for path in (member, member.with_suffix('.jpg')):
646 if not path.is_file():
647 continue
648 target = backup_dir / path.relative_to(root)
649 target.parent.mkdir(parents=True, exist_ok=True)
650 shutil.copy2(path, target)
652 content, kept = build_m3u_content(
653 merged,
654 download_dir=root,
655 m3u_dir=keeper.parent,
656 slskd_dir=slskd_dir,
657 extra_dirs=extra_dirs,
658 )
659 keeper.parent.mkdir(parents=True, exist_ok=True)
660 temp = keeper.with_name(keeper.name + '.doctor.tmp')
661 temp.write_text(content, encoding='utf-8')
662 os.replace(temp, keeper)
664 cover = keeper.with_suffix('.jpg')
665 cover_from = ''
666 if not cover.is_file():
667 for member in order:
668 candidate = member.with_suffix('.jpg')
669 if candidate.is_file():
670 shutil.copy2(candidate, cover)
671 cover_from = candidate.relative_to(root).as_posix()
672 break
673 removed: list[str] = []
674 for member in members:
675 if member == keeper:
676 continue
677 for path in (member, member.with_suffix('.jpg')):
678 if path.is_file() and path != cover:
679 path.unlink()
680 removed.append(path.relative_to(root).as_posix())
681 return {
682 'keeper': keeper.relative_to(root).as_posix(),
683 'songs': kept,
684 'removed': removed,
685 'cover_from': cover_from,
686 }
689# ---------------------------------------------------------------------------
690# Navidrome
691# ---------------------------------------------------------------------------
694def navidrome_root(
695 playlists: Iterable[dict[str, Any]], m3u_paths: Collection[str]
696) -> str:
697 """Where Navidrome sees Downtify's downloads folder.
699 Found from imported playlists whose path ends with an M3U Downtify
700 has: ``/music/Road Trip/Road Trip.m3u`` and ``Road Trip/Road
701 Trip.m3u`` put it at ``/music/``. ``''`` when no path says.
702 """
704 votes: dict[str, int] = defaultdict(int)
705 known = set(m3u_paths)
706 for playlist in playlists:
707 path = norm(playlist.get('path'))
708 if not path.startswith('/'):
709 continue
710 for rel in known:
711 if path.endswith('/' + rel):
712 votes[path[: len(path) - len(rel)]] += 1
713 if not votes:
714 return ''
715 return max(votes.items(), key=lambda item: (item[1], len(item[0])))[0]
718def navidrome_playlist_findings(
719 playlists: Sequence[dict[str, Any]],
720 m3us: Sequence[M3UInfo],
721 *,
722 root: str,
723) -> dict[str, Any]:
724 """What a Navidrome client sees wrong in the playlists.
726 * ``duplicates`` - two or more playlists one name folds to
727 (:func:`playlist_key`), with why: an imported M3U *and* Downtify's
728 own playlist (``m3u_and_api``), several imports (``imports``), or
729 several of Downtify's (``api``).
730 * ``orphans`` - imported playlists whose M3U is no longer in
731 Downtify's folder. Navidrome keeps those forever.
732 * ``incomplete`` - imported playlists with fewer songs than their
733 M3U lists; ``outside`` counts the songs in slskd's folder or an
734 extra folder, which Navidrome can't see unless its library covers
735 them.
736 """
738 by_path = {m.path: m for m in m3us}
739 groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
740 for playlist in playlists:
741 groups[playlist_key(str(playlist.get('name') or ''))].append(playlist)
743 duplicates: list[dict[str, Any]] = []
744 for key, members in sorted(groups.items()):
745 if len(members) < 2 or not key:
746 continue
747 imported = [p for p in members if imported_from(p)]
748 ours = [p for p in members if made_by_downtify(p)]
749 if imported and ours:
750 reason = 'm3u_and_api'
751 elif len(imported) > 1:
752 reason = 'imports'
753 elif len(ours) > 1:
754 reason = 'api'
755 else:
756 reason = 'name'
757 duplicates.append({
758 'id': 'nd:' + key,
759 'name': str(members[0].get('name') or ''),
760 'reason': reason,
761 'members': [_nd_row(p, root, by_path) for p in members],
762 })
764 orphans: list[dict[str, Any]] = []
765 incomplete: list[dict[str, Any]] = []
766 for playlist in playlists:
767 source = imported_from(playlist)
768 if not source:
769 continue
770 rel = _relative_to_root(source, root)
771 if rel is None:
772 continue
773 info = by_path.get(rel)
774 if info is None:
775 orphans.append(_nd_row(playlist, root, by_path))
776 continue
777 songs = int(playlist.get('songCount') or 0)
778 if songs < len(info.entries):
779 outside = sum(
780 1
781 for entry in info.entries
782 if entry.startswith(SLSKD_LIBRARY_PREFIX)
783 or entry.startswith('ext/')
784 )
785 incomplete.append({
786 **_nd_row(playlist, root, by_path),
787 'missing': len(info.entries) - songs,
788 'outside': outside,
789 })
790 return {
791 'duplicates': duplicates,
792 'orphans': orphans,
793 'incomplete': incomplete,
794 }
797def _relative_to_root(path: str, root: str) -> Optional[str]:
798 if not root or not path.startswith(root):
799 return None
800 return path[len(root) :]
803def _nd_row(
804 playlist: dict[str, Any], root: str, by_path: dict[str, M3UInfo]
805) -> dict[str, Any]:
806 source = imported_from(playlist)
807 rel = _relative_to_root(source, root) if source else None
808 info = by_path.get(rel or '') if rel else None
809 return {
810 'id': str(playlist.get('id') or ''),
811 'name': str(playlist.get('name') or ''),
812 'songs': int(playlist.get('songCount') or 0),
813 'owner': str(playlist.get('owner') or playlist.get('ownerName') or ''),
814 'kind': 'import'
815 if source
816 else ('downtify' if made_by_downtify(playlist) else 'other'),
817 'm3u': rel or '',
818 'm3u_exists': info is not None,
819 'm3u_songs': len(info.entries) if info else None,
820 }
823def navidrome_song_duplicates(
824 songs: Iterable[dict[str, Any]], *, limit: int = 50
825) -> dict[str, Any]:
826 """Songs Navidrome lists more than once: same artist and title
827 (:func:`downtify.library_catalog.song_key`), lengths within
828 :data:`DURATION_TOLERANCE`. ``slskd`` counts copies whose path is in a
829 folder named ``slskd``."""
831 by_song: dict[str, list[dict[str, Any]]] = defaultdict(list)
832 for song in songs:
833 key = song_key(
834 str(song.get('artist') or ''), str(song.get('title') or '')
835 )
836 if key != '|':
837 by_song[key].append(song)
838 groups: list[dict[str, Any]] = []
839 for rows in by_song.values():
840 if len(rows) < 2:
841 continue
842 rows.sort(key=lambda r: float(r.get('duration') or 0))
843 run: list[dict[str, Any]] = []
844 for row in [*rows, None]:
845 length = float(row.get('duration') or 0) if row else None
846 if run and (
847 row is None
848 or not length
849 or length - float(run[-1].get('duration') or 0)
850 > DURATION_TOLERANCE
851 ):
852 if len(run) > 1:
853 groups.append(_song_group(run))
854 run = []
855 if row is not None and length:
856 run.append(row)
857 groups.sort(key=lambda g: (-len(g['copies']), g['artist'], g['title']))
858 return {
859 'groups': len(groups),
860 'extra_copies': sum(len(g['copies']) - 1 for g in groups),
861 'slskd_copies': sum(g['slskd'] for g in groups),
862 'examples': groups[:limit],
863 }
866def _song_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
867 first = rows[0]
868 paths = [norm(r.get('path')) for r in rows]
869 return {
870 'artist': str(first.get('artist') or ''),
871 'title': str(first.get('title') or ''),
872 'copies': paths,
873 'slskd': sum(1 for p in paths if '/slskd/' in f'/{p}'.casefold()),
874 }
877# ---------------------------------------------------------------------------
878# Monitored playlists: the same song under a new Spotify id
879# ---------------------------------------------------------------------------
882def relinked_tracks(
883 rows: Iterable[dict[str, Any]],
884 playlists: dict[int, str],
885 listing: dict[str, dict[str, Any]],
886 *,
887 limit: int = 50,
888) -> dict[str, Any]:
889 """Watches holding one song under two or more Spotify ids, in two
890 or more files.
892 Spotify replaces a playlist's track with another id now and then (a
893 re-release, a region swap). Downtify remembers a watch's tracks by
894 id, so the replacement looks new and, unless the library already
895 has a file *for that id*, is downloaded again - a duplicate nobody
896 added. *rows* are ``downloaded_tracks`` rows (``playlist_id``,
897 ``track_spotify_id``, ``filename``); *listing* maps a library path
898 to its listing row (artist/title).
899 """
901 by_watch: dict[int, dict[str, set[str]]] = defaultdict(
902 lambda: defaultdict(set)
903 )
904 ids: dict[tuple[int, str], set[str]] = defaultdict(set)
905 for row in rows:
906 name = norm(row.get('filename'))
907 entry = listing.get(name)
908 if not entry:
909 continue
910 key = song_key(
911 str(entry.get('artist') or ''), str(entry.get('title') or '')
912 )
913 if key == '|':
914 continue
915 watch = int(row.get('playlist_id') or 0)
916 by_watch[watch][key].add(name)
917 ids[(watch, key)].add(str(row.get('track_spotify_id') or ''))
918 found: list[dict[str, Any]] = []
919 for watch, songs in by_watch.items():
920 for key, files in songs.items():
921 if len(files) > 1 and len(ids[(watch, key)]) > 1:
922 entry = listing[sorted(files)[0]]
923 found.append({
924 'playlist': playlists.get(watch, ''),
925 'artist': str(entry.get('artist') or ''),
926 'title': str(entry.get('title') or ''),
927 'files': sorted(files),
928 'spotify_ids': sorted(ids[(watch, key)]),
929 })
930 found.sort(key=lambda f: (f['playlist'], f['artist'], f['title']))
931 per_playlist: dict[str, int] = defaultdict(int)
932 for item in found:
933 per_playlist[item['playlist']] += len(item['files']) - 1
934 return {
935 'songs': len(found),
936 'extra_files': sum(len(f['files']) - 1 for f in found),
937 'playlists': dict(sorted(per_playlist.items())),
938 'examples': found[:limit],
939 }