Coverage for downtify/doctor_routes.py: 38%
690 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Doctor API (admins): health check, slskd leftover cleanup, playlist
2and Navidrome repairs. The logic is in :mod:`downtify.doctor`.
4One Doctor job runs at a time, in a thread; the page polls
5``GET /api/doctor`` for its progress. Every repair re-reads what it
6changed afterwards (the files, Navidrome's playlists) and reports what
7it found, rather than assuming it worked.
9The slskd cleanup is the only one that deletes audio, and it is gated
10three times: the download queue is paused (and drained) before the
11scan, only files whose audio is proven identical to a library file are
12offered, and each one is checked again - same size and modification
13time, its library twin still there - right before it is removed.
14"""
16from __future__ import annotations
18import asyncio
19import os
20import secrets
21import threading
22import time
23from collections.abc import Iterable
24from datetime import datetime, timezone
25from pathlib import Path
26from typing import Any, Callable, Optional
28from fastapi import APIRouter, Depends, HTTPException, Request
29from loguru import logger
30from pydantic import BaseModel, Field
32from . import api, m3u, trash_routes
33from .admin_access import admin_user
34from .db.runtime import db_target
35from .doctor import (
36 STATUS_VERIFIED,
37 CopyResult,
38 M3UInfo,
39 PlaylistGroup,
40 m3u_duplicate_groups,
41 merge_m3u_files,
42 navidrome_playlist_findings,
43 navidrome_root,
44 navidrome_song_duplicates,
45 norm,
46 prune_empty_dirs,
47 relinked_tracks,
48 slskd_folder_breakdown,
49 stale_after,
50 summarize_copies,
51 verify_copies,
52 working_copy_rows,
53)
54from .doctor_audio import AudioHashCache
55from .duplicate_merge import (
56 PLAYLIST_PREFIX,
57 KeptApart,
58 MergeError,
59 MergePlan,
60 MergeStores,
61 finish_merge,
62 leftover_references,
63 plan_merge,
64 relink_duplicates,
65)
66from .library_catalog import (
67 AUDIO_EXTENSIONS,
68 drop_library_listing_files,
69 list_library_entries,
70)
71from .library_delete import delete_library_file
72from .library_paths import SLSKD_LIBRARY_PREFIX, locate_library_file
73from .library_paths_cache import invalidate_library_paths_cache
74from .library_trash import TrashError
75from .library_usage import duplicates
76from .navidrome import NavidromeClient, _effective_navidrome_settings
77from .playlist_identity import made_by_downtify, playlist_key
79router = APIRouter(prefix='/api/doctor', dependencies=[Depends(admin_user)])
81#: How long the queue may wait for running downloads before the scan.
82DRAIN_TIMEOUT = 30 * 60
83#: A scan nobody acts on gives the queue back after this.
84PAUSE_EXPIRY = 60 * 60
85#: Examples returned per finding; counts always cover everything.
86EXAMPLES = 50
87#: Duplicate groups the review lists (the counts cover all of them).
88REVIEW_GROUPS = 500
89#: Copies one merge may remove.
90MERGE_MAX = 20
92_lock = threading.Lock()
93_job: dict[str, Any] = {}
94#: The last result of each kind of job, as the page shows it.
95_last: dict[str, Any] = {}
96#: What the Doctor holds between requests: the queue it paused
97#: (``hold``), the last slskd check (``copies``), the Navidrome playlists
98#: the last health check flagged (``flagged``).
99_state: dict[str, Any] = {}
100#: Sections whose findings a repair has changed since they were last
101#: checked (:func:`downtify.doctor.stale_after`). Their repairs wait for
102#: a new check, so nothing acts on what the page showed before.
103_stale: set[str] = set()
104_hashes: Optional[AudioHashCache] = None
105_kept: Optional[KeptApart] = None
108# ---------------------------------------------------------------------------
109# Jobs
110# ---------------------------------------------------------------------------
113def _now() -> str:
114 return datetime.now(timezone.utc).isoformat()
117def _data_dir() -> Path:
118 path = api.state.settings_path
119 return Path(path).parent if path else Path('/data')
122def _hash_cache() -> AudioHashCache:
123 global _hashes
124 if _hashes is None:
125 _hashes = AudioHashCache(db_target('doctor_hashes', _data_dir()))
126 return _hashes
129def _kept_apart() -> KeptApart:
130 global _kept
131 if _kept is None:
132 _kept = KeptApart(db_target('doctor_dupes', _data_dir()))
133 return _kept
136def _busy() -> bool:
137 return _job.get('state') == 'running'
140def _start(kind: str, work: Callable[[dict[str, Any]], Any]) -> dict[str, Any]:
141 """Run *work* in a thread as the Doctor's one job."""
143 with _lock:
144 if _busy():
145 raise HTTPException(409, detail={'code': 'doctor_busy'})
146 job: dict[str, Any] = {
147 'id': secrets.token_urlsafe(8),
148 'kind': kind,
149 'state': 'running',
150 'started_at': _now(),
151 'progress': {'done': 0, 'total': 0, 'step': ''},
152 'cancel': False,
153 }
154 _job.clear()
155 _job.update(job)
157 async def run() -> None:
158 try:
159 result = await work(_job)
160 _job.update(state='done', result=result, finished_at=_now())
161 _last[kind] = {'at': _now(), 'result': result}
162 _mark(kind, done=True)
163 except HTTPException as exc:
164 _job.update(state='failed', error=exc.detail, finished_at=_now())
165 _mark(kind, done=False)
166 except Exception as exc:
167 logger.exception('doctor: {} failed', kind)
168 _job.update(
169 state='failed',
170 error={'code': 'doctor_failed', 'message': str(exc)},
171 finished_at=_now(),
172 )
173 _mark(kind, done=False)
175 asyncio.get_running_loop().create_task(run())
176 return _public_job()
179def _mark(kind: str, *, done: bool) -> None:
180 """Update :data:`_stale` after a *kind* job. A failed check proves
181 nothing; a failed repair may still have changed something."""
183 if done:
184 fresh = stale_after(_stale, kind)
185 else:
186 fresh = _stale | stale_after(set(), kind)
187 _stale.clear()
188 _stale.update(fresh)
191def _step(job: dict[str, Any], step: str, done: int = 0, total: int = 0):
192 job['progress'] = {'done': done, 'total': total, 'step': step}
195def _public_job() -> dict[str, Any]:
196 return {k: v for k, v in _job.items() if k != 'cancel'}
199@router.get('')
200async def doctor_status() -> dict[str, Any]:
201 """The running or last job, the last result of each kind, and
202 whether the Doctor is holding the download queue."""
204 return {
205 'job': _public_job() or None,
206 'last': _last,
207 'queue': api._global_queue_status(),
208 'holding_queue': bool(_state.get('hold')),
209 'stale': sorted(_stale),
210 }
213def _require_fresh(section: str) -> None:
214 """Refuse a repair while *section*'s findings are out of date."""
216 if section in _stale:
217 raise HTTPException(
218 409, detail={'code': 'rescan_first', 'section': section}
219 )
222@router.post('/cancel')
223async def doctor_cancel() -> dict[str, Any]:
224 _job['cancel'] = True
225 return _public_job()
228# ---------------------------------------------------------------------------
229# Inputs
230# ---------------------------------------------------------------------------
233def _ctx():
234 return api.library_context()
237def _organized() -> bool:
238 dl = api.state.downloader
239 return bool(
240 dl is not None
241 and (
242 getattr(dl, 'organize_by_artist', False)
243 or getattr(dl, 'organize_by_album', False)
244 )
245 )
248def _m3u_infos(ctx) -> list[M3UInfo]:
249 root = Path(ctx.download_dir)
250 organize = _organized()
251 infos: list[M3UInfo] = []
252 for path in m3u.iter_m3u_files(root):
253 try:
254 rel = path.relative_to(root).as_posix()
255 except ValueError:
256 continue
257 entries = m3u.read_m3u_entries(
258 path, root, ctx.slskd_dir, ctx.extra_dirs
259 )
260 present = [
261 stored
262 for stored, _extinf in entries
263 if locate_library_file(stored, root, ctx.slskd_dir, ctx.extra_dirs)
264 ]
265 text = m3u.read_m3u_text(path) or ''
266 lines = sum(
267 1
268 for line in text.splitlines()
269 if line.strip() and not line.strip().startswith('#')
270 )
271 name = path.stem
272 expected = m3u.m3u_path_for(
273 root,
274 name,
275 playlist_subdir=None
276 if organize
277 else m3u.sanitize_playlist_name(name),
278 )
279 infos.append(
280 M3UInfo(
281 path=rel,
282 name=name,
283 manual=m3u.is_manual_m3u(path),
284 entries=present,
285 lines=lines,
286 cover=path.with_suffix('.jpg').is_file(),
287 expected=expected == path,
288 )
289 )
290 return infos
293def _referenced(inputs: dict[str, Any], infos: list[M3UInfo]) -> set[str]:
294 """Every library path something still points at: the track index,
295 the provider records, every M3U and every monitored playlist."""
297 referenced = set(inputs['referenced'])
298 for info in infos:
299 referenced.update(info.entries)
300 db = api.state.monitor_db
301 if db is not None:
302 for row in db.list_all_downloaded_tracks():
303 referenced.add(norm(row.get('filename')))
304 referenced.discard('')
305 return referenced
308def _navidrome() -> Optional[tuple[NavidromeClient, dict[str, Any]]]:
309 cfg = _effective_navidrome_settings(api.state.settings)
310 if not cfg.get('enabled'):
311 return None
312 client = NavidromeClient(cfg)
313 if not client.configured():
314 return None
315 return client, cfg
318def _navidrome_playlists(client: NavidromeClient) -> tuple[list[dict], bool]:
319 native = client.native_playlists()
320 if native is not None:
321 return native, True
322 return client.list_playlists(admin=True), False
325def _rescan(
326 client: NavidromeClient, cfg: dict[str, Any], *, full: bool
327) -> bool:
328 try:
329 client.start_scan(full=full)
330 except Exception as exc:
331 logger.info('doctor: navidrome scan failed err={}', exc)
332 return False
333 return client.wait_scan_complete(int(cfg.get('scan_wait_seconds') or 120))
336# ---------------------------------------------------------------------------
337# Health check
338# ---------------------------------------------------------------------------
341@router.post('/scan')
342async def doctor_scan() -> dict[str, Any]:
343 """Look for everything the Doctor knows how to find. Changes nothing."""
345 async def work(job: dict[str, Any]) -> dict[str, Any]:
346 return await asyncio.to_thread(_health, job)
348 return _start('health', work)
351def _health(job: dict[str, Any]) -> dict[str, Any]:
352 ctx = _ctx()
353 _step(job, 'library')
354 inputs = api._usage_library_inputs(ctx)
355 _step(job, 'playlists')
356 infos = _m3u_infos(ctx)
357 referenced = _referenced(inputs, infos)
358 library, leftovers = working_copy_rows(
359 inputs['entries'],
360 sources=inputs['sources'],
361 slskd_in_place=inputs['slskd_in_place'],
362 referenced=referenced,
363 )
364 _step(job, 'duplicates')
365 dupes = duplicates(library, inputs['providers'], limit=None)
366 found_groups = m3u_duplicate_groups(infos)
367 groups = _playlist_review(found_groups)
368 report: dict[str, Any] = {
369 'scanned_at': _now(),
370 'slskd': {
371 'in_place': inputs['slskd_in_place'],
372 'leftovers': len(leftovers),
373 'bytes': sum(int(r.get('size') or 0) for r in leftovers),
374 'folder': _slskd_folder(ctx, library, leftovers),
375 },
376 'duplicates': _duplicate_review(dupes.get('examples', []), infos),
377 'playlists': {
378 'm3u': len(infos),
379 'groups': [g.as_dict() for g in groups],
380 'kept_apart': len(found_groups) - len(groups),
381 'list': [i.as_dict() for i in sorted(infos, key=lambda i: i.path)],
382 },
383 }
384 _step(job, 'monitor')
385 report['relinked'] = _relinked(inputs['entries'])
386 _step(job, 'navidrome')
387 report['navidrome'] = _navidrome_health(infos)
388 _state['flagged'] = {
389 m['id']
390 for g in report['navidrome'].get('duplicates', [])
391 for m in g['members']
392 } | {o['id'] for o in report['navidrome'].get('orphans', [])}
393 return report
396def _slskd_files(root: Path) -> Iterable[tuple[str, int]]:
397 """``(stored path, size)`` of every file under slskd's folder,
398 without following links out of it."""
400 for folder, dirs, names in os.walk(root):
401 dirs[:] = [d for d in dirs if not (Path(folder) / d).is_symlink()]
402 for name in names:
403 path = Path(folder) / name
404 try:
405 if path.is_symlink() or not path.is_file():
406 continue
407 size = path.stat().st_size
408 except OSError:
409 continue
410 rel = path.relative_to(root).as_posix()
411 yield f'{SLSKD_LIBRARY_PREFIX}{rel}', size
414def _slskd_folder(
415 ctx, library: list[dict[str, Any]], leftovers: list[dict[str, Any]]
416) -> Optional[dict[str, Any]]:
417 """What slskd's folder holds next to what Downtify counts (see
418 :func:`downtify.doctor.slskd_folder_breakdown`), or ``None`` without
419 one."""
421 root = Path(ctx.slskd_dir) if ctx.slskd_dir else None
422 if root is None or not root.is_dir():
423 return None
424 return slskd_folder_breakdown(
425 _slskd_files(root),
426 library=[
427 norm(r.get('file'))
428 for r in library
429 if norm(r.get('file')).startswith(SLSKD_LIBRARY_PREFIX)
430 ],
431 leftovers=[norm(r.get('file')) for r in leftovers],
432 audio_extensions=AUDIO_EXTENSIONS,
433 )
436def _duplicate_review(
437 found: list[dict[str, Any]], infos: list[M3UInfo]
438) -> dict[str, Any]:
439 """The duplicate groups to decide on: all but those kept apart, each
440 file with the playlists it is in (from the M3U files)."""
442 kept = _kept_apart()
443 pairs = kept.pairs()
444 review = [
445 g
446 for g in found
447 if not kept.settled([f['file'] for f in g['files']], pairs)
448 ]
449 playlists: dict[str, list[str]] = {}
450 for info in infos:
451 for stored in info.entries:
452 names = playlists.setdefault(norm(stored), [])
453 if info.name not in names:
454 names.append(info.name)
455 for group in review:
456 for item in group['files']:
457 item['playlists'] = sorted(playlists.get(norm(item['file']), []))
458 _state['duplicates'] = {
459 norm(item['file']): index
460 for index, group in enumerate(review)
461 for item in group['files']
462 }
463 return {
464 'totals': _duplicate_totals(review),
465 'groups': review[:REVIEW_GROUPS],
466 'kept_apart': len(found) - len(review),
467 # Where a merge's extra copies go: the music trash, or for good.
468 'trash': api.state.library_trash is not None,
469 }
472def _duplicate_totals(groups: list[dict[str, Any]]) -> dict[str, Any]:
473 return {
474 kind: {
475 'groups': sum(1 for g in groups if g['kind'] == kind),
476 'files': sum(
477 len(g['files']) - 1 for g in groups if g['kind'] == kind
478 ),
479 'bytes': sum(g['bytes'] for g in groups if g['kind'] == kind),
480 }
481 for kind in ('exact', 'likely')
482 }
485def _relinked(entries: list[dict[str, Any]]) -> dict[str, Any]:
486 db = api.state.monitor_db
487 if db is None:
488 return {'songs': 0, 'extra_files': 0, 'playlists': {}, 'examples': []}
489 names = {p.id: p.name for p in db.list_playlists()}
490 listing = {norm(e.get('file')): e for e in entries}
491 return relinked_tracks(
492 db.list_all_downloaded_tracks(), names, listing, limit=EXAMPLES
493 )
496def _navidrome_health(infos: list[M3UInfo]) -> dict[str, Any]:
497 found = _navidrome()
498 if found is None:
499 return {'enabled': False}
500 client, cfg = found
501 if not client.ping():
502 return {'enabled': True, 'reachable': False}
503 playlists, native = _navidrome_playlists(client)
504 root = navidrome_root(playlists, [i.path for i in infos])
505 findings = navidrome_playlist_findings(playlists, infos, root=root)
506 try:
507 songs = client.all_songs()
508 except Exception as exc:
509 logger.info('doctor: navidrome song list failed err={}', exc)
510 songs = []
511 pairs = sum(
512 1 for d in findings['duplicates'] if d['reason'] == 'm3u_and_api'
513 )
514 return {
515 'enabled': True,
516 'reachable': True,
517 'native': native,
518 'admin': bool(cfg.get('admin_username') and cfg.get('admin_password')),
519 'root': root,
520 'm3u_import': bool(cfg.get('m3u_import')),
521 'suggest_m3u_import': pairs > 0 and not cfg.get('m3u_import'),
522 'playlists': len(playlists),
523 'songs': len(songs),
524 'song_duplicates': navidrome_song_duplicates(songs, limit=EXAMPLES),
525 **findings,
526 }
529# ---------------------------------------------------------------------------
530# slskd leftovers
531# ---------------------------------------------------------------------------
534@router.post('/slskd/scan')
535async def slskd_scan() -> dict[str, Any]:
536 """Pause the queue, wait for running downloads, then check every
537 slskd leftover against the library. Deletes nothing."""
539 async def work(job: dict[str, Any]) -> dict[str, Any]:
540 await _hold_queue(job)
541 results = await asyncio.to_thread(_slskd_verify, job)
542 hold = _state.get('hold') or {}
543 hold['token'] = job['id']
544 _state['hold'] = hold
545 _state['copies'] = {r.file: r for r in results}
546 asyncio.get_running_loop().create_task(_expire_hold(job['id']))
547 return {
548 'summary': summarize_copies(results),
549 'checked': len(results),
550 'paused_by_doctor': bool(hold.get('paused_by_doctor')),
551 'expires_in': PAUSE_EXPIRY,
552 }
554 return _start('slskd', work)
557async def _hold_queue(job: dict[str, Any]) -> None:
558 paused_before = api.queue_pause.paused
559 if not paused_before:
560 await api._set_queue_paused(True)
561 _state['hold'] = {'paused_by_doctor': not paused_before}
562 deadline = time.monotonic() + DRAIN_TIMEOUT
563 while True:
564 status = api._global_queue_status()
565 if status.get('state') != 'pausing':
566 return
567 if time.monotonic() > deadline or job.get('cancel'):
568 await _release_queue()
569 raise HTTPException(409, detail={'code': 'queue_busy'})
570 _step(job, 'pausing', 0, int(status.get('active') or 0))
571 await asyncio.sleep(2)
574async def _release_queue() -> None:
575 hold = _state.pop('hold', None) or {}
576 _state.pop('copies', None)
577 if hold.get('paused_by_doctor'):
578 await api._set_queue_paused(False)
581async def _expire_hold(token: str) -> None:
582 """Give the queue back if the scan *token* started is still
583 waiting for a decision after :data:`PAUSE_EXPIRY`."""
585 await asyncio.sleep(PAUSE_EXPIRY)
586 if (_state.get('hold') or {}).get('token') == token and not _busy():
587 logger.info('doctor: slskd cleanup not confirmed; queue resumed')
588 await _release_queue()
591@router.get('/slskd/files')
592async def slskd_files(
593 status: str = '', offset: int = 0, limit: int = 100
594) -> dict[str, Any]:
595 """The last slskd check, a page at a time, biggest first."""
597 copies: dict[str, CopyResult] = _state.get('copies') or {}
598 rows = [r for r in copies.values() if not status or r.status == status]
599 rows.sort(key=lambda r: (-r.bytes, r.file))
600 offset = max(0, offset)
601 limit = min(500, max(1, limit))
602 return {
603 'total': len(rows),
604 'files': [r.as_dict() for r in rows[offset : offset + limit]],
605 }
608def _slskd_verify(job: dict[str, Any]) -> list[CopyResult]:
609 ctx = _ctx()
610 _step(job, 'listing')
611 inputs = api._usage_library_inputs(ctx)
612 infos = _m3u_infos(ctx)
613 library, leftovers = working_copy_rows(
614 inputs['entries'],
615 sources=inputs['sources'],
616 slskd_in_place=inputs['slskd_in_place'],
617 referenced=_referenced(inputs, infos),
618 )
620 def locate(stored: str) -> Optional[Path]:
621 return locate_library_file(
622 stored, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
623 )
625 cache = _hash_cache()
626 return verify_copies(
627 leftovers,
628 library,
629 locate=locate,
630 digest=cache.digest,
631 now=time.time(),
632 progress=lambda done, total: _step(job, 'verifying', done, total),
633 cancelled=lambda: bool(job.get('cancel')),
634 )
637class Cleanup(BaseModel):
638 files: list[str] = Field(default_factory=list, max_length=200_000)
641@router.post('/slskd/clean')
642async def slskd_clean(payload: Cleanup) -> dict[str, Any]:
643 """Delete verified leftovers from the last scan (all of them, or the
644 listed ones), resume the queue, and have Navidrome rescan."""
646 results: dict[str, CopyResult] = _state.get('copies') or {}
647 if not results:
648 raise HTTPException(409, detail={'code': 'scan_first'})
649 _require_fresh('slskd')
650 wanted = set(payload.files) or set(results)
651 chosen = [
652 r
653 for name, r in results.items()
654 if name in wanted and r.status == STATUS_VERIFIED
655 ]
657 async def work(job: dict[str, Any]) -> dict[str, Any]:
658 outcome = await asyncio.to_thread(_slskd_delete, job, chosen)
659 await _release_queue()
660 outcome['queue'] = api._global_queue_status()
661 outcome['navidrome'] = await asyncio.to_thread(_after_slskd_clean, job)
662 return outcome
664 return _start('slskd_clean', work)
667@router.post('/slskd/release')
668async def slskd_release() -> dict[str, Any]:
669 """Leave the leftovers alone and give the queue back."""
671 if _busy() and _job.get('kind') == 'slskd':
672 _job['cancel'] = True
673 await _release_queue()
674 return {'queue': api._global_queue_status()}
677def _slskd_delete(job: dict[str, Any], chosen: list[CopyResult]) -> dict:
678 ctx = _ctx()
679 root = Path(ctx.slskd_dir) if ctx.slskd_dir else None
680 deleted: list[str] = []
681 kept: list[dict[str, str]] = []
682 freed = 0
683 cache = _hash_cache()
684 for done, result in enumerate(chosen):
685 if job.get('cancel'):
686 break
687 _step(job, 'deleting', done, len(chosen))
688 path = locate_library_file(
689 result.file, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
690 )
691 twin = locate_library_file(
692 result.match, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
693 )
694 reason = ''
695 if path is None:
696 reason = 'gone'
697 elif twin is None:
698 reason = 'library_copy_gone'
699 else:
700 try:
701 stat = path.stat()
702 except OSError:
703 reason = 'gone'
704 else:
705 if (
706 stat.st_size != result.bytes
707 or stat.st_mtime_ns != result.mtime_ns
708 ):
709 reason = 'changed'
710 if reason or path is None:
711 kept.append({'file': result.file, 'reason': reason})
712 continue
713 try:
714 path.unlink()
715 except OSError as exc:
716 kept.append({'file': result.file, 'reason': f'error: {exc}'})
717 continue
718 deleted.append(result.file)
719 freed += result.bytes
720 cache.forget([str(path)])
721 if root is not None:
722 prune_empty_dirs(path.parent, root)
723 if deleted:
724 drop_library_listing_files(ctx, deleted)
725 invalidate_library_paths_cache()
726 still_there = [
727 name
728 for name in deleted
729 if locate_library_file(
730 name, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
731 )
732 ]
733 logger.info(
734 'doctor: removed {} slskd leftover(s), {} bytes; kept {}',
735 len(deleted),
736 freed,
737 len(kept),
738 )
739 return {
740 'deleted': len(deleted),
741 'bytes': freed,
742 'kept': kept,
743 'verified_gone': not still_there,
744 }
747def _after_slskd_clean(job: dict[str, Any]) -> dict[str, Any]:
748 found = _navidrome()
749 if found is None:
750 return {'enabled': False}
751 client, cfg = found
752 _step(job, 'navidrome_scan')
753 complete = _rescan(client, cfg, full=False)
754 try:
755 songs = client.all_songs()
756 except Exception:
757 return {'enabled': True, 'scan_complete': complete}
758 return {
759 'enabled': True,
760 'scan_complete': complete,
761 'song_duplicates': navidrome_song_duplicates(songs, limit=0),
762 }
765# ---------------------------------------------------------------------------
766# Playlists
767# ---------------------------------------------------------------------------
770class Merge(BaseModel):
771 paths: list[str] = Field(min_length=2, max_length=20)
774@router.post('/playlists/preview')
775async def playlists_preview(payload: Merge) -> dict[str, Any]:
776 """What merging these M3U files would do. Changes nothing."""
778 group = await asyncio.to_thread(_group_for, payload.paths)
779 return group.as_dict()
782def _playlist_names(paths: list[str]) -> list[str]:
783 return [PLAYLIST_PREFIX + norm(p) for p in paths]
786def _playlist_review(groups: list[PlaylistGroup]) -> list[PlaylistGroup]:
787 """The playlist groups to decide on: all but those kept apart."""
789 kept = _kept_apart()
790 pairs = kept.pairs()
791 return [
792 g
793 for g in groups
794 if not kept.settled(
795 _playlist_names([m.path for m in g.members]), pairs
796 )
797 ]
800def _group_for(
801 paths: list[str], infos: Optional[dict[str, M3UInfo]] = None
802) -> PlaylistGroup:
803 if infos is None:
804 infos = {i.path: i for i in _m3u_infos(_ctx())}
805 members: list[M3UInfo] = []
806 for raw in dict.fromkeys(norm(p) for p in paths):
807 info = infos.get(raw)
808 if info is None:
809 raise HTTPException(
810 404, detail={'code': 'm3u_not_found', 'path': raw}
811 )
812 if info.manual:
813 raise HTTPException(
814 400, detail={'code': 'm3u_manual', 'path': raw}
815 )
816 members.append(info)
817 if len(members) < 2:
818 raise HTTPException(400, detail={'code': 'need_two'})
819 return PlaylistGroup('manual:' + members[0].path, members, 'manual')
822@router.post('/playlists/merge')
823async def playlists_merge(payload: Merge) -> dict[str, Any]:
824 """Merge M3U files of one playlist, then tidy Navidrome to match."""
826 _require_fresh('playlists')
827 group = await asyncio.to_thread(_group_for, payload.paths)
828 if not group.safe():
829 raise HTTPException(
830 400,
831 detail={
832 'code': 'playlists_differ',
833 'overlap': round(group.min_overlap(), 3),
834 },
835 )
837 async def work(job: dict[str, Any]) -> dict[str, Any]:
838 return await asyncio.to_thread(_merge, job, group)
840 return _start('merge', work)
843class PlaylistDecisions(BaseModel):
844 """The duplicate-playlist decisions made on the page, applied at once:
845 groups to merge, and groups that are different playlists."""
847 merge: list[list[str]] = Field(default_factory=list, max_length=500)
848 apart: list[list[str]] = Field(default_factory=list, max_length=500)
851@router.post('/playlists/apply')
852async def playlists_apply(payload: PlaylistDecisions) -> dict[str, Any]:
853 """Keep the *apart* groups apart (they stop being listed), and merge
854 every *merge* group in one job: each is merged as by
855 ``/playlists/merge``, then Navidrome is tidied and rescanned once
856 for all of them, not once per playlist."""
858 _require_fresh('playlists')
859 infos = {i.path: i for i in await asyncio.to_thread(_m3u_infos, _ctx())}
860 groups: list[PlaylistGroup] = []
861 seen: set[str] = set()
862 for paths in payload.merge:
863 group = _group_for(paths, infos)
864 if not group.safe():
865 raise HTTPException(
866 400,
867 detail={
868 'code': 'playlists_differ',
869 'path': group.keeper.path,
870 'overlap': round(group.min_overlap(), 3),
871 },
872 )
873 names = {m.path for m in group.members}
874 if names & seen:
875 raise HTTPException(
876 400,
877 detail={'code': 'm3u_twice', 'path': sorted(names & seen)[0]},
878 )
879 seen |= names
880 groups.append(group)
881 apart = [
882 _playlist_names([m.path for m in _group_for(paths, infos).members])
883 for paths in payload.apart
884 ]
885 kept = 0
886 for names in apart:
887 kept += await asyncio.to_thread(_kept_apart().keep, names)
888 _settle_playlists([m.path for g in groups for m in g.members], apart)
889 if not groups:
890 return {'job': None, 'apart': len(apart), 'pairs': kept}
892 async def work(job: dict[str, Any]) -> dict[str, Any]:
893 result = await asyncio.to_thread(_merge_many, job, groups)
894 result['apart'] = len(apart)
895 return result
897 return {'job': _start('playlists', work), 'apart': len(apart)}
900@router.post('/playlists/apart/reset')
901async def playlists_apart_reset() -> dict[str, Any]:
902 """List every playlist group kept apart again (next health check)."""
904 return {
905 'forgotten': await asyncio.to_thread(
906 lambda: _kept_apart().forget(playlists=True)
907 )
908 }
911def _settle_playlists(merging: list[str], apart: list[list[str]]) -> None:
912 """Take the groups just decided out of the last health check, so the
913 page shows only what is left to decide."""
915 last = (_last.get('health') or {}).get('result') or {}
916 review = last.get('playlists')
917 if not review:
918 return
919 gone = set(merging)
920 apart_sets = [set(names) for names in apart]
921 left = []
922 for group in review.get('groups', []):
923 paths = {m['path'] for m in group['members']}
924 names = set(_playlist_names(sorted(paths)))
925 if paths & gone:
926 continue
927 if any(names <= decided for decided in apart_sets):
928 review['kept_apart'] = review.get('kept_apart', 0) + 1
929 continue
930 left.append(group)
931 review['groups'] = left
934def _merge_many(
935 job: dict[str, Any], groups: list[PlaylistGroup]
936) -> dict[str, Any]:
937 ctx = _ctx()
938 root = Path(ctx.download_dir)
939 stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
940 backup = _data_dir() / 'doctor' / 'backups' / stamp
941 merged: list[dict[str, Any]] = []
942 failed: list[dict[str, str]] = []
943 removed: set[str] = set()
944 for done, group in enumerate(groups):
945 if job.get('cancel'):
946 break
947 _step(job, 'merging', done, len(groups))
948 try:
949 outcome = merge_m3u_files(
950 [root / m.path for m in group.members],
951 root / group.keeper.path,
952 download_dir=root,
953 slskd_dir=ctx.slskd_dir,
954 extra_dirs=ctx.extra_dirs,
955 backup_dir=backup,
956 )
957 except Exception as exc:
958 logger.exception('doctor: merging {} failed', group.keeper.path)
959 failed.append({
960 'keeper': group.keeper.path,
961 'error': str(exc)[:200],
962 })
963 continue
964 removed |= {m.path for m in group.members if m is not group.keeper}
965 merged.append({
966 'keeper': group.keeper.path,
967 'name': group.keeper.name,
968 'songs': int(outcome.get('songs') or 0),
969 })
970 _step(job, 'merging', len(groups), len(groups))
971 after = {i.path for i in _m3u_infos(ctx)}
972 return {
973 'merged': merged,
974 'failed': failed,
975 'backup': str(backup),
976 'others_gone': not (removed & after),
977 'navidrome': _navidrome_after_merges(job, removed),
978 }
981def _navidrome_after_merges(job: dict[str, Any], removed: set[str]) -> dict:
982 """Remove Navidrome's imports of the M3Us a batch of merges removed,
983 then rescan once."""
985 found = _navidrome()
986 if found is None:
987 return {'enabled': False}
988 client, cfg = found
989 infos = _m3u_infos(_ctx())
990 playlists, _native = _navidrome_playlists(client)
991 root = navidrome_root(playlists, [i.path for i in infos] + sorted(removed))
992 deleted: list[str] = []
993 for playlist in playlists:
994 path = norm(playlist.get('path'))
995 if root and path.startswith(root) and path[len(root) :] in removed:
996 try:
997 client.delete_playlist(str(playlist['id']), admin=True)
998 deleted.append(str(playlist['id']))
999 except Exception as exc:
1000 logger.info('doctor: could not delete {} err={}', path, exc)
1001 _step(job, 'navidrome_scan')
1002 return {
1003 'enabled': True,
1004 'deleted': deleted,
1005 'scan_complete': _rescan(client, cfg, full=False),
1006 }
1009def _merge(job: dict[str, Any], group: PlaylistGroup) -> dict[str, Any]:
1010 ctx = _ctx()
1011 root = Path(ctx.download_dir)
1012 stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
1013 backup = _data_dir() / 'doctor' / 'backups' / stamp
1014 _step(job, 'merging')
1015 done = merge_m3u_files(
1016 [root / m.path for m in group.members],
1017 root / group.keeper.path,
1018 download_dir=root,
1019 slskd_dir=ctx.slskd_dir,
1020 extra_dirs=ctx.extra_dirs,
1021 backup_dir=backup,
1022 )
1023 done['backup'] = str(backup)
1024 after = {i.path: i for i in _m3u_infos(ctx)}
1025 keeper = after.get(group.keeper.path)
1026 done['check'] = {
1027 'keeper_songs': len(keeper.entries) if keeper else 0,
1028 'keeper_cover': bool(keeper and keeper.cover),
1029 'others_gone': all(
1030 m.path not in after for m in group.members if m is not group.keeper
1031 ),
1032 }
1033 removed = {m.path for m in group.members if m is not group.keeper}
1034 done['navidrome'] = _navidrome_after_merge(job, group, removed)
1035 return done
1038def _navidrome_after_merge(
1039 job: dict[str, Any], group: PlaylistGroup, removed: set[str]
1040) -> dict[str, Any]:
1041 found = _navidrome()
1042 if found is None:
1043 return {'enabled': False}
1044 client, cfg = found
1045 infos = _m3u_infos(_ctx())
1046 playlists, _native = _navidrome_playlists(client)
1047 root = navidrome_root(playlists, [i.path for i in infos] + sorted(removed))
1048 deleted: list[str] = []
1049 for playlist in playlists:
1050 path = norm(playlist.get('path'))
1051 if root and path.startswith(root) and path[len(root) :] in removed:
1052 try:
1053 client.delete_playlist(str(playlist['id']), admin=True)
1054 deleted.append(str(playlist['id']))
1055 except Exception as exc:
1056 logger.info('doctor: could not delete {} err={}', path, exc)
1057 _step(job, 'navidrome_scan')
1058 complete = _rescan(client, cfg, full=False)
1059 playlists, _native = _navidrome_playlists(client)
1060 key = playlist_key(group.keeper.name)
1061 same = [
1062 p for p in playlists if playlist_key(str(p.get('name') or '')) == key
1063 ]
1064 return {
1065 'enabled': True,
1066 'deleted': deleted,
1067 'scan_complete': complete,
1068 'now': [
1069 {
1070 'id': str(p.get('id') or ''),
1071 'songs': int(p.get('songCount') or 0),
1072 'downtify': made_by_downtify(p),
1073 'm3u': norm(p.get('path'))[len(root) :] if root else '',
1074 }
1075 for p in same
1076 ],
1077 }
1080# ---------------------------------------------------------------------------
1081# Navidrome
1082# ---------------------------------------------------------------------------
1085class NavidromeRepair(BaseModel):
1086 delete: list[str] = Field(default_factory=list, max_length=500)
1087 enable_m3u_import: bool = False
1088 full_scan: bool = False
1091@router.post('/navidrome/repair')
1092async def navidrome_repair(payload: NavidromeRepair) -> dict[str, Any]:
1093 """Delete playlists the last health check flagged (duplicates and
1094 orphans only), optionally switch on M3U import, rescan, re-check."""
1096 _require_fresh('navidrome')
1097 flagged = _state.get('flagged') or set()
1098 unknown = [pid for pid in payload.delete if pid not in flagged]
1099 if unknown:
1100 raise HTTPException(
1101 400, detail={'code': 'not_flagged', 'ids': unknown[:20]}
1102 )
1104 async def work(job: dict[str, Any]) -> dict[str, Any]:
1105 return await asyncio.to_thread(_navidrome_repair, job, payload)
1107 return _start('navidrome', work)
1110def _navidrome_repair(job: dict[str, Any], payload: NavidromeRepair) -> dict:
1111 found = _navidrome()
1112 if found is None:
1113 raise HTTPException(409, detail={'code': 'navidrome_off'})
1114 client, cfg = found
1115 if payload.enable_m3u_import:
1116 settings = api.state.settings
1117 settings.setdefault('navidrome', {})['m3u_import'] = True
1118 if api.state.settings_path is not None:
1119 api._save_settings(api.state.settings_path, settings)
1120 cfg['m3u_import'] = True
1121 deleted: list[str] = []
1122 failed: list[dict[str, str]] = []
1123 for pid in payload.delete:
1124 _step(job, 'deleting', len(deleted), len(payload.delete))
1125 try:
1126 try:
1127 client.delete_playlist(pid)
1128 except Exception:
1129 client.delete_playlist(pid, admin=True)
1130 deleted.append(pid)
1131 except Exception as exc:
1132 failed.append({'id': pid, 'error': str(exc)[:200]})
1133 _step(job, 'navidrome_scan')
1134 complete = _rescan(client, cfg, full=payload.full_scan)
1135 playlists, _native = _navidrome_playlists(client)
1136 present = {str(p.get('id') or '') for p in playlists}
1137 infos = _m3u_infos(_ctx())
1138 root = navidrome_root(playlists, [i.path for i in infos])
1139 findings = navidrome_playlist_findings(playlists, infos, root=root)
1140 return {
1141 'deleted': deleted,
1142 'failed': failed,
1143 'still_listed': [pid for pid in deleted if pid in present],
1144 'scan_complete': complete,
1145 'm3u_import': bool(cfg.get('m3u_import')),
1146 'remaining': {
1147 'duplicates': len(findings['duplicates']),
1148 'orphans': len(findings['orphans']),
1149 'incomplete': len(findings['incomplete']),
1150 },
1151 }
1154# ---------------------------------------------------------------------------
1155# Duplicate songs: merge, keep both
1156# ---------------------------------------------------------------------------
1159class DuplicateMerge(BaseModel):
1160 keep: str = Field(min_length=1, max_length=4096)
1161 remove: list[str] = Field(min_length=1, max_length=MERGE_MAX)
1164class DuplicateFiles(BaseModel):
1165 files: list[str] = Field(min_length=2, max_length=MERGE_MAX + 1)
1168def _same_group(files: list[str]) -> None:
1169 """Only files the last health check grouped together are merged or
1170 kept apart: a stale page can't act on files it never saw grouped."""
1172 groups = _state.get('duplicates')
1173 if groups is None:
1174 raise HTTPException(409, detail={'code': 'dedupe_rescan'})
1175 found = {groups.get(norm(f)) for f in files}
1176 if None in found or len(found) != 1:
1177 raise HTTPException(409, detail={'code': 'dedupe_rescan'})
1180def _settle_review(gone: list[str], kept_apart: list[str]) -> None:
1181 """Take merged files (*gone*) and groups kept apart out of the last
1182 health check's list, so the page doesn't need a new scan."""
1184 last = (_last.get('health') or {}).get('result') or {}
1185 review = last.get('duplicates')
1186 if not review:
1187 return
1188 gone_set = {norm(f) for f in gone}
1189 apart = {norm(f) for f in kept_apart}
1190 groups = []
1191 for group in review['groups']:
1192 names = {norm(f['file']) for f in group['files']}
1193 if apart and names <= apart:
1194 review['kept_apart'] = review.get('kept_apart', 0) + 1
1195 continue
1196 group['files'] = [
1197 f for f in group['files'] if norm(f['file']) not in gone_set
1198 ]
1199 if len(group['files']) > 1:
1200 sizes = [int(f.get('bytes') or 0) for f in group['files']]
1201 group['bytes'] = sum(sizes) - max(sizes)
1202 groups.append(group)
1203 review['groups'] = groups
1204 review['totals'] = _duplicate_totals(groups)
1205 for name in gone_set:
1206 _state.get('duplicates', {}).pop(name, None)
1209@router.post('/duplicates/keep')
1210async def duplicates_keep(payload: DuplicateFiles) -> dict[str, Any]:
1211 """Keep both (or all): the files are different songs. The Doctor
1212 stops listing them until another look-alike turns up."""
1214 _require_fresh('duplicates')
1215 _same_group(payload.files)
1216 added = await asyncio.to_thread(_kept_apart().keep, payload.files)
1217 _settle_review([], payload.files)
1218 return {'pairs': added}
1221@router.post('/duplicates/keep/reset')
1222async def duplicates_keep_reset() -> dict[str, Any]:
1223 """List every group kept apart again (on the next health check)."""
1225 return {'forgotten': await asyncio.to_thread(_kept_apart().forget)}
1228@router.post('/duplicates/merge')
1229async def duplicates_merge(
1230 payload: DuplicateMerge, request: Request
1231) -> dict[str, Any]:
1232 """Keep *keep*, move every reference to the copies in *remove* onto
1233 it, then remove them - to the music trash when it is on."""
1235 return start_dedupe(payload, admin_user(request))
1238def start_dedupe(
1239 payload: DuplicateMerge, user_id: int, *, grouped: bool = True
1240) -> dict[str, Any]:
1241 """Start the merge of *payload* as the Doctor's job.
1243 *grouped* (the page): only copies the last health check put in one
1244 group. The AI helper passes ``False`` for copies it found itself (a
1245 remix kept apart from the original by its length, say), which the
1246 user approved; the merge's own checks still apply.
1247 """
1249 _require_fresh('duplicates')
1250 if grouped:
1251 _same_group([payload.keep, *payload.remove])
1253 async def work(job: dict[str, Any]) -> dict[str, Any]:
1254 return await _merge_duplicates(job, payload, user_id)
1256 return _start('dedupe', work)
1259def _merge_stores(ctx) -> MergeStores:
1260 if api.state.audio_replacements is None:
1261 raise HTTPException(503, detail={'code': 'doctor_not_ready'})
1262 return MergeStores(
1263 ctx=ctx,
1264 replacements=api.state.audio_replacements,
1265 monitor_db=api.state.monitor_db,
1266 library_visibility=api.state.library_visibility,
1267 playlist_hidden=api.state.playlist_hidden,
1268 file_providers=api.state.file_providers,
1269 playlist_catalog=api.state.playlist_catalog,
1270 )
1273async def _merge_duplicates(
1274 job: dict[str, Any], payload: DuplicateMerge, user_id: int
1275) -> dict[str, Any]:
1276 ctx = _ctx()
1277 stores = _merge_stores(ctx)
1278 _step(job, 'dedupe_check')
1279 try:
1280 plan = await asyncio.to_thread(
1281 plan_merge, payload.keep, payload.remove, ctx
1282 )
1283 except MergeError as exc:
1284 raise HTTPException(
1285 409, detail={'code': exc.code, 'file': exc.file}
1286 ) from None
1287 extras = [name for name, _path in plan.extras]
1288 trash = api.state.library_trash
1289 batch = ''
1290 if trash is not None:
1291 # Checked before anything moves: a copy the trash can't take
1292 # (a folder it doesn't cover, a song playing) stops the merge.
1293 try:
1294 tracks = await asyncio.to_thread(
1295 trash_routes._selection, trash_routes.Selection(files=extras)
1296 )
1297 batch = (await asyncio.to_thread(trash.preview, user_id, tracks))[
1298 'confirmation'
1299 ]
1300 except TrashError as exc:
1301 raise HTTPException(
1302 409, detail={'code': 'dedupe_trash', 'reason': str(exc)}
1303 ) from None
1304 _step(job, 'dedupe_relink')
1305 try:
1306 await asyncio.to_thread(relink_duplicates, plan, stores)
1307 except MergeError as exc:
1308 raise HTTPException(
1309 409, detail={'code': exc.code, 'file': exc.file}
1310 ) from None
1311 _step(job, 'dedupe_remove')
1312 removed, failed = await _remove_copies(plan, ctx, user_id, batch)
1313 await asyncio.to_thread(finish_merge, plan, stores, removed)
1314 await asyncio.to_thread(_kept_apart().forget_files, removed)
1315 # A fresh context: *ctx* was built before the copies went to the
1316 # music trash, so its listing would still show them as library songs
1317 # (and the apps' sync list would try to add them back).
1318 await asyncio.to_thread(_publish_library, _ctx())
1319 if plan.playlists:
1320 await api._schedule_playlist_refresh_after_delete(set(plan.playlists))
1321 await api.announce_playlists_changed()
1322 leftovers = await asyncio.to_thread(leftover_references, plan, stores)
1323 _settle_review(removed, [])
1324 return {
1325 'kept': plan.keep,
1326 'removed': removed,
1327 'failed': failed,
1328 'playlists': sorted(plan.playlists),
1329 'trash': bool(batch),
1330 'leftovers': leftovers,
1331 }
1334async def _remove_copies(
1335 plan: MergePlan, ctx, user_id: int, batch: str
1336) -> tuple[list[str], list[dict[str, str]]]:
1337 extras = [name for name, _path in plan.extras]
1338 if batch:
1339 trash = api.state.library_trash
1340 try:
1341 await asyncio.to_thread(trash.confirm, user_id, batch)
1342 except TrashError as exc:
1343 return [], [{'file': f, 'error': str(exc)} for f in extras]
1344 await trash_routes.publish_changes()
1345 trash_routes.wake_worker()
1346 return extras, []
1347 removed, failed = [], []
1348 for name in extras:
1349 result = await asyncio.to_thread(
1350 delete_library_file,
1351 name,
1352 ctx,
1353 cover_cache=api.state.cover_cache,
1354 metadata_cache=api.state.metadata_cache,
1355 playlist_catalog=api.state.playlist_catalog,
1356 track_index=api.state.track_index,
1357 navidrome_index=api.state.navidrome_index,
1358 )
1359 if result.get('deleted'):
1360 removed.append(name)
1361 else:
1362 failed.append({
1363 'file': name,
1364 'error': str(result.get('error') or 'Delete failed'),
1365 })
1366 return removed, failed
1369def _publish_library(ctx) -> None:
1370 """What Replace audio does after publishing: hidden songs, listings
1371 and the mobile apps' library see the change."""
1373 if api.state.library_visibility is not None:
1374 api.state.library_visibility.invalidate()
1375 invalidate_library_paths_cache(drop_entries=True)
1376 if api.state.library_sync is not None:
1377 api.state.library_sync.refresh(list_library_entries(ctx))