Coverage for downtify/library_upgrade.py: 89%
673 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Scan an existing library for tracks worth improving, then improve them.
3A library built up over releases carries whatever each Downtify version
4could do at the time: covers at the smallest size a source offered, no
5lyrics, tags missing an album or a year. Re-downloading everything to fix
6that wastes the files that are already fine, so this module looks at what
7is on disk and repairs only what is actually behind.
9Three categories are handled, all of them writes to tags:
11``artwork``
12 Replace a cover smaller than the target with the largest one Spotify,
13 iTunes or YouTube Music has for the track.
14``lyrics``
15 Fill in lyrics for tracks that have none, through the ordered
16 provider list (see :mod:`downtify.lyrics`). With the
17 ``lyrics_synced_only`` setting, a track counts as having lyrics only
18 when they are time-synced, and only synced lyrics are written.
19``metadata``
20 Re-read the track from Spotify and write back album, album artist,
21 release date and track number.
23Audio replacement is a separate, explicitly selected Deemix mode. It
24prepares verified recordings, publishes playlist/catalog references,
25then optionally removes only unchanged originals with durable receipts.
26The default artwork/lyrics/metadata workflow is unchanged.
28Every write goes to a copy beside the original, which is verified and
29only then moved into place, so an interrupted or failed upgrade leaves
30the existing file untouched.
31"""
33from __future__ import annotations
35import errno
36import os
37import re
38import shutil
39import tempfile
40import threading
41import time
42from dataclasses import dataclass, field, replace
43from pathlib import Path
44from typing import Any, Callable, Optional
46from loguru import logger
47from mutagen import File as MutagenFile
49from . import cover_sources, providers, spotify
50from . import isrc as isrc_mod
51from . import lyrics as lyrics_mod
52from .download_queue import queue_pause
53from .downloader import embed_lyrics, embed_metadata
54from .library_cache_keys import file_content_key
55from .library_catalog import (
56 UPGRADE_STAGING_MARKER,
57 LibraryContext,
58 list_library_entries,
59)
60from .library_paths import locate_library_file
61from .library_upgrade_db import (
62 JOB_DONE,
63 JOB_FAILED,
64 JOB_SKIPPED,
65 STATE_CANCELLED,
66 STATE_DONE,
67 STATE_PAUSED,
68 STATE_READY,
69 STATE_RUNNING,
70 STATE_SCANNING,
71 LibraryUpgradeDB,
72 check_is_fresh,
73 lyrics_outcome,
74)
75from .lyrics import read_track_lyrics
77#: The name an upgrade writes its working copy under. The extension is
78#: kept because the tag writers dispatch on it, and
79#: ``UPGRADE_STAGING_MARKER`` keeps the library listing from picking the
80#: copy up as a track while it exists.
81STAGING_TEMPLATE = '{stem}' + UPGRADE_STAGING_MARKER + '{suffix}'
83CATEGORY_ARTWORK = 'artwork'
84CATEGORY_LYRICS = 'lyrics'
85CATEGORY_METADATA = 'metadata'
86#: Add the recording's ISRC to files without one (:mod:`downtify.isrc`).
87#: Opt-in, like ``audio``: a normal scan never looks for it; a scan
88#: asked for ``isrc`` looks for nothing else.
89CATEGORY_ISRC = 'isrc'
90#: Sent with ``isrc``: look the ISRCs up and list them, write nothing.
91#: A category, so a paused or restarted run stays look-up only.
92ISRC_LOOKUP_ONLY = 'isrc_lookup'
93_ISRC_MODE = frozenset({CATEGORY_ISRC, ISRC_LOOKUP_ONLY})
95#: Every category a scan reports on, in the order the UI shows them.
96CATEGORIES = (CATEGORY_ARTWORK, CATEGORY_LYRICS, CATEGORY_METADATA)
97#: The lyrics check made with Time-synced lyrics only on, kept apart:
98#: a track found to have lyrics before may have only plain ones.
99CHECK_LYRICS_SYNCED = 'lyrics_synced'
102def check_key(category: str, synced_only: bool = False) -> str:
103 """Name a category's check is remembered under."""
105 if synced_only and category == CATEGORY_LYRICS:
106 return CHECK_LYRICS_SYNCED
107 return category
110SUPPORTED_CATEGORIES = (*CATEGORIES, CATEGORY_ISRC, ISRC_LOOKUP_ONLY, 'audio')
112#: How far a rewritten file's duration may drift from the original
113#: before it is rejected. Tag writes don't touch audio, so this only
114#: catches a truncated or corrupted copy.
115_DURATION_TOLERANCE_SECONDS = 2.0
117DEFAULT_ARTWORK_MIN_PX = 1200
118DEFAULT_RECHECK_DAYS = 30
120#: Upper bound on how often progress is pushed to connected clients: a
121#: run is tens of thousands of tracks long and every message costs a
122#: count query plus a broadcast.
123_PUBLISH_INTERVAL_SECONDS = 1.0
125_READ_ONLY_DETAIL = 'Read-only filesystem; original file left unchanged'
128@dataclass(frozen=True)
129class UpgradeOptions:
130 """What an upgrade run should do, and how picky it should be."""
132 categories: tuple[str, ...] = CATEGORIES
133 artwork_min_px: int = DEFAULT_ARTWORK_MIN_PX
134 artwork_source: str = cover_sources.PREFERENCE_HIGHEST
135 recheck_days: int = DEFAULT_RECHECK_DAYS
136 replace_audio_deemix: bool = False
137 delete_replaced_audio: bool = False
138 files: tuple[str, ...] = ()
139 deemix_recordings: dict[str, str] = field(default_factory=dict)
140 #: Start upgrading the chosen categories as soon as the scan ends.
141 start_after_scan: bool = False
143 def as_dict(self) -> dict[str, Any]:
144 result = {
145 'categories': list(self.categories),
146 'artwork_min_px': self.artwork_min_px,
147 'artwork_source': self.artwork_source,
148 'recheck_days': self.recheck_days,
149 }
150 if self.replace_audio_deemix:
151 result['replace_audio_deemix'] = True
152 result['delete_replaced_audio'] = self.delete_replaced_audio
153 if self.deemix_recordings:
154 result['deemix_recordings'] = dict(self.deemix_recordings)
155 if self.files:
156 result['files'] = list(self.files)
157 if self.start_after_scan:
158 result['start_after_scan'] = True
159 return result
162def normalize_categories(raw: Any) -> tuple[str, ...]:
163 """Keep the known categories from *raw*, in the canonical order."""
165 if not isinstance(raw, (list, tuple)):
166 return CATEGORIES
167 wanted = {str(item) for item in raw}
168 if CATEGORY_ISRC in wanted or ISRC_LOOKUP_ONLY in wanted:
169 # ISRCs are a run of their own: nothing else is looked at.
170 return (
171 (CATEGORY_ISRC, ISRC_LOOKUP_ONLY)
172 if (ISRC_LOOKUP_ONLY in wanted)
173 else (CATEGORY_ISRC,)
174 )
175 picked = tuple(name for name in SUPPORTED_CATEGORIES if name in wanted)
176 return picked or CATEGORIES
179def options_from(payload: dict[str, Any]) -> UpgradeOptions:
180 """Build options from a request body, clamped to sane values."""
182 def _int(key: str, default: int, low: int, high: int) -> int:
183 try:
184 value = int(payload.get(key, default))
185 except (TypeError, ValueError):
186 value = default
187 return min(high, max(low, value))
189 raw_recordings = payload.get('deemix_recordings')
190 recordings = {}
191 if isinstance(raw_recordings, dict):
192 for stored, ident in list(raw_recordings.items())[:1000]:
193 if (
194 isinstance(stored, str)
195 and not stored.startswith(('/', 'ext/', 'slskd/'))
196 and '..' not in stored.replace('\\', '/').split('/')
197 and re.fullmatch(r'[1-9][0-9]{0,19}', str(ident))
198 ):
199 recordings[stored] = str(ident)
200 source = str(payload.get('artwork_source') or '')
201 if source not in cover_sources.ARTWORK_SOURCES:
202 source = cover_sources.PREFERENCE_HIGHEST
203 return UpgradeOptions(
204 categories=normalize_categories(payload.get('categories')),
205 artwork_min_px=_int(
206 'artwork_min_px', DEFAULT_ARTWORK_MIN_PX, 100, 3000
207 ),
208 artwork_source=source,
209 recheck_days=_int('recheck_days', DEFAULT_RECHECK_DAYS, 0, 3650),
210 replace_audio_deemix=payload.get('replace_audio_deemix') is True,
211 delete_replaced_audio=payload.get('delete_replaced_audio') is True,
212 deemix_recordings=recordings,
213 files=tuple(str(f) for f in payload.get('files', [])[:1000])
214 if isinstance(payload.get('files'), list)
215 else (),
216 start_after_scan=payload.get('start_after_scan') is True,
217 )
220# ── Detection ─────────────────────────────────────────────────────────
221def _metadata_gaps(entry: dict[str, Any]) -> list[str]:
222 """Tag fields a downloaded track should have but doesn't."""
224 gaps: list[str] = []
225 if not str(entry.get('album') or '').strip():
226 gaps.append('album')
227 if not str(entry.get('album_artist') or '').strip():
228 gaps.append('album_artist')
229 if not str(entry.get('year') or '').strip():
230 gaps.append('year')
231 if not int(entry.get('track_number') or 0):
232 gaps.append('track_number')
233 return gaps
236def _summarize(
237 jobs: list[dict[str, Any]],
238 recently_checked: int = 0,
239 plain_lyrics_only: int = 0,
240) -> dict[str, Any]:
241 """How many tracks (and bytes) each category accounts for."""
243 counts = {name: 0 for name in CATEGORIES}
244 sizes = {name: 0 for name in CATEGORIES}
245 for extra in ('audio', CATEGORY_ISRC):
246 if any(extra in job.get('categories', []) for job in jobs):
247 counts[extra] = 0
248 sizes[extra] = 0
249 for job in jobs:
250 for name in job.get('categories') or []:
251 if name in counts:
252 counts[name] += 1
253 sizes[name] += int(job.get('size') or 0)
254 return {
255 'categories': counts,
256 'category_bytes': sizes,
257 'tracks': len(jobs),
258 # Tracks the scan didn't even read, because every category had
259 # been looked at recently enough.
260 'recently_checked': int(recently_checked),
261 # Of the tracks missing lyrics, those that only have plain
262 # (unsynced) ones, with Time-synced lyrics only on.
263 'plain_lyrics_only': int(plain_lyrics_only),
264 }
267def track_findings(
268 entry: dict[str, Any],
269 full_path: Path,
270 *,
271 artwork_min_px: int,
272 synced_only: bool = False,
273) -> list[str]:
274 """Which categories this track is behind on."""
276 found: list[str] = []
277 if int(entry.get('cover_px') or 0) < artwork_min_px:
278 found.append(CATEGORY_ARTWORK)
279 if not has_lyrics(full_path, synced_only=synced_only):
280 found.append(CATEGORY_LYRICS)
281 if _metadata_gaps(entry):
282 found.append(CATEGORY_METADATA)
283 return found
286def has_lyrics(full_path: Path, *, synced_only: bool = False) -> bool:
287 """Whether a track has lyrics: any, or with *synced_only* time-synced
288 ones (a ``.lrc`` sidecar or synced lyrics in its tags)."""
290 if synced_only:
291 return lyrics_mod.has_synced_lyrics(full_path)
292 existing = read_track_lyrics(full_path)
293 return bool(existing.get('synced') or existing.get('plain'))
296# ── The song a track describes ────────────────────────────────────────
297def _song_from_tags(entry: dict[str, Any]) -> dict[str, Any]:
298 artist = str(entry.get('artist') or '')
299 artists = [part.strip() for part in artist.split(';') if part.strip()]
300 return {
301 'name': str(entry.get('title') or ''),
302 'artists': artists or ([artist] if artist else []),
303 'artist': artist,
304 'album_name': str(entry.get('album') or ''),
305 'album_artist': str(entry.get('album_artist') or ''),
306 'year': str(entry.get('year') or ''),
307 'track_number': int(entry.get('track_number') or 0),
308 'duration': float(entry.get('duration') or 0.0),
309 # A file already marked explicit stays so when its tags are
310 # written back (the mark is only ever added, never cleared).
311 'explicit': bool(entry.get('explicit')),
312 }
315def _merge_remote(
316 local: dict[str, Any], remote: dict[str, Any]
317) -> dict[str, Any]:
318 """Remote metadata wins, but never blanks something the file has."""
320 merged = dict(local)
321 for key, value in remote.items():
322 if value in (None, '', [], 0):
323 continue
324 merged[key] = value
325 if not merged.get('name'):
326 merged['name'] = local.get('name', '')
327 return merged
330def song_for_track(
331 entry: dict[str, Any],
332 *,
333 spotify_id: str = '',
334 refresh_metadata: bool = False,
335) -> dict[str, Any]:
336 """The track as Downtify would describe it when tagging it today.
338 Without ``refresh_metadata`` this is only what the file's own tags
339 say, which is what an artwork- or lyrics-only upgrade writes back:
340 nothing else about the file then changes.
341 """
343 song = _song_from_tags(entry)
344 if spotify_id:
345 song['song_id'] = spotify_id
346 if not (refresh_metadata and spotify_id):
347 return song
348 try:
349 # The album lookup costs two more requests; only a file with no
350 # album tag of its own needs it (single-track downloads before the
351 # fix that made them fetch it).
352 remote = spotify.track_from_id(
353 spotify_id, with_album=not song.get('album_name')
354 )
355 except Exception:
356 logger.opt(exception=True).debug(
357 'Library upgrade: Spotify lookup failed for {}', spotify_id
358 )
359 return song
360 return _merge_remote(song, remote or {})
363# ── Safe file replacement ─────────────────────────────────────────────
364def _audio_length(path: Path) -> float:
365 try:
366 audio = MutagenFile(str(path))
367 except Exception:
368 return 0.0
369 info = getattr(audio, 'info', None)
370 return float(getattr(info, 'length', 0.0) or 0.0)
373def _verify_replacement(staged: Path, expected_seconds: float) -> bool:
374 """A rewritten file must still be readable audio of the same length."""
376 try:
377 if staged.stat().st_size <= 0:
378 return False
379 except OSError:
380 return False
381 length = _audio_length(staged)
382 if length <= 0:
383 return False
384 if expected_seconds <= 0:
385 return True
386 return abs(length - expected_seconds) <= _DURATION_TOLERANCE_SECONDS
389@dataclass
390class _Staged:
391 original: Path
392 staged: Path
393 mtime_ns: int
394 atime_ns: int
395 duration: float
398def staging_path(path: Path) -> Path:
399 return path.with_name(
400 STAGING_TEMPLATE.format(stem=path.stem, suffix=path.suffix)
401 )
404def _stage(path: Path) -> _Staged:
405 staged = staging_path(path)
406 shutil.copy2(str(path), str(staged))
407 st = path.stat()
408 return _Staged(
409 original=path,
410 staged=staged,
411 mtime_ns=int(st.st_mtime_ns),
412 atime_ns=int(st.st_atime_ns),
413 duration=_audio_length(path),
414 )
417def _commit(stage: _Staged) -> bool:
418 """Swap the working copy in, or throw it away if it doesn't verify."""
420 if not _verify_replacement(stage.staged, stage.duration):
421 logger.warning(
422 'Library upgrade: staged copy of {} failed verification',
423 stage.original.name,
424 )
425 stage.staged.unlink(missing_ok=True)
426 return False
427 os.replace(str(stage.staged), str(stage.original))
428 # Keep the original timestamp: the Library sorts "recently added" by
429 # it, and an upgrade is not a new download.
430 try:
431 os.utime(str(stage.original), ns=(stage.atime_ns, stage.mtime_ns))
432 except OSError:
433 pass
434 return True
437def _discard(stage: _Staged) -> None:
438 stage.staged.unlink(missing_ok=True)
441# ── Applying one track ────────────────────────────────────────────────
442@dataclass
443class UpgradeOutcome:
444 status: str = JOB_SKIPPED
445 detail: str = ''
446 changed: list[str] = field(default_factory=list)
447 artwork_px: int = 0
448 artwork_source: str = ''
449 checked: bool = True
452def _read_only_filesystem(path: Path) -> bool:
453 """Inspect mount flags, including when the process runs as root.
455 Windows has no statvfs; staging still reports any write failure.
456 Do not use permission bits to turn unexpected access errors into skips.
457 """
459 statvfs = getattr(os, 'statvfs', None)
460 if statvfs is None:
461 return False
462 read_only = getattr(os, 'ST_RDONLY', 1)
463 return any(
464 statvfs(target).f_flag & read_only for target in (path, path.parent)
465 )
468@dataclass
469class UpgradeDeps:
470 """Everything an upgrade needs from the running app."""
472 context: Callable[[], LibraryContext]
473 settings: Callable[[], dict[str, Any]]
474 version: str = '0.0.0'
475 spotify_id_for: Optional[Callable[[str], str]] = None
476 lyrics_cache: Any = None
477 publish: Optional[Callable[[dict[str, Any]], None]] = None
478 rekey_owners: Optional[Callable[[str, str, str], None]] = None
479 audio_replacement: Any = None
480 # Files whose audio an admin replaced by hand: their audio is kept.
481 hand_replaced: Any = None
482 # Each file's upload (downtify/download_sources.py): a file swapped on
483 # disk since Downtify wrote it is kept like one replaced by hand.
484 download_sources: Any = None
487def resolve_track(stored_path: str, ctx: LibraryContext) -> Optional[Path]:
488 return locate_library_file(
489 stored_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs
490 )
493@dataclass
494class _Artwork:
495 """The cover an upgrade decided on, and where it came from."""
497 data: Optional[bytes] = None
498 px: int = 0
499 source: str = ''
502def _library_entry(
503 stored_path: str, ctx: LibraryContext
504) -> Optional[tuple[Path, dict[str, Any]]]:
505 full = resolve_track(stored_path, ctx)
506 if full is None or ctx.metadata_cache is None:
507 return None
508 entries = ctx.metadata_cache.get_entries_batch([(stored_path, full)])
509 return (full, entries[0]) if entries else None
512def _pick_artwork(
513 song: dict[str, Any],
514 full: Path,
515 entry: dict[str, Any],
516 *,
517 options: UpgradeOptions,
518) -> _Artwork:
519 """A bigger cover than the file has, or the file's own size."""
521 have = int(entry.get('cover_px') or 0)
522 if CATEGORY_ARTWORK not in options.categories:
523 return _Artwork(px=have)
524 if have >= options.artwork_min_px:
525 return _Artwork(px=have)
526 better = cover_sources.best_cover(
527 song,
528 current=cover_sources.current_cover(full),
529 preference=options.artwork_source,
530 # Asked for at least the size being upgraded to.
531 px=max(options.artwork_min_px, providers.cover_resolution()),
532 )
533 if better is None:
534 return _Artwork(px=have)
535 return _Artwork(data=better.data, px=better.width, source=better.source)
538def _pick_lyrics(
539 song: dict[str, Any],
540 full: Path,
541 *,
542 deps: UpgradeDeps,
543 options: UpgradeOptions,
544) -> Optional[lyrics_mod.Lyrics]:
545 """Lyrics for a track that has none, or ``None`` to leave it be."""
547 if CATEGORY_LYRICS not in options.categories:
548 return None
549 settings = deps.settings()
550 synced_only = bool(settings.get('lyrics_synced_only'))
551 if has_lyrics(full, synced_only=synced_only):
552 return None
553 providers = list(settings.get('lyrics_providers') or [])
554 if not providers:
555 return None
556 found = lyrics_mod.fetch(
557 song, providers, deps.lyrics_cache, synced_only=synced_only
558 )
559 if found is None or not found.usable(synced_only):
560 return None
561 return found
564def _publish_staged_lrc(source: Path, destination: Path) -> None:
565 """Publish on the destination filesystem, keeping source until verified.
567 Audio and centralized lyrics may be different Docker bind mounts;
568 os.replace(source, destination) would then raise EXDEV.
569 """
570 destination.parent.mkdir(parents=True, exist_ok=True)
571 temporary: Optional[Path] = None
572 try:
573 with tempfile.NamedTemporaryFile(
574 dir=destination.parent,
575 prefix='.downtify-upgrade-lrc-',
576 suffix='.tmp',
577 delete=False,
578 ) as output:
579 temporary = Path(output.name)
580 with source.open('rb') as incoming:
581 shutil.copyfileobj(incoming, output)
582 shutil.copymode(source, temporary)
583 output.flush()
584 os.fsync(output.fileno())
585 if temporary.read_bytes() != source.read_bytes():
586 raise OSError('Staged lyrics copy failed verification')
587 os.replace(str(temporary), str(destination))
588 source.unlink()
589 finally:
590 if temporary is not None:
591 temporary.unlink(missing_ok=True)
594def _write_upgrade(
595 full: Path,
596 song: dict[str, Any],
597 *,
598 artwork: _Artwork,
599 new_lyrics: Optional[lyrics_mod.Lyrics],
600 write_tags: bool,
601 embed_synced: bool = False,
602) -> Optional[str]:
603 """Rewrite *full* through a verified working copy; error text or ``None``."""
605 try:
606 stage = _stage(full)
607 except OSError as exc:
608 if exc.errno == errno.EROFS:
609 return _READ_ONLY_DETAIL
610 return f'Could not stage a copy: {exc}'
612 try:
613 if write_tags:
614 # The existing cover is written back when nothing better was
615 # found, so a metadata-only pass never strips artwork.
616 keep = cover_sources.current_cover(full)
617 embed_metadata(
618 stage.staged,
619 song,
620 download_cover=True,
621 cover_bytes=artwork.data
622 or (keep.data if keep is not None else None),
623 )
624 if new_lyrics is not None:
625 embed_lyrics(
626 stage.staged,
627 new_lyrics,
628 sidecar=stage.staged.with_suffix('.lrc'),
629 embed_synced=embed_synced,
630 )
631 except Exception as exc:
632 _discard(stage)
633 logger.opt(exception=True).warning(
634 'Library upgrade failed for {}', full.name
635 )
636 return str(exc)[:200]
638 staged_lrc = stage.staged.with_suffix('.lrc')
639 if not _commit(stage):
640 staged_lrc.unlink(missing_ok=True)
641 return 'The rewritten file did not verify; kept the original'
642 if new_lyrics is not None and staged_lrc.exists():
643 dest_lrc = lyrics_mod.lrc_path_for(full)
644 dest_lrc.parent.mkdir(parents=True, exist_ok=True)
645 _publish_staged_lrc(staged_lrc, dest_lrc)
646 beside = full.with_suffix('.lrc')
647 if dest_lrc != beside:
648 beside.unlink(missing_ok=True)
649 return None
652def _write_isrc(full: Path, code: str) -> Optional[str]:
653 """Add *code* as *full*'s ISRC through a verified working copy (only
654 that tag changes); error text or ``None``."""
656 try:
657 stage = _stage(full)
658 except OSError as exc:
659 if exc.errno == errno.EROFS:
660 return _READ_ONLY_DETAIL
661 return f'Could not stage a copy: {exc}'
662 if not isrc_mod.write(stage.staged, code):
663 _discard(stage)
664 return 'Could not write the ISRC tag'
665 if not _commit(stage):
666 return 'The rewritten file did not verify; kept the original'
667 return None
670def _add_isrc(
671 stored_path: str,
672 full: Path,
673 song: dict[str, Any],
674 *,
675 deps: 'UpgradeDeps',
676 ctx: LibraryContext,
677 lookup_only: bool,
678 stage_now: Callable[[str], None],
679) -> UpgradeOutcome:
680 """The ISRC run for one track: confident or nothing.
682 The same rules as for a new download (:func:`downtify.isrc.for_song`:
683 same first artist and title, length within 3 s, every match agreeing).
684 An unmatched file is left exactly as it is. *lookup_only* writes
685 nothing and doesn't mark the track as checked, so the real run looks
686 it up again.
687 """
689 if isrc_mod.read(full):
690 return UpgradeOutcome(JOB_SKIPPED, 'Already has an ISRC')
691 stage_now('isrc')
692 code = isrc_mod.lookup(song) or ''
693 if lookup_only:
694 if code:
695 return UpgradeOutcome(
696 JOB_DONE, f'Would add ISRC {code}', checked=False
697 )
698 return UpgradeOutcome(
699 JOB_SKIPPED,
700 'No ISRC found for sure; would be left as it is',
701 checked=False,
702 )
703 if not code:
704 return UpgradeOutcome(
705 JOB_SKIPPED, 'No ISRC found for sure; left as it is'
706 )
707 stage_now('writing')
708 try:
709 old_key = file_content_key(full) or ''
710 except Exception:
711 old_key = ''
712 error = _write_isrc(full, code)
713 if error == _READ_ONLY_DETAIL:
714 return UpgradeOutcome(JOB_SKIPPED, error, checked=False)
715 if error:
716 return UpgradeOutcome(JOB_FAILED, error)
717 # A tag changes the file's size, so its content key moves: owners,
718 # likes and the rest follow it, as after any other upgrade.
719 if old_key and deps.rekey_owners is not None:
720 try:
721 new_key = file_content_key(full) or ''
722 if new_key:
723 deps.rekey_owners(old_key, new_key, stored_path)
724 except Exception:
725 logger.opt(exception=True).debug(
726 'Library upgrade: could not rekey owners for {}',
727 stored_path,
728 )
729 if ctx.metadata_cache is not None:
730 try:
731 ctx.metadata_cache.refresh(stored_path, full)
732 except Exception:
733 logger.debug('Library upgrade: metadata cache refresh failed')
734 return UpgradeOutcome(JOB_DONE, f'ISRC {code}', changed=[CATEGORY_ISRC])
737def _outcome_detail(changed: list[str], artwork: _Artwork) -> str:
738 parts: list[str] = []
739 for name in changed:
740 if name == CATEGORY_ARTWORK and artwork.px:
741 parts.append(f'artwork {artwork.px}px ({artwork.source})')
742 else:
743 parts.append(name)
744 return ', '.join(parts)
747def upgrade_track(
748 stored_path: str,
749 *,
750 deps: UpgradeDeps,
751 options: UpgradeOptions,
752 on_stage: Optional[Callable[[str], None]] = None,
753) -> UpgradeOutcome:
754 """Apply the requested categories to one library track."""
756 def stage_now(name: str) -> None:
757 if on_stage:
758 on_stage(name)
760 ctx = deps.context()
761 found = _library_entry(stored_path, ctx)
762 if found is None:
763 return UpgradeOutcome(JOB_SKIPPED, 'File is no longer in the library')
764 full, entry = found
765 if _read_only_filesystem(full):
766 return UpgradeOutcome(JOB_SKIPPED, _READ_ONLY_DETAIL, checked=False)
768 spotify_id = (
769 deps.spotify_id_for(stored_path) or ''
770 if deps.spotify_id_for is not None
771 else ''
772 )
773 stage_now('matching')
774 song = song_for_track(
775 entry,
776 spotify_id=spotify_id,
777 refresh_metadata=CATEGORY_METADATA in options.categories,
778 )
779 if CATEGORY_ISRC in options.categories:
780 return _add_isrc(
781 stored_path,
782 full,
783 song,
784 deps=deps,
785 ctx=ctx,
786 lookup_only=ISRC_LOOKUP_ONLY in options.categories,
787 stage_now=stage_now,
788 )
790 stage_now('artwork')
791 artwork = _pick_artwork(song, full, entry, options=options)
792 stage_now('lyrics')
793 new_lyrics = _pick_lyrics(song, full, deps=deps, options=options)
795 changed: list[str] = []
796 if artwork.data is not None:
797 changed.append(CATEGORY_ARTWORK)
798 if (
799 CATEGORY_METADATA in options.categories
800 and spotify_id
801 and _metadata_gaps(entry)
802 ):
803 changed.append(CATEGORY_METADATA)
804 if not changed and new_lyrics is None:
805 return UpgradeOutcome(
806 JOB_SKIPPED,
807 'Already up to date',
808 artwork_px=artwork.px,
809 artwork_source=artwork.source,
810 )
812 stage_now('writing')
813 try:
814 old_key = file_content_key(full) or ''
815 except Exception:
816 old_key = ''
817 error = _write_upgrade(
818 full,
819 song,
820 artwork=artwork,
821 new_lyrics=new_lyrics,
822 write_tags=bool(changed),
823 embed_synced=bool(deps.settings().get('lyrics_synced_only')),
824 )
825 if error == _READ_ONLY_DETAIL:
826 return UpgradeOutcome(JOB_SKIPPED, error, checked=False)
827 if error:
828 return UpgradeOutcome(JOB_FAILED, error)
829 if old_key and deps.rekey_owners is not None:
830 try:
831 new_key = file_content_key(full) or ''
832 if new_key:
833 deps.rekey_owners(old_key, new_key, stored_path)
834 except Exception:
835 logger.opt(exception=True).debug(
836 'Library upgrade: could not rekey owners for {}',
837 stored_path,
838 )
840 if new_lyrics is not None:
841 changed.append(CATEGORY_LYRICS)
842 if ctx.metadata_cache is not None:
843 try:
844 ctx.metadata_cache.refresh(stored_path, full)
845 except Exception:
846 logger.debug('Library upgrade: metadata cache refresh failed')
848 return UpgradeOutcome(
849 JOB_DONE,
850 _outcome_detail(changed, artwork),
851 changed=changed,
852 artwork_px=artwork.px,
853 artwork_source=artwork.source,
854 )
857# ── Scan + queue runner ───────────────────────────────────────────────
858class LibraryUpgradeRunner:
859 """Owns the scan and the worker thread that drains the queue."""
861 def __init__(self, db: LibraryUpgradeDB, deps: UpgradeDeps) -> None:
862 self.db = db
863 self.deps = deps
864 self._lock = threading.Lock()
865 self._worker: Optional[threading.Thread] = None
866 self._stop = threading.Event()
867 self._cancelled = False
868 self._scanned = 0
869 self._scan_total = 0
870 self._last_publish = 0.0
871 self._finalization_error = ''
873 # -- helpers --
874 def _publish(
875 self,
876 extra: Optional[dict[str, Any]] = None,
877 *,
878 throttle: bool = False,
879 ) -> None:
880 """Broadcast progress, at most once a second while grinding."""
882 if self.deps.publish is None:
883 return
884 now = time.monotonic()
885 if throttle and now - self._last_publish < _PUBLISH_INTERVAL_SECONDS:
886 return
887 self._last_publish = now
888 try:
889 self.deps.publish(self.status(extra))
890 except Exception:
891 logger.debug('Library upgrade: progress broadcast failed')
893 def status(self, extra: Optional[dict[str, Any]] = None) -> dict[str, Any]:
894 run = self.db.latest_run()
895 if run is None:
896 return {'state': 'idle', 'run': None, 'counts': {}, 'scan': {}}
897 counts = self.db.counts(int(run['id']))
898 payload: dict[str, Any] = {
899 'state': run['state'],
900 'run': run,
901 'counts': counts,
902 # How the lyrics part went: added, none found, failed.
903 'lyrics': lyrics_outcome(self.db, int(run['id'])),
904 # Carried by every progress message, not only the initial
905 # GET, so a client that only ever sees WebSocket updates
906 # still knows what the scan found.
907 'summary': self._summary_for(run),
908 'scan': {
909 'scanned': self._scanned,
910 'total': self._scan_total,
911 },
912 }
913 if extra:
914 payload.update(extra)
915 if self._finalization_error:
916 payload['finalization_error'] = self._finalization_error
917 if self.deps.audio_replacement is not None:
918 payload['replacements'] = (
919 self.deps.audio_replacement.store.summary(int(run['id']))
920 )
921 return payload
923 def busy(self) -> bool:
924 worker = self._worker
925 return worker is not None and worker.is_alive()
927 # -- scanning --
928 def start_scan(self, options: UpgradeOptions) -> dict[str, Any]:
929 with self._lock:
930 if self.busy():
931 raise RuntimeError('A library upgrade is already running')
932 if (
933 options.replace_audio_deemix
934 and self.deps.audio_replacement is None
935 ):
936 raise RuntimeError('Deemix replacement is unavailable')
937 run_id = self.db.create_run(list(CATEGORIES), options.as_dict())
938 self._cancelled = False
939 self._stop.clear()
940 self._scanned = 0
941 self._scan_total = 0
942 self._worker = threading.Thread(
943 target=self._scan,
944 args=(run_id, options),
945 name='downtify-upgrade-scan',
946 daemon=True,
947 )
948 self._worker.start()
949 return self.status()
951 def _scan(self, run_id: int, options: UpgradeOptions) -> None:
952 try:
953 ctx = self.deps.context()
954 entries = (
955 list_library_entries(ctx, fold_extra=False)
956 if options.replace_audio_deemix
957 else list_library_entries(ctx)
958 )
959 if options.replace_audio_deemix:
960 entries = [
961 e
962 for e in entries
963 if not str(e['file']).startswith(('ext/', 'slskd/'))
964 ]
965 if options.files:
966 wanted_files = set(options.files)
967 entries = [e for e in entries if e['file'] in wanted_files]
968 self._scan_total = len(entries)
969 synced_only = bool(self.deps.settings().get('lyrics_synced_only'))
970 checks = self.db.checks_for([str(e['file']) for e in entries])
971 jobs: list[dict[str, Any]] = []
972 checked: list[tuple[str, list[str]]] = []
973 total_bytes = 0
974 # Tracks not even read (all checked recently), and tracks
975 # missing lyrics that only have plain ones.
976 tally = {'skipped': 0, 'plain_only': 0}
977 for index, entry in enumerate(entries, start=1):
978 if self._stop.is_set():
979 self.db.set_state(run_id, STATE_CANCELLED)
980 self._publish()
981 return
982 stored = str(entry['file'])
983 total_bytes += int(entry.get('size') or 0)
984 self._scanned = index
985 self._publish(throttle=True)
986 if options.replace_audio_deemix:
987 jobs.append({
988 'file': stored,
989 'categories': ['audio'],
990 'size': int(entry.get('size') or 0),
991 'title': str(entry.get('title') or ''),
992 'artist': str(entry.get('artist') or ''),
993 })
994 continue
995 if CATEGORY_ISRC in options.categories:
996 jobs.extend(self._isrc_jobs(entry, checks, options, tally))
997 continue
998 due = self._categories_due(
999 checks.get(stored) or {},
1000 options.recheck_days,
1001 synced_only=synced_only,
1002 )
1003 if not due:
1004 # Every category was looked at recently: don't even
1005 # read the file.
1006 tally['skipped'] += 1
1007 continue
1008 full = resolve_track(stored, ctx)
1009 if full is None:
1010 continue
1011 behind = set(
1012 track_findings(
1013 entry,
1014 full,
1015 artwork_min_px=options.artwork_min_px,
1016 synced_only=synced_only,
1017 )
1018 )
1019 found = [name for name in CATEGORIES if name in due & behind]
1020 # Looking and finding nothing counts as a check, so the
1021 # next scan doesn't read this file again for those.
1022 clean = sorted(due - behind)
1023 if clean:
1024 checked.append((
1025 stored,
1026 [check_key(n, synced_only) for n in clean],
1027 ))
1028 if (
1029 synced_only
1030 and CATEGORY_LYRICS in found
1031 and read_track_lyrics(full, raw=True).get('plain')
1032 ):
1033 tally['plain_only'] += 1
1034 if found:
1035 jobs.append({
1036 'file': stored,
1037 'categories': found,
1038 'size': int(entry.get('size') or 0),
1039 'title': str(entry.get('title') or ''),
1040 'artist': str(entry.get('artist') or ''),
1041 })
1042 self.db.record_checks(checked, app_version=self.deps.version)
1043 self.db.add_jobs(run_id, jobs)
1044 self.db.set_totals(run_id, len(entries), total_bytes)
1045 self.db.set_summary(
1046 run_id, _summarize(jobs, tally['skipped'], tally['plain_only'])
1047 )
1048 self.db.set_state(run_id, STATE_READY)
1049 if options.start_after_scan and jobs and not queue_pause.paused:
1050 # "Scan, then upgrade": the chosen categories, at once.
1051 self.db.set_categories(run_id, list(options.categories))
1052 self._begin(run_id, options)
1053 return
1054 except Exception:
1055 logger.opt(exception=True).error('Library upgrade scan failed')
1056 self.db.set_state(run_id, STATE_CANCELLED)
1057 self._publish()
1059 def _isrc_jobs(
1060 self,
1061 entry: dict[str, Any],
1062 checks: dict[str, Any],
1063 options: UpgradeOptions,
1064 tally: dict[str, int],
1065 ) -> list[dict[str, Any]]:
1066 """An ISRC scan's job for *entry* (none or one). Only the tag the
1067 listing already read: no file is opened, nothing else looked for."""
1069 if isrc_mod.normalize(entry.get('isrc')):
1070 return []
1071 stored = str(entry['file'])
1072 if check_is_fresh(
1073 (checks.get(stored) or {}).get(CATEGORY_ISRC),
1074 app_version=self.deps.version,
1075 max_age_days=options.recheck_days,
1076 ):
1077 # Looked up recently and nothing sure was found.
1078 tally['skipped'] += 1
1079 return []
1080 return [
1081 {
1082 'file': stored,
1083 'categories': [CATEGORY_ISRC],
1084 'size': int(entry.get('size') or 0),
1085 'title': str(entry.get('title') or ''),
1086 'artist': str(entry.get('artist') or ''),
1087 }
1088 ]
1090 def _categories_due(
1091 self,
1092 track_checks: dict[str, Any],
1093 recheck_days: int,
1094 *,
1095 synced_only: bool = False,
1096 ) -> set[str]:
1097 """Categories this track hasn't had looked at recently.
1099 Kept per category: repairing a track's artwork says nothing
1100 about whether anyone ever went looking for its lyrics, so a
1101 later run for another category must still consider it. With
1102 *synced_only*, only a lyrics check made that way counts.
1103 """
1105 return {
1106 name
1107 for name in CATEGORIES
1108 if not check_is_fresh(
1109 track_checks.get(check_key(name, synced_only)),
1110 app_version=self.deps.version,
1111 max_age_days=recheck_days,
1112 )
1113 }
1115 @staticmethod
1116 def _summary_for(run: dict[str, Any]) -> dict[str, Any]:
1117 stored = run.get('summary') or {}
1118 return {
1119 'categories': stored.get('categories') or {},
1120 'category_bytes': stored.get('category_bytes') or {},
1121 'tracks': int(stored.get('tracks') or 0),
1122 'recently_checked': int(stored.get('recently_checked') or 0),
1123 'plain_lyrics_only': int(stored.get('plain_lyrics_only') or 0),
1124 'library_tracks': int(run['total_tracks'] or 0),
1125 'library_bytes': int(run['total_bytes'] or 0),
1126 }
1128 def scan_summary(self) -> dict[str, Any]:
1129 """Per-category counts for the finished scan of the latest run."""
1131 run = self.db.latest_run()
1132 if run is None:
1133 return {'categories': {}, 'category_bytes': {}, 'tracks': 0}
1134 return self._summary_for(run)
1136 # -- running --
1137 def start(self, categories: tuple[str, ...]) -> dict[str, Any]:
1138 if queue_pause.paused:
1139 raise RuntimeError('Resume the global queue before upgrading')
1140 with self._lock:
1141 if self.busy():
1142 raise RuntimeError('A library upgrade is already running')
1143 run = self.db.latest_run()
1144 if run is None or run['state'] not in {
1145 STATE_READY,
1146 STATE_PAUSED,
1147 }:
1148 raise RuntimeError('Scan the library before upgrading it')
1149 run_id = int(run['id'])
1150 scanned = options_from(run['options'])
1151 if (
1152 ISRC_LOOKUP_ONLY in scanned.categories
1153 and CATEGORY_ISRC in categories
1154 ):
1155 # A "look up only" scan never turns into a writing run.
1156 categories = (CATEGORY_ISRC, ISRC_LOOKUP_ONLY)
1157 # Remembered so a resume after a restart runs the same
1158 # categories the user picked, not everything the scan found.
1159 self.db.set_categories(run_id, list(categories))
1160 options = replace(scanned, categories=categories)
1161 self._begin(run_id, options)
1162 return self.status()
1164 def _begin(self, run_id: int, options: UpgradeOptions) -> None:
1165 self._cancelled = False
1166 self._finalization_error = ''
1167 self._stop.clear()
1168 self.db.requeue_running(run_id)
1169 self.db.set_state(run_id, STATE_RUNNING)
1170 self._worker = threading.Thread(
1171 target=self._run,
1172 args=(run_id, options),
1173 name='downtify-upgrade',
1174 daemon=True,
1175 )
1176 self._worker.start()
1178 def _run(self, run_id: int, options: UpgradeOptions) -> None:
1179 wanted = set(options.categories)
1180 while not self._stop.is_set():
1181 job = self.db.take_next(run_id)
1182 if job is None:
1183 if options.replace_audio_deemix and 'audio' in wanted:
1184 try:
1185 completed = self.deps.audio_replacement.finalize(
1186 run_id, self._stop.is_set
1187 )
1188 for receipt in completed or []:
1189 detail = (
1190 'Deemix imported; old copy removed'
1191 if receipt['source_deleted']
1192 else 'Deemix imported; original kept'
1193 )
1194 self.db.finish_job(
1195 run_id,
1196 receipt['old_file'],
1197 JOB_DONE,
1198 detail=detail,
1199 changed=['audio'],
1200 )
1201 except Exception as exc:
1202 self._finalization_error = str(exc)[:200]
1203 self.db.set_state(
1204 run_id,
1205 STATE_CANCELLED
1206 if self._cancelled
1207 else STATE_PAUSED,
1208 )
1209 break
1210 self.db.set_state(run_id, STATE_DONE)
1211 break
1212 stored = str(job['file'])
1213 todo = [name for name in job['categories'] if name in wanted]
1214 if not todo:
1215 self.db.finish_job(
1216 run_id, stored, JOB_SKIPPED, detail='Not selected'
1217 )
1218 continue
1219 self._process(run_id, stored, options, todo)
1220 self._publish(throttle=True)
1221 else:
1222 state = STATE_CANCELLED if self._cancelled else STATE_PAUSED
1223 self.db.requeue_running(run_id)
1224 self.db.set_state(run_id, state)
1225 self._publish()
1227 def _kept_by_hand(self, stored: str) -> bool:
1228 """Whether *stored*'s audio was replaced by hand: through
1229 Replace audio, or swapped on disk since Downtify wrote it (which
1230 is then remembered as such)."""
1232 hand = self.deps.hand_replaced
1233 if hand is not None and hand.protected(stored):
1234 return True
1235 sources = self.deps.download_sources
1236 if sources is None:
1237 return False
1238 try:
1239 full = resolve_track(stored, self.deps.context())
1240 if full is None or not sources.replaced_on_disk(stored, full):
1241 return False
1242 if hand is not None:
1243 hand.mark(stored, 'disk')
1244 sources.reject_file_source(stored)
1245 except Exception:
1246 logger.opt(exception=True).debug(
1247 'Hand-replaced check failed for {}', stored
1248 )
1249 return False
1250 return True
1252 def _process(
1253 self,
1254 run_id: int,
1255 stored: str,
1256 options: UpgradeOptions,
1257 todo: list[str],
1258 ) -> None:
1259 def on_stage(stage: str) -> None:
1260 self.db.set_stage(run_id, stored, stage)
1262 if 'audio' in todo and self._kept_by_hand(stored):
1263 # Audio an admin picked by hand isn't swapped for a match.
1264 todo = [name for name in todo if name != 'audio']
1265 if not todo:
1266 self.db.finish_job(
1267 run_id,
1268 stored,
1269 JOB_SKIPPED,
1270 detail='Audio replaced by hand; kept',
1271 )
1272 return
1274 try:
1275 if 'audio' in todo:
1276 if (
1277 not options.replace_audio_deemix
1278 or self.deps.audio_replacement is None
1279 ):
1280 raise ValueError(
1281 'Audio replacement was not enabled for this scan'
1282 )
1283 outcome = self.deps.audio_replacement.prepare(
1284 stored, options, run_id, on_stage
1285 )
1286 else:
1287 outcome = upgrade_track(
1288 stored,
1289 deps=self.deps,
1290 options=UpgradeOptions(
1291 # The look-up-only marker rides along: it is no
1292 # job's category, so *todo* never has it.
1293 categories=tuple(todo)
1294 + tuple(
1295 name
1296 for name in options.categories
1297 if name == ISRC_LOOKUP_ONLY
1298 ),
1299 artwork_min_px=options.artwork_min_px,
1300 artwork_source=options.artwork_source,
1301 recheck_days=options.recheck_days,
1302 ),
1303 on_stage=on_stage,
1304 )
1305 except Exception as exc:
1306 logger.opt(exception=True).warning(
1307 'Library upgrade: {} failed', stored
1308 )
1309 outcome = UpgradeOutcome(JOB_FAILED, str(exc)[:200])
1311 self.db.finish_job(
1312 run_id,
1313 stored,
1314 outcome.status,
1315 detail=outcome.detail,
1316 changed=outcome.changed,
1317 )
1318 if outcome.status != JOB_FAILED and outcome.checked:
1319 # Only the categories this run actually looked at: the rest
1320 # stay due, so a later run for them still considers the
1321 # track.
1322 synced_only = bool(self.deps.settings().get('lyrics_synced_only'))
1323 self.db.record_check(
1324 stored,
1325 [check_key(name, synced_only) for name in todo],
1326 app_version=self.deps.version,
1327 artwork_px=outcome.artwork_px,
1328 artwork_source=outcome.artwork_source,
1329 )
1331 def pause(self) -> dict[str, Any]:
1332 self._cancelled = False
1333 self._stop.set()
1334 return self.status()
1336 def cancel(self) -> dict[str, Any]:
1337 run = self.db.latest_run()
1338 self._cancelled = True
1339 self._stop.set()
1340 if run is not None and not self.busy():
1341 self.db.set_state(int(run['id']), STATE_CANCELLED)
1342 return self.status()
1344 def resume(self) -> dict[str, Any]:
1345 run = self.db.latest_run()
1346 if run is None or run['state'] != STATE_PAUSED:
1347 raise RuntimeError('Nothing to resume')
1348 return self.start(normalize_categories(run['categories']))
1350 def resume_after_restart(self) -> None:
1351 """Pick a run back up when the process died mid-upgrade."""
1353 run = self.db.latest_run()
1354 if run is None:
1355 return
1356 run_id = int(run['id'])
1357 if run['state'] == STATE_SCANNING:
1358 # A half-finished scan can't be trusted; drop it.
1359 self.db.delete_run_jobs(run_id)
1360 self.db.set_state(run_id, STATE_CANCELLED)
1361 return
1362 if run['state'] != STATE_RUNNING:
1363 return
1364 requeued = self.db.requeue_running(run_id)
1365 if queue_pause.paused:
1366 self.db.set_state(run_id, STATE_PAUSED)
1367 return
1368 options = replace(
1369 options_from(run['options']),
1370 categories=normalize_categories(run['categories']),
1371 )
1372 logger.info(
1373 'Library upgrade: resuming run {} ({} track(s) re-queued)',
1374 run_id,
1375 requeued,
1376 )
1377 with self._lock:
1378 self._begin(run_id, options)