Coverage for downtify/downloader.py: 84%
1041 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Download a track from YouTube and tag it with the chosen metadata."""
3from __future__ import annotations
5import base64
6import contextlib
7import functools
8import os
9import re
10import shutil
11import threading
12from concurrent.futures import Future, ThreadPoolExecutor
13from pathlib import Path
14from typing import Any, Callable, Optional
16import httpx
17import yt_dlp
18from loguru import logger
19from mutagen import File as MutagenFile
20from mutagen.flac import FLAC, Picture
21from mutagen.id3 import (
22 APIC,
23 ID3,
24 TALB,
25 TCMP,
26 TCON,
27 TDRC,
28 TIT2,
29 TPE1,
30 TPE2,
31 TPOS,
32 TRCK,
33 TXXX,
34)
35from mutagen.mp3 import MP3
36from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm
37from mutagen.oggopus import OggOpus
38from mutagen.oggvorbis import OggVorbis
39from yt_dlp.postprocessor.ffmpeg import FFmpegExtractAudioPP
41from . import cover_sources
42from . import deezer as deezer_mod
43from . import isrc as isrc_mod
44from . import lyrics as lyrics_mod
45from . import spotify as spotify_mod
46from .album_artist import VARIOUS_ARTISTS, filing_artist
47from .content_advisory import (
48 ADVISORY_EXPLICIT,
49 ADVISORY_TAG,
50 MP4_ADVISORY_ATOM,
51)
52from .cookie_refresh import note_ytdlp_message
53from .cookies import CookiesStore, private_cookiefile
54from .cover_art import extract_embedded_cover
55from .deemix_provider import download_from_deemix, normalize_settings
56from .download_log import (
57 OUTCOME_COMPLETED,
58 OUTCOME_FAILED,
59 OUTCOME_SKIPPED,
60)
61from .download_policy import DownloadPolicyBlocked
62from .download_sources import track_keys
63from .file_naming import sanitize_file_name
64from .image_size import image_short_side
65from .itunes import fetch_genre as _fetch_itunes_genre
66from .library_paths import library_stored_path, slskd_dir_from_downloader
67from .m3u import sanitize_playlist_name
68from .providers import (
69 cover_resolution,
70 enrich_from_match,
71 excluded_source,
72 excluding_sources,
73 find_match,
74 find_match_for_video,
75 find_match_soundcloud,
76 find_match_youtube_only,
77 parse_soundcloud_url,
78 soundcloud_track_url,
79)
80from .release_files import matches_release
81from .slskd_provider import (
82 download_from_slskd,
83 normalize_slskd_quality,
84)
85from .slskd_provider import take_miss as take_slskd_miss
87# Extensions that count as "this song is already downloaded". Deliberately
88# excludes sidecars written next to the audio (.lrc, cover.jpg, .m3u) so a
89# leftover lyrics file can never stand in for a missing track.
90_AUDIO_EXTENSIONS = frozenset({'mp3', 'flac', 'ogg', 'opus', 'm4a'})
93def _valid_imported_audio(path: Path) -> bool:
94 """Whether *path* is non-empty audio that Mutagen can inspect."""
96 try:
97 if not path.is_file() or path.stat().st_size <= 0:
98 return False
99 audio = MutagenFile(path)
100 return audio is not None and float(audio.info.length) > 0
101 except Exception:
102 return False
105# ``(percent, message, provider)`` — ``provider`` is the audio source
106# handling the download (``'slskd'``, ``'youtube-music'``, ``'youtube'``,
107# ``'soundcloud'``, ``'deemix'``), shown next to each queue row.
108ProgressCallback = Callable[[float, str, Optional[str]], None]
110AUDIO_PROVIDERS = (
111 'youtube-music',
112 'youtube',
113 'soundcloud',
114 'slskd',
115 'deemix',
116)
119class NoAudioMatchError(RuntimeError):
120 """No enabled audio provider found a source for the song."""
123#: Songs longer than this are refused before any download: mixes and
124#: hour-long uploads are never what a library track should be. Episodes
125#: never reach this check (see ``podcasts.download_episode``). The
126#: setting ``max_track_duration_seconds`` (Settings → Downloads) moves
127#: this bound, for downloads and streams alike; ``0`` means no limit.
128MAX_TRACK_DURATION_SECONDS = 600
131class TrackTooLongError(RuntimeError):
132 """The song's known duration is past the duration limit."""
135def describe_duration(seconds: float) -> str:
136 """``90`` → ``"90 seconds"``, ``600`` → ``"10 minutes"``."""
138 if seconds >= 90:
139 return f'{seconds / 60:.0f} minutes'
140 return f'{seconds:.0f} seconds'
143def reject_overlong_song(
144 song: dict[str, Any], max_seconds: Optional[int] = None
145) -> None:
146 """Raise :class:`TrackTooLongError` when *song* is too long to take.
148 Unknown durations (missing or ``0``) pass: with nothing to judge by
149 the download goes ahead. So does everything when *max_seconds* is
150 ``0`` (no limit); ``None`` keeps the default bound.
151 """
153 limit = MAX_TRACK_DURATION_SECONDS if max_seconds is None else max_seconds
154 try:
155 duration = float(song.get('duration') or 0)
156 except (TypeError, ValueError):
157 return
158 if limit and duration > limit:
159 name = song.get('name') or 'This track'
160 raise TrackTooLongError(
161 f'"{name}" is {describe_duration(duration)} long; '
162 f'tracks over {describe_duration(limit)} are not downloaded'
163 )
166# Upper bound of the "Parallel downloads" setting (see api.py's clamp).
167MAX_PARALLEL_DOWNLOADS = 30
169# A download holds a thread for its whole run (yt-dlp + ffmpeg, often
170# minutes). Running them on asyncio's default executor — only
171# min(32, cpu_count + 4) threads, e.g. 6 on a 2-core NAS — silently capped
172# "Parallel downloads" below the chosen value and left every
173# `asyncio.to_thread()` call (M3U writes, Monitor DB queries, playlist
174# fetches) queued behind in-flight downloads. Idle threads are cheap; the
175# download semaphore still bounds how many actually run. The extra room
176# covers Playlist Monitor sweeps, which download outside that semaphore.
177DOWNLOAD_EXECUTOR = ThreadPoolExecutor(
178 max_workers=MAX_PARALLEL_DOWNLOADS + 8,
179 thread_name_prefix='downtify-download',
180)
182# Downtify format name → yt-dlp ExtractAudio codec name. yt-dlp calls Ogg
183# Vorbis "vorbis" (output extension .ogg); passing "ogg" crashes the
184# conversion with KeyError: 'ogg'.
185_YTDLP_AUDIO_CODECS = {'ogg': 'vorbis'}
187# yt-dlp codec → the source codec it stream-copies instead of encoding.
188_COPYABLE_SOURCE_CODECS = {'m4a': 'aac', 'opus': 'opus', 'vorbis': 'vorbis'}
190# A same-codec source within this fraction of the chosen bitrate is kept
191# as a lossless stream copy rather than re-encoded to the same bitrate.
192_BITRATE_COPY_TOLERANCE = 0.1
195def _audio_bit_rate(metadata: dict[str, Any]) -> Optional[float]:
196 """Audio bitrate (bits/s) from an ``ffprobe -show_streams -show_format``
197 JSON object, or ``None`` when it can't be told.
199 The audio stream's own ``bit_rate`` wins. WebM/Ogg streams don't carry
200 one, so the container's ``bit_rate`` is used instead — but only for an
201 audio-only file, where it isn't inflated by a video stream.
202 """
204 streams = metadata.get('streams') or []
205 audio = [s for s in streams if s.get('codec_type') == 'audio']
206 if not audio:
207 return None
208 for value in (
209 audio[0].get('bit_rate'),
210 (metadata.get('format') or {}).get('bit_rate')
211 if len(audio) == len(streams)
212 else None,
213 ):
214 try:
215 if value is not None and float(value) > 0:
216 return float(value)
217 except (TypeError, ValueError):
218 continue
219 return None
222class _ExtractAudioPP(FFmpegExtractAudioPP):
223 """yt-dlp's audio extraction, honoring the chosen bitrate.
225 yt-dlp stream-copies instead of encoding whenever the downloaded
226 stream already has the target codec (an AAC source for M4A, Opus for
227 OPUS), silently ignoring ``preferredquality``. YouTube's AAC stream is
228 ~128 kbps, so M4A came out at 128 kbps whatever bitrate was chosen.
230 The copy is kept only when the source is already at the chosen bitrate
231 (within :data:`_BITRATE_COPY_TOLERANCE`) or its bitrate is unknown.
232 Otherwise the source codec is reported under a non-matching name, which
233 sends yt-dlp's ``run()`` down its regular encode path — same encoder
234 choice, bitrate arguments and temp-file handling as any other format.
235 """
237 def __init__(
238 self, downloader: Any, audio_format: str, bitrate_kbps: str
239 ) -> None:
240 self._codec = _YTDLP_AUDIO_CODECS.get(audio_format, audio_format)
241 super().__init__(
242 downloader,
243 preferredcodec=self._codec,
244 preferredquality=bitrate_kbps,
245 )
246 try:
247 self._bitrate_bps = float(bitrate_kbps) * 1000
248 except (TypeError, ValueError):
249 self._bitrate_bps = None
251 def get_audio_codec(self, path: str) -> Optional[str]:
252 codec = super().get_audio_codec(path)
253 if (
254 not self._bitrate_bps
255 or codec is None
256 or codec != _COPYABLE_SOURCE_CODECS.get(self._codec)
257 ):
258 return codec
259 try:
260 source_bps = _audio_bit_rate(self.get_metadata_object(path))
261 except Exception:
262 logger.opt(exception=True).debug('ffprobe bitrate read failed')
263 return codec
264 if source_bps is None or (
265 abs(source_bps - self._bitrate_bps)
266 <= self._bitrate_bps * _BITRATE_COPY_TOLERANCE
267 ):
268 return codec
269 logger.info(
270 'Re-encoding {} source at {:.0f} kbps to {:.0f} kbps {}',
271 codec,
272 source_bps / 1000,
273 self._bitrate_bps / 1000,
274 self._codec,
275 )
276 return f'{codec}-reencode'
279def _sanitize(text: str) -> str:
280 return sanitize_file_name(text)
283# Order matters — yt-dlp tries clients top-to-bottom and uses the first one
284# that yields usable formats. `ios` and `android` lead because they still
285# provide audio formats in containers without a JS runtime, even though
286# YouTube now requires a GVS PO Token for their HTTPS/HLS formats (those
287# are skipped with a warning, but lower-quality streams remain available).
288# `web_embedded` and `web` need a JS runtime for signature/n-challenge
289# solving; without one, they yield no audio at all — so they're kept as
290# last-resort fallbacks only. `mweb` and `tv` are included as hail-mary
291# clients: `mweb` needs a PO Token too, and `tv` is affected by a DRM
292# experiment (yt-dlp #12563), but including them costs nothing.
293_DEFAULT_YT_PLAYER_CLIENTS = (
294 'ios',
295 'android',
296 'web_embedded',
297 'mweb',
298 'web',
299 'tv',
300)
302# Warning substrings emitted by yt-dlp that are known-harmless: they mean
303# some optional format sources are skipped, but other clients in the list
304# still serve usable audio. Suppressed to keep logs readable.
305_SUPPRESSED_YT_WARNING_FRAGMENTS = (
306 'GVS PO Token which was not provided',
307 'Some tv client https formats have been skipped as they are DRM',
308 'Signature solving failed: Some formats may be missing',
309 'n challenge solving failed: Some formats may be missing',
310)
313class _YtdlpLogger:
314 @staticmethod
315 def debug(msg: str) -> None:
316 pass
318 @staticmethod
319 def info(msg: str) -> None:
320 pass
322 @staticmethod
323 def warning(msg: str) -> None:
324 note_ytdlp_message(msg)
325 if not any(frag in msg for frag in _SUPPRESSED_YT_WARNING_FRAGMENTS):
326 logger.warning('yt-dlp: {}', msg)
328 @staticmethod
329 def error(msg: str) -> None:
330 note_ytdlp_message(msg)
331 logger.error('yt-dlp: {}', msg)
334def _yt_player_clients() -> list[str]:
335 raw = os.getenv('DOWNTIFY_YT_PLAYER_CLIENTS', '').strip()
336 if not raw:
337 return list(_DEFAULT_YT_PLAYER_CLIENTS)
338 clients = [c.strip() for c in raw.split(',') if c.strip()]
339 return clients or list(_DEFAULT_YT_PLAYER_CLIENTS)
342def _yt_po_tokens() -> list[str]:
343 """Comma-separated PO Tokens, each in the form ``<client>.<context>+<token>``.
345 Example: ``mweb.gvs+ABC123,web.gvs+XYZ987``
346 """
347 raw = os.getenv('DOWNTIFY_YT_PO_TOKEN', '').strip()
348 if not raw:
349 return []
350 return [t.strip() for t in raw.split(',') if t.strip()]
353# Substrings YouTube/yt-dlp use when a video is behind the age wall.
354_AGE_GATE_ERROR_FRAGMENTS = (
355 'confirm your age',
356 'age-restricted',
357 'inappropriate for some users',
358 'age_verification_required',
359 'age_check_required',
360)
363def is_age_restricted_error(message: str) -> bool:
364 return any(
365 fragment in message.lower() for fragment in _AGE_GATE_ERROR_FRAGMENTS
366 )
369def _is_missing_format_error(exc: Exception) -> bool:
370 """Whether yt-dlp found the media but none of the wanted formats."""
372 return 'requested format is not available' in str(exc).lower()
375def _translate_download_error(
376 exc: Exception, song: dict[str, Any], has_cookies: bool
377) -> Exception:
378 """Replace yt-dlp's age-gate error with something actionable.
380 The raw error is a wall of yt-dlp CLI advice (``--cookies-from-browser``,
381 wiki links) that means nothing to someone using the web UI, and it's
382 the single most common "downloads are broken" report. Every other
383 failure is passed through untouched.
384 """
386 if not is_age_restricted_error(str(exc)):
387 return exc
388 name = song.get('name') or 'This track'
389 if has_cookies:
390 return RuntimeError(
391 f'"{name}" is age-restricted on YouTube and the configured '
392 'cookies file was not accepted. Export a fresh cookies.txt '
393 'while logged into a YouTube account that has completed '
394 "Google's age verification, then upload it again in "
395 'Settings > YouTube cookies.'
396 )
397 return RuntimeError(
398 f'"{name}" has explicit content and YouTube only serves it to a '
399 'signed-in adult account. Upload a YouTube cookies.txt in '
400 'Settings > YouTube cookies to download it.'
401 )
404#: Why slskd had nothing for the download running on this thread (set
405#: when it falls through to the next source, reported with the outcome).
406_SLSKD_MISS = threading.local()
409class Downloader:
410 """Wraps ``yt-dlp`` plus ``mutagen`` tagging."""
412 #: Called as ``on_downloaded(song, filename)`` once a *new* file has been
413 #: downloaded and tagged - not for a song that was already in the library.
414 #: ``song['download_provider']`` says which audio provider fetched it
415 #: (``youtube-music``, ``youtube``, ``soundcloud``, ``slskd``,
416 #: ``deemix``).
417 #: Set by the app (``main.py``) to hook work on to a finished download
418 #: (see ``api.enrich_artist_after_download``); it runs on the download's
419 #: own thread, so it must only hand the work off and return. Whatever it
420 #: raises is logged and never fails the download.
421 #: ``song['download_source']`` is the absolute path of the transfer a
422 #: file was copied from (slskd, not left in place), else absent.
423 on_downloaded: Optional[Callable[[dict[str, Any], str], None]] = None
424 #: Called as ``on_outcome(song, outcome, filename)`` whenever a
425 #: download ends - ``completed``, ``skipped`` (the song was already in
426 #: the library) or ``failed`` (*filename* empty), see
427 #: :mod:`downtify.download_log` - on the download's own thread, like
428 #: :attr:`on_downloaded`, and just as unable to fail the download.
429 on_outcome: Optional[Callable[[dict[str, Any], str, str], None]] = None
430 #: Called as ``album_override(song)`` once a song's metadata is complete,
431 #: before its path and tags are worked out: the ``album_artist`` /
432 #: ``compilation`` an admin set for its album by hand, or ``{}``. Set by
433 #: the app (``main.py``, see ``compilation.CompilationMarks``). Whatever
434 #: it raises is logged and the song keeps its source's values.
435 album_override: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None
436 #: Each new file's source, and the sources rejected per song
437 #: (:class:`downtify.download_sources.DownloadSources`), set by the
438 #: app: a rejected upload is never picked again for its song, and a
439 #: file swapped on disk is kept like one replaced through Replace
440 #: audio.
441 download_sources: Any = None
442 #: Files whose audio an admin replaced by hand
443 #: (:class:`downtify.hand_replaced.HandReplacedFiles`), set by the app.
444 hand_replaced: Any = None
446 def __init__(
447 self,
448 download_dir: Path | str,
449 audio_format: str = 'mp3',
450 audio_bitrate: str = '320',
451 output_template: str = '{artists} - {title}',
452 lyrics_providers: Optional[list[str]] = None,
453 lyrics_cache: Optional[Any] = None,
454 organize_by_artist: bool = False,
455 organize_by_album: bool = False,
456 download_cover_art: bool = True,
457 cover_min_resolution: int = 0,
458 cover_largest: bool = False,
459 write_isrc: bool = True,
460 overwrite_existing_files: bool = True,
461 cookies_store: Optional[CookiesStore] = None,
462 audio_providers: Optional[list[str]] = None,
463 slskd_settings: Optional[dict[str, Any]] = None,
464 max_track_duration_seconds: int = MAX_TRACK_DURATION_SECONDS,
465 deemix_settings: Optional[dict[str, Any]] = None,
466 ):
467 self.download_dir = Path(download_dir)
468 self.download_dir.mkdir(parents=True, exist_ok=True)
469 # The smallest cover (short side, px) worth embedding; 0 = take
470 # what the track's source offers (see _download_cover_meeting).
471 self.cover_min_resolution = max(0, int(cover_min_resolution or 0))
472 # Look for a larger cover on every download, not only below the
473 # minimum (see _download_cover_meeting).
474 self.cover_largest = bool(cover_largest)
475 # Write each download's ISRC (downtify.isrc).
476 self.write_isrc = bool(write_isrc)
477 self.deemix_settings = normalize_settings(deemix_settings)
478 self.audio_providers = self._normalize_audio_providers(audio_providers)
479 self.slskd_settings = self._normalize_slskd_settings(slskd_settings)
480 # slskd copies a finished transfer here when it isn't left in place.
481 self.slskd_settings['output_dir'] = str(self.download_dir)
482 # Resolves DOWNTIFY_COOKIES_FILE and the cookies.txt uploaded
483 # through the settings UI, in that order of precedence. ``None``
484 # keeps the env-var-only behavior for direct/standalone use.
485 self.cookies_store = cookies_store
486 self.audio_format = audio_format
487 self.audio_bitrate = audio_bitrate
488 self.output_template = output_template
489 self.lyrics_providers = list(lyrics_providers or [])
490 #: Keep only time-synced lyrics (the ``lyrics_synced_only``
491 #: setting): plain lyrics count as none found.
492 self.lyrics_synced_only = False
493 # Optional LyricsLookupCache: skips providers that already came
494 # back empty for a song.
495 self.lyrics_cache = lyrics_cache
496 self.organize_by_artist = organize_by_artist
497 self.organize_by_album = organize_by_album
498 self.overwrite_existing_files = overwrite_existing_files
499 self.download_cover_art = download_cover_art
500 # ``0``: overlong tracks are taken, like unknown durations.
501 self.max_track_duration_seconds = max_track_duration_seconds
502 self._target_locks: dict[str, threading.Lock] = {}
503 self._target_locks_guard = threading.Lock()
505 @staticmethod
506 def _normalize_audio_providers(
507 providers: Optional[list[str]],
508 ) -> list[str]:
509 """Known providers in the given order, deduplicated.
511 Defaults to YouTube Music, which already falls back to standard
512 YouTube inside :func:`providers.find_match`.
513 """
515 out: list[str] = []
516 for raw in providers or []:
517 name = str(raw or '').strip()
518 if name in AUDIO_PROVIDERS and name not in out:
519 out.append(name)
520 return out or ['youtube-music']
522 @staticmethod
523 def _normalize_slskd_settings(
524 settings: Optional[dict[str, Any]],
525 ) -> dict[str, Any]:
526 raw = settings if isinstance(settings, dict) else {}
528 def _int(value: Any, default: int, low: int, high: int) -> int:
529 try:
530 number = int(value if value is not None else default)
531 except (TypeError, ValueError):
532 number = default
533 return min(high, max(low, number))
535 download_dir = str(raw.get('download_dir') or '/downloads').strip()
536 return {
537 'enabled': bool(raw.get('enabled', False)),
538 'base_url': str(raw.get('base_url') or '').strip().rstrip('/'),
539 'api_key': str(raw.get('api_key') or '').strip(),
540 'download_dir': download_dir,
541 'source_dir': str(raw.get('source_dir') or download_dir).strip(),
542 'timeout_seconds': _int(raw.get('timeout_seconds'), 20, 1, 3600),
543 'search_retries': _int(raw.get('search_retries'), 5, 1, 100),
544 'search_poll_seconds': _int(
545 raw.get('search_poll_seconds'), 15, 1, 3600
546 ),
547 'download_attempts': _int(raw.get('download_attempts'), 5, 1, 100),
548 'poll_interval_seconds': _int(
549 raw.get('poll_interval_seconds'), 5, 1, 3600
550 ),
551 'poll_max_attempts': _int(
552 raw.get('poll_max_attempts'), 60, 1, 10000
553 ),
554 'download_timeout_seconds': _int(
555 raw.get('download_timeout_seconds'), 600, 30, 3600
556 ),
557 'queued_timeout_seconds': _int(
558 raw.get('queued_timeout_seconds'), 180, 15, 3600
559 ),
560 'duration_tolerance_seconds': _int(
561 raw.get('duration_tolerance_seconds'), 10, 1, 120
562 ),
563 'duration_tolerance_percent': _int(
564 raw.get('duration_tolerance_percent'), 15, 1, 100
565 ),
566 'mix_duration_tolerance_percent': _int(
567 raw.get('mix_duration_tolerance_percent'), 50, 1, 200
568 ),
569 'extensions': raw.get('extensions') or ['mp3', 'flac'],
570 'strict_format_priority': bool(
571 raw.get('strict_format_priority', False)
572 ),
573 'extensions_whitelist': bool(
574 raw.get('extensions_whitelist', False)
575 ),
576 'min_bitrate': _int(raw.get('min_bitrate'), 256, 0, 10000),
577 **normalize_slskd_quality(raw),
578 'leave_in_place': bool(raw.get('leave_in_place', True)),
579 'keep_file_cover': bool(raw.get('keep_file_cover', False)),
580 'max_parallel_downloads': _int(
581 raw.get('max_parallel_downloads'), 3, 1, 8
582 ),
583 }
585 def _resolve_source(
586 self,
587 song: dict[str, Any],
588 progress_cb: Optional[ProgressCallback],
589 cancel_check=None,
590 ) -> tuple[
591 Optional[str], Optional[dict[str, Any]], Optional[str], Optional[Path]
592 ]:
593 """Try each audio provider in order.
595 Returns ``(video_id, ytm_match, provider, local_path)``: a YouTube
596 video id (with its YouTube Music result, if any), a SoundCloud
597 track URL or, for slskd/deemix, the path of the already-downloaded
598 file. All ``None`` when every provider came up empty.
599 """
601 known_video_id = (
602 None if song.get('_match_by_metadata') else song.get('youtube_id')
603 )
604 if (
605 not known_video_id
606 and not song.get('_match_by_metadata')
607 and song.get('source') == 'youtube'
608 ):
609 known_video_id = song.get('song_id')
610 known_soundcloud_url = soundcloud_track_url(song)
611 if excluded_source(known_soundcloud_url):
612 known_soundcloud_url = None
614 youtube_searched = False
615 for provider in self.audio_providers:
616 if cancel_check:
617 cancel_check()
618 if provider == 'deemix':
619 if not self.deemix_settings.get('enabled'):
620 continue
621 local = download_from_deemix(
622 song,
623 self.deemix_settings,
624 progress_cb=progress_cb,
625 **({'cancel_check': cancel_check} if cancel_check else {}),
626 )
627 if local is not None:
628 return None, None, 'deemix', local
629 _report(progress_cb, 0.0, 'Trying next source', 'deemix')
630 elif provider == 'slskd':
631 if not self.slskd_settings.get('enabled'):
632 continue
633 local = download_from_slskd(
634 song,
635 self.slskd_settings,
636 progress_cb=progress_cb,
637 **({'cancel_check': cancel_check} if cancel_check else {}),
638 )
639 if local is not None:
640 return None, None, 'slskd', local
641 # Why: told with the outcome (see _report_outcome).
642 _SLSKD_MISS.value = take_slskd_miss() or {
643 'reason': 'unknown',
644 'detail': '',
645 }
646 logger.info(
647 'Audio provider slskd: no match for {!r}', song.get('name')
648 )
649 _report(progress_cb, 0.0, 'Trying next source', 'slskd')
650 elif provider == 'youtube-music':
651 if known_video_id:
652 return known_video_id, None, provider, None
653 # find_match falls back to standard YouTube on its own when
654 # YouTube Music has nothing (or only a far-off length).
655 video_id, match = find_match(song)
656 youtube_searched = True
657 if video_id:
658 return (
659 video_id,
660 match,
661 'youtube-music' if match is not None else 'youtube',
662 None,
663 )
664 elif provider == 'youtube' and not youtube_searched:
665 if known_video_id:
666 return known_video_id, None, provider, None
667 video_id = find_match_youtube_only(song)
668 youtube_searched = True
669 if video_id:
670 return video_id, None, 'youtube', None
671 elif provider == 'soundcloud':
672 # Like a known video: the exact track, but only here, at
673 # SoundCloud's place in the order.
674 track_url = known_soundcloud_url or find_match_soundcloud(song)
675 if track_url:
676 return track_url, None, 'soundcloud', None
677 logger.info(
678 'Audio provider soundcloud: no match for {!r}',
679 song.get('name'),
680 )
681 _report(progress_cb, 0.0, 'Trying next source', 'soundcloud')
682 return None, None, None, None
684 def _resolve_cookies_file(self) -> str:
685 """Path to the cookies.txt yt-dlp should use, or ``''``.
687 With a store configured this is DOWNTIFY_COOKIES_FILE, else the
688 file uploaded through the settings UI. Resolved per download so
689 uploading or deleting one takes effect without a restart.
690 """
691 if self.cookies_store is not None:
692 active = self.cookies_store.active_path()
693 return str(active) if active else ''
694 return os.getenv('DOWNTIFY_COOKIES_FILE', '').strip()
696 @staticmethod
697 def _artist_subdir(song: dict[str, Any]) -> str:
698 # The album artist the source declares, so every track of an album
699 # lands in the same folder even when one track's own `artists`
700 # differs (a feature, a remix credit, ...) - but a compilation's
701 # tracks go to their own first artist, never a "Various Artists"
702 # folder. See `album_artist.filing_artist`.
703 return _sanitize(filing_artist(song) or 'unknown')
705 @staticmethod
706 def _album_subdir(song: dict[str, Any]) -> str:
707 return _sanitize(song.get('album_name') or 'unknown')
709 def _effective_subdir(
710 self, song: dict[str, Any], fallback: Optional[str]
711 ) -> Optional[str]:
712 """Resolve the destination sub-directory for ``song``.
714 The ``organize_by_*`` toggles take precedence over the caller's
715 ``fallback`` (e.g. a per-playlist folder). When both are enabled
716 the layout nests as ``<Artist>/<Album>/``; a single toggle yields
717 just that one level. When neither is set the ``fallback`` is used
718 unchanged.
719 """
721 parts: list[str] = []
722 if self.organize_by_artist:
723 parts.append(self._artist_subdir(song))
724 if self.organize_by_album:
725 parts.append(self._album_subdir(song))
726 if parts:
727 return '/'.join(parts)
728 return fallback
730 def _save_album_cover(
731 self,
732 target_dir: Path,
733 song: dict[str, Any],
734 cover_bytes: Optional[bytes] = None,
735 ) -> None:
736 """Save the album art as ``cover.jpg`` inside the album folder.
738 Only runs when organizing by album, so ``target_dir`` is the
739 album's own folder. ``song['cover_url']`` already points at the
740 largest image Spotify offers (see ``spotify._largest_image``).
741 Skipped when ``cover.jpg`` already exists so the art is fetched
742 once per album rather than once per track. ``cover_bytes``, when
743 already fetched for embedding, is written instead of downloading
744 the image again.
745 """
747 if not self.organize_by_album:
748 return
749 cover_path = target_dir / 'cover.jpg'
750 if cover_path.exists():
751 return
752 data = cover_bytes or _download_cover(song.get('cover_url', ''))
753 if not data:
754 return
755 try:
756 cover_path.write_bytes(data)
757 except OSError:
758 logger.opt(exception=True).warning(
759 'Could not write album cover {}', cover_path
760 )
762 @staticmethod
763 def _template_values(song: dict[str, Any]) -> dict[str, str]:
764 artist_names = [_sanitize(a) for a in (song.get('artists') or [])]
765 artists = ', '.join(a for a in artist_names if a) or 'Unknown Artist'
766 # Same normalization used for the embedded tag (see
767 # _album_track_index_for_tags): only a positive integer counts,
768 # anything else (missing, non-numeric, a free-text/YouTube search
769 # result with no Spotify track_number) renders as ''. Zero-padded
770 # to 2 digits so "{tracknumber} - {title}" sorts correctly in a
771 # file browser (2 < 10 lexicographically without padding).
772 track_number, _ = _album_track_index_for_tags(song)
773 # Same source as the embedded tag (_recording_date_for_tags): a
774 # full release date or a bare year, so just the leading 4 digits.
775 # Empty when the source has no date at all, same tolerance as
776 # tracknumber above.
777 date = _recording_date_for_tags(song)
778 year = date[:4] if date[:4].isdigit() else ''
779 return {
780 'title': _sanitize(song.get('name', 'Unknown')),
781 'artists': artists,
782 # The first credited artist alone, as the Settings preview shows.
783 'artist': next((a for a in artist_names if a), 'Unknown Artist'),
784 'album': _sanitize(song.get('album_name', '')),
785 'tracknumber': f'{track_number:02d}' if track_number else '',
786 'year': year,
787 }
789 def _format_output_parts(self, song: dict[str, Any]) -> list[str]:
790 """Render the output template as safe relative path components."""
792 template = self.output_template.replace('.{output-ext}', '')
793 try:
794 rendered = template.format(**self._template_values(song))
795 except (KeyError, IndexError):
796 values = self._template_values(song)
797 rendered = f'{values["artists"]} - {values["title"]}'
799 parts = [_sanitize(part) for part in re.split(r'[\\/]+', rendered)]
800 parts = [part for part in parts if part] or ['unknown']
801 # An artist watch whose file name is taken by a different
802 # recording saves the track as "<name> (<album>)" instead (see
803 # monitor._download_release_track).
804 suffix = str(song.get('_output_suffix') or '').strip()
805 if suffix:
806 parts[-1] = f'{parts[-1]} ({_sanitize(suffix)})'
807 return parts
809 def _format_basename(self, song: dict[str, Any]) -> str:
810 return '/'.join(self._format_output_parts(song))
812 def existing_filename_for(
813 self,
814 song: dict[str, Any],
815 subdir: Optional[str] = None,
816 ) -> Optional[str]:
817 """Return the on-disk filename for ``song`` if any matching file exists.
819 Mirrors :meth:`download`'s post-conversion path resolution: prefers
820 ``{basename}.{audio_format}`` and falls back to any
821 ``{basename}.*`` since yt-dlp occasionally keeps the upstream
822 extension (opus, m4a). Returns ``None`` when no file matches.
824 When ``subdir`` is given the lookup is scoped to that
825 sub-directory and the returned name is relative to
826 ``download_dir`` (``<subdir>/<file>.<ext>``).
827 """
829 target_dir, prefix, basename = self._expected_location(song, subdir)
830 primary = target_dir / f'{basename}.{self.audio_format}'
831 if primary.exists():
832 return f'{prefix}{primary.name}'
833 # Not a glob: titles like "Song [Live]" are glob character classes.
834 if target_dir.is_dir():
835 for candidate in sorted(target_dir.iterdir()):
836 stem, dot, ext = candidate.name.rpartition('.')
837 if (
838 dot
839 and stem == basename
840 and ext.lower() in _AUDIO_EXTENSIONS
841 and candidate.is_file()
842 ):
843 return f'{prefix}{candidate.name}'
844 return None
846 def _expected_location(
847 self, song: dict[str, Any], subdir: Optional[str]
848 ) -> tuple[Path, str, str]:
849 """``(target_dir, relative_prefix, basename)`` a download of
850 ``song`` lands at - no disk access."""
852 output_parts = self._format_output_parts(song)
853 basename = output_parts[-1]
854 output_subdir = (
855 Path(*output_parts[:-1]) if len(output_parts) > 1 else None
856 )
857 effective_subdir = self._effective_subdir(song, subdir)
858 target_dir, prefix = self._resolve_target_dir(effective_subdir)
859 if output_subdir is not None:
860 target_dir /= output_subdir
861 prefix = f'{prefix}{output_subdir.as_posix()}/'
862 return target_dir, prefix, basename
864 def expected_stem_for(
865 self, song: dict[str, Any], subdir: Optional[str] = None
866 ) -> str:
867 """The library path :meth:`existing_filename_for` looks for,
868 without the extension (``<subdir>/<file>``) - no disk access, so
869 many songs can be checked against an in-memory listing."""
871 _target_dir, prefix, basename = self._expected_location(song, subdir)
872 return f'{prefix}{basename}'
874 def find_existing_download(
875 self,
876 song: dict[str, Any],
877 subdir: Optional[str] = None,
878 ) -> Optional[str]:
879 """Return a file anywhere in the library that already holds ``song``.
881 Checks the exact destination first, then the whole library for an
882 audio file whose name (and any folders the output template adds)
883 matches, case-insensitively. That catches the same song saved by
884 a single-track download (library root), by another playlist's
885 folder, or under a previous organize-by-artist/album layout.
886 Matching is by rendered filename, not by track id: the files carry
887 no id to compare against.
888 """
890 exact = self.existing_filename_for(song, subdir)
891 if exact is not None:
892 if song.get('_monitor_release'):
893 if not matches_release(self.download_dir / exact, song):
894 raise FileExistsError(
895 'A different recording occupies the requested release path'
896 )
897 return exact
898 wanted = [p.casefold() for p in self._format_output_parts(song)]
899 if song.get('_monitor_release'):
900 return self._scan_library(wanted[-1], wanted[:-1], song)
901 return self._scan_library(wanted[-1], wanted[:-1])
903 def _scan_library(
904 self,
905 stem: str,
906 parents: list[str],
907 release: Optional[dict[str, Any]] = None,
908 ) -> Optional[str]:
909 root_dir = self.download_dir
910 for root, dirs, files in os.walk(root_dir):
911 dirs[:] = sorted(d for d in dirs if not d.startswith('.'))
912 for name in sorted(files):
913 base, dot, ext = name.rpartition('.')
914 if (
915 not dot
916 or ext.lower() not in _AUDIO_EXTENSIONS
917 or base.casefold() != stem
918 ):
919 continue
920 rel = Path(root, name).relative_to(root_dir)
921 if parents:
922 folders = [p.casefold() for p in rel.parts[:-1]]
923 if folders[-len(parents) :] != parents:
924 continue
925 if release:
926 if not matches_release(root_dir / rel, release):
927 continue
928 return rel.as_posix()
929 return None
931 def _can_resolve_path_early(self, song: dict[str, Any]) -> bool:
932 """Whether ``song`` already has every field its output path needs.
934 True for Spotify-sourced songs, which lets the existing-file check
935 run before the YouTube Music search instead of after it. Anything
936 the template needs that only enrichment could fill in (album,
937 track number) makes this False, so an incomplete early path can
938 never match an unrelated file.
939 """
941 if not (song.get('name') and song.get('artists')):
942 return False
943 template = self.output_template
944 if (self.organize_by_album or '{album}' in template) and not song.get(
945 'album_name'
946 ):
947 return False
948 if '{tracknumber}' in template:
949 track_number, _ = _album_track_index_for_tags(song)
950 if not track_number:
951 return False
952 return True
954 def _target_lock(self, song: dict[str, Any]) -> threading.Lock:
955 key = '/'.join(p.casefold() for p in self._format_output_parts(song))
956 with self._target_locks_guard:
957 return self._target_locks.setdefault(key, threading.Lock())
959 def _skip_existing(
960 self,
961 song: dict[str, Any],
962 existing: str,
963 progress_cb: Optional[ProgressCallback],
964 ) -> str:
965 logger.info('Skipping download, file already exists: {}', existing)
966 _report(progress_cb, 100.0, 'Already downloaded')
967 self._report_outcome(song, OUTCOME_SKIPPED, existing)
968 return existing
970 def _hand_replaced_at(
971 self, song: dict[str, Any], subdir: Optional[str]
972 ) -> Optional[str]:
973 """The library file a download of *song* would write over, when
974 an admin replaced its audio by hand
975 (:mod:`downtify.hand_replaced`); ``None`` otherwise.
977 Only matters with ``overwrite_existing_files`` on: off, nothing
978 existing is ever overwritten."""
980 store = getattr(self, 'hand_replaced', None)
981 sources = getattr(self, 'download_sources', None)
982 if store is None and sources is None:
983 return None
984 try:
985 existing = self.existing_filename_for(song, subdir)
986 if existing is None:
987 return None
988 if store is not None and store.protected(existing):
989 return existing
990 if self._replaced_on_disk(existing, song):
991 return existing
992 return None
993 except Exception:
994 logger.opt(exception=True).debug('Hand-replaced check failed')
995 return None
997 def _replaced_on_disk(self, existing: str, song: dict[str, Any]) -> bool:
998 """Whether *existing* (a file Downtify wrote) was swapped on disk
999 since: its audio isn't what was written. Such a file is then
1000 remembered as replaced by hand, and the upload it came from is
1001 rejected for the song (issue #582)."""
1003 sources = getattr(self, 'download_sources', None)
1004 if sources is None:
1005 return False
1006 full = self.download_dir / existing
1007 if not sources.replaced_on_disk(existing, full):
1008 return False
1009 logger.info(
1010 '{} was replaced outside Downtify (its audio changed); keeping it',
1011 existing,
1012 )
1013 store = getattr(self, 'hand_replaced', None)
1014 if store is not None:
1015 store.mark(existing, 'disk')
1016 sources.reject_file_source(existing, track_keys(song))
1017 sources.record(existing, full, '', track_keys(song))
1018 return True
1020 def _skip_hand_replaced(
1021 self,
1022 song: dict[str, Any],
1023 existing: str,
1024 progress_cb: Optional[ProgressCallback],
1025 ) -> str:
1026 logger.info(
1027 'Keeping {}: its audio was replaced by hand, not downloading '
1028 'over it',
1029 existing,
1030 )
1031 _report(progress_cb, 100.0, 'Kept: audio replaced by hand')
1032 self._report_outcome(song, OUTCOME_SKIPPED, existing)
1033 return existing
1035 def _resolve_target_dir(self, subdir: Optional[str]) -> tuple[Path, str]:
1036 """Return ``(target_dir, relative_prefix)`` for an optional subdir.
1038 ``subdir`` may contain ``/`` separators to express a nested
1039 layout (e.g. ``<Artist>/<Album>``); each path component is
1040 sanitised individually so the separators survive. The
1041 ``relative_prefix`` is empty when ``subdir`` is not used and
1042 otherwise terminates with ``'/'`` so callers can build the
1043 download-dir-relative path with simple concatenation.
1044 """
1046 if not subdir:
1047 return self.download_dir, ''
1048 parts = [
1049 sanitize_playlist_name(p) for p in subdir.split('/') if p.strip()
1050 ]
1051 if not parts:
1052 return self.download_dir, ''
1053 rel = '/'.join(parts)
1054 return self.download_dir / Path(*parts), f'{rel}/'
1056 def download(
1057 self,
1058 song: dict[str, Any],
1059 progress_cb: Optional[ProgressCallback] = None,
1060 subdir: Optional[str] = None,
1061 ) -> str:
1062 policy = getattr(self, 'download_policy', None)
1063 if policy is None:
1064 return self._download_inner(song, progress_cb, subdir)
1065 limit_getter = getattr(self, 'download_policy_limit', lambda: 0)
1066 try:
1067 limit_minutes = max(0, min(1440, int(limit_getter())))
1068 except (TypeError, ValueError, OverflowError):
1069 limit_minutes = 0
1070 try:
1071 attempt = policy.begin(song, limit_minutes * 60)
1072 except ValueError as exc:
1073 if str(exc) != 'identity_uncertain':
1074 raise
1075 return self._download_inner(song, progress_cb, subdir)
1076 successful = False
1077 try:
1078 attempt.check(force=True)
1079 result = self._download_inner(
1080 song, progress_cb, subdir, attempt.check
1081 )
1082 attempt.check(force=True)
1083 successful = bool(result) and not song.get('_monitor_release')
1084 return result
1085 finally:
1086 attempt.finish(successful=successful)
1088 def _download_inner(
1089 self,
1090 song: dict[str, Any],
1091 progress_cb: Optional[ProgressCallback] = None,
1092 subdir: Optional[str] = None,
1093 cancel_check=None,
1094 ) -> str:
1095 """Download ``song`` and return the resulting file name.
1097 When ``subdir`` is provided the file is written under
1098 ``download_dir/<sanitized_subdir>/`` and the returned name is
1099 relative to ``download_dir`` (``<subdir>/<file>.<ext>``). This
1100 is how playlist downloads are grouped into per-playlist folders.
1102 With ``overwrite_existing_files`` off, a song already anywhere in
1103 the library is not downloaded again; the existing file's name is
1104 returned instead (see :meth:`find_existing_download`).
1106 However it ends, :attr:`on_outcome` hears of it.
1107 """
1109 def progress(*args):
1110 if cancel_check:
1111 cancel_check()
1112 if progress_cb:
1113 progress_cb(*args)
1115 _SLSKD_MISS.value = None
1117 try:
1118 if cancel_check:
1119 return self._download(song, progress, subdir, cancel_check)
1120 return self._download(song, progress_cb, subdir)
1121 except DownloadPolicyBlocked:
1122 raise
1123 except Exception:
1124 self._report_outcome(song, OUTCOME_FAILED)
1125 raise
1127 def _download(
1128 self,
1129 song: dict[str, Any],
1130 progress_cb: Optional[ProgressCallback],
1131 subdir: Optional[str],
1132 cancel_check=None,
1133 ) -> str:
1134 if getattr(self, 'overwrite_existing_files', True):
1135 return self._download_impl(song, progress_cb, subdir, cancel_check)
1136 with self._target_lock(song):
1137 return self._download_impl(song, progress_cb, subdir, cancel_check)
1139 def _download_impl(
1140 self,
1141 song: dict[str, Any],
1142 progress_cb: Optional[ProgressCallback],
1143 subdir: Optional[str],
1144 cancel_check=None,
1145 ) -> str:
1146 """:meth:`download` itself."""
1148 # A song sent to the music trash stays out until an admin restores
1149 # it or allows it again; like an exclusion, not a failed attempt.
1150 trash = getattr(self, 'trash_store', None)
1151 if trash is not None and trash.blocked(song):
1152 raise DownloadPolicyBlocked('library_trash_excluded')
1153 skip_existing = not self.overwrite_existing_files
1154 if skip_existing and self._can_resolve_path_early(song):
1155 existing = self.find_existing_download(song, subdir)
1156 if existing is not None:
1157 return self._skip_existing(song, existing, progress_cb)
1158 elif self._can_resolve_path_early(song):
1159 kept = self._hand_replaced_at(song, subdir)
1160 if kept is not None:
1161 return self._skip_hand_replaced(song, kept, progress_cb)
1163 # Known-long tracks are refused before any search or fetch.
1164 reject_overlong_song(song, self.max_track_duration_seconds)
1166 # Uploads found wrong for this song before are never picked again.
1167 rejected = self._rejected_sources(song)
1168 if song.get('youtube_id') in rejected or (
1169 song.get('source') == 'youtube' and song.get('song_id') in rejected
1170 ):
1171 song = {**song, '_match_by_metadata': True}
1173 known_video_id = (
1174 None if song.get('_match_by_metadata') else song.get('youtube_id')
1175 )
1176 if (
1177 not known_video_id
1178 and not song.get('_match_by_metadata')
1179 and song.get('source') == 'youtube'
1180 ):
1181 known_video_id = song.get('song_id')
1183 if not known_video_id:
1184 # Playlist rows lack year/track number; the per-track embed has
1185 # them, and they can feed the output path and the slskd match.
1186 song = spotify_mod.enrich_track_from_spotify_if_sparse(song)
1187 # A Deezer album/playlist row names only its lead artist.
1188 song = deezer_mod.enrich_track_from_deezer_if_sparse(song)
1189 # A known video is a candidate for an enabled YouTube provider,
1190 # not permission to bypass earlier sources or disabled providers.
1191 with excluding_sources(rejected):
1192 video_id, match, provider, local_source = self._resolve_source(
1193 song,
1194 progress_cb,
1195 **({'cancel_check': cancel_check} if cancel_check else {}),
1196 )
1197 if cancel_check:
1198 cancel_check()
1199 if (
1200 video_id
1201 and video_id == known_video_id
1202 and (not song.get('album_name') or not song.get('cover_url'))
1203 ):
1204 # We already have a target video, but the metadata is incomplete.
1205 # Look up the YT Music entry for THIS specific videoId so we
1206 # don't risk switching to a karaoke / cover that happens to
1207 # rank higher.
1208 try:
1209 match = find_match_for_video(song, video_id)
1210 except Exception:
1211 logger.opt(exception=True).debug('enrichment match failed')
1212 match = None
1214 if not video_id and local_source is None:
1215 source = (
1216 'a YouTube match'
1217 if all(
1218 source in {'youtube-music', 'youtube'}
1219 for source in self.audio_providers
1220 )
1221 else 'an audio match'
1222 )
1223 hint = (
1224 ' - turn SoundCloud on in Settings > Audio sources to '
1225 'download SoundCloud tracks'
1226 if soundcloud_track_url(song)
1227 and 'soundcloud' not in self.audio_providers
1228 else ''
1229 )
1230 raise NoAudioMatchError(
1231 f'Could not find {source} for {song.get("name")!r}{hint}'
1232 )
1234 song = self._with_album_override(enrich_from_match(song, match))
1236 # Sparse rows learn their duration from the per-track embed
1237 # above; re-check now that it is known.
1238 reject_overlong_song(song, self.max_track_duration_seconds)
1240 if not skip_existing:
1241 # The path is complete now (album, track number).
1242 kept = self._hand_replaced_at(song, subdir)
1243 if kept is not None:
1244 return self._skip_hand_replaced(song, kept, progress_cb)
1246 if local_source is not None:
1247 stored, copied_from = self._finalize_local_source(
1248 song, local_source, provider, progress_cb, subdir
1249 )
1250 return self._downloaded(song, stored, provider, copied_from)
1252 if not skip_existing:
1253 return self._downloaded(
1254 song,
1255 self._fetch_and_tag(
1256 song, video_id, progress_cb, subdir, provider
1257 ),
1258 provider,
1259 source_id=video_id,
1260 )
1262 # Re-check after enrichment while the target lock is still held,
1263 # so every provider (including local slskd transfers) is serialized.
1264 existing = self.find_existing_download(song, subdir)
1265 if existing is not None:
1266 return self._skip_existing(song, existing, progress_cb)
1267 return self._downloaded(
1268 song,
1269 self._fetch_and_tag(song, video_id, progress_cb, subdir, provider),
1270 provider,
1271 source_id=video_id,
1272 )
1274 def _with_album_override(self, song: dict[str, Any]) -> dict[str, Any]:
1275 """*song* with what :attr:`album_override` says for its album."""
1277 hook = self.album_override
1278 if hook is None:
1279 return song
1280 try:
1281 override = hook(song)
1282 except Exception:
1283 logger.opt(exception=True).warning(
1284 'The album override failed for {!r}', song.get('name')
1285 )
1286 return song
1287 return {**song, **override} if override else song
1289 def _downloaded(
1290 self,
1291 song: dict[str, Any],
1292 filename: str,
1293 provider: Optional[str] = None,
1294 source: Optional[Path] = None,
1295 source_id: Optional[str] = None,
1296 ) -> str:
1297 """Tell :attr:`on_downloaded` that *filename* is new (which
1298 *provider* fetched it, the *source* file it was copied from, if
1299 any, and the upload - *source_id*, a YouTube video id or a
1300 SoundCloud track URL); returns it. A hook that fails is logged and
1301 ignored: the file is already there."""
1303 song = {**song, 'download_provider': provider or ''}
1304 if source is not None:
1305 song['download_source'] = str(source)
1306 if source_id:
1307 song['download_source_id'] = str(source_id)
1308 self._record_source(song, filename, source_id)
1309 hook = self.on_downloaded
1310 if hook is not None:
1311 try:
1312 hook(song, filename)
1313 except Exception:
1314 logger.opt(exception=True).warning(
1315 'The after-download hook failed for {}', filename
1316 )
1317 self._report_outcome(song, OUTCOME_COMPLETED, filename)
1318 return filename
1320 def _rejected_sources(self, song: dict[str, Any]) -> set[str]:
1321 """Uploads rejected for *song* (best effort: none on error)."""
1323 sources = getattr(self, 'download_sources', None)
1324 if sources is None:
1325 return set()
1326 try:
1327 return sources.rejected_for(song)
1328 except Exception:
1329 logger.opt(exception=True).debug('Rejected-source lookup failed')
1330 return set()
1332 def _record_source(
1333 self, song: dict[str, Any], filename: str, source_id: Optional[str]
1334 ) -> None:
1335 """Remember the upload *filename* was written from, and what its
1336 audio is, so a later swap on disk is noticed (best effort)."""
1338 sources = getattr(self, 'download_sources', None)
1339 if sources is None or not filename:
1340 return
1341 full = self.download_dir / filename
1342 if not full.is_file():
1343 # Left in place in slskd's folder: not a file Downtify wrote.
1344 return
1345 try:
1346 sources.record(filename, full, source_id or '', track_keys(song))
1347 except Exception:
1348 logger.opt(exception=True).debug(
1349 'Could not record the source of {}', filename
1350 )
1352 def _report_outcome(
1353 self, song: dict[str, Any], outcome: str, filename: str = ''
1354 ) -> None:
1355 """Tell :attr:`on_outcome` how a download ended (best effort)."""
1357 hook = self.on_outcome
1358 miss = getattr(_SLSKD_MISS, 'value', None)
1359 _SLSKD_MISS.value = None
1360 if hook is None:
1361 return
1362 if miss is not None:
1363 # slskd was asked first and had nothing: say why.
1364 song = {**song, 'slskd_miss': miss}
1365 try:
1366 hook(song, outcome, filename)
1367 except Exception:
1368 logger.opt(exception=True).warning(
1369 'The download outcome hook failed for {}', song.get('name')
1370 )
1372 def _target_location(
1373 self, song: dict[str, Any], subdir: Optional[str]
1374 ) -> tuple[Path, str, str]:
1375 """``(target_dir, rel_prefix, basename)`` for a new file of ``song``.
1377 Applies the organize-by-artist/album folders and any folders the
1378 output template adds, and creates the directory.
1379 """
1381 output_parts = self._format_output_parts(song)
1382 basename = output_parts[-1]
1383 output_subdir = (
1384 Path(*output_parts[:-1]) if len(output_parts) > 1 else None
1385 )
1386 effective_subdir = self._effective_subdir(song, subdir)
1387 target_dir, rel_prefix = self._resolve_target_dir(effective_subdir)
1388 if output_subdir is not None:
1389 target_dir /= output_subdir
1390 rel_prefix = f'{rel_prefix}{output_subdir.as_posix()}/'
1391 target_dir.mkdir(parents=True, exist_ok=True)
1392 return target_dir, rel_prefix, basename
1394 def _tag_file(
1395 self,
1396 final_path: Path,
1397 target_dir: Path,
1398 song: dict[str, Any],
1399 found: _LookupResults,
1400 *,
1401 save_album_cover: bool = True,
1402 ) -> bool:
1403 """Embed metadata, album cover.jpg and lyrics into ``final_path``."""
1405 succeeded = True
1406 if found.genre:
1407 song = {**song, 'genre': found.genre}
1409 try:
1410 embed_metadata(
1411 final_path,
1412 song,
1413 download_cover=self.download_cover_art,
1414 cover_bytes=found.cover_bytes,
1415 )
1416 except Exception:
1417 succeeded = False
1418 logger.exception('Failed to embed metadata into {}', final_path)
1420 if self.download_cover_art and save_album_cover:
1421 try:
1422 self._save_album_cover(target_dir, song, found.cover_bytes)
1423 except Exception:
1424 succeeded = False
1425 logger.exception(
1426 'Failed to write album cover in {}', target_dir
1427 )
1429 if found.lyrics is not None:
1430 try:
1431 embed_lyrics(
1432 final_path,
1433 found.lyrics,
1434 embed_synced=self.lyrics_synced_only,
1435 )
1436 except Exception:
1437 succeeded = False
1438 logger.exception('Failed to embed lyrics into {}', final_path)
1440 # The recording's own ID (see downtify.isrc); a file without one
1441 # is no less a download.
1442 if found.isrc:
1443 isrc_mod.write(final_path, found.isrc)
1444 return succeeded
1446 def _remove_slskd_source_after_import(
1447 self, source_path: Path, final_path: Path
1448 ) -> None:
1449 """Remove a completed slskd source only inside its configured root."""
1451 if source_path.is_symlink():
1452 logger.warning(
1453 'slskd: source cleanup skipped for symlink source={!r}',
1454 source_path,
1455 )
1456 return
1457 try:
1458 source = source_path.resolve(strict=True)
1459 destination = final_path.resolve(strict=True)
1460 source_root = Path(
1461 str(self.slskd_settings.get('source_dir') or '')
1462 ).resolve(strict=True)
1463 except OSError as exc:
1464 logger.warning(
1465 'slskd: source cleanup skipped; paths could not be verified '
1466 'source={!r} destination={!r} err={}',
1467 source_path,
1468 final_path,
1469 exc,
1470 )
1471 return
1472 if source == destination:
1473 return
1474 if not source.is_relative_to(source_root) or not source.is_file():
1475 logger.warning(
1476 'slskd: source cleanup skipped; file is outside the configured '
1477 'source folder source={!r} source_dir={!r}',
1478 source,
1479 source_root,
1480 )
1481 return
1482 try:
1483 source.unlink()
1484 except OSError as exc:
1485 logger.warning(
1486 'slskd: imported file is ready but source cleanup failed '
1487 'source={!r} destination={!r} err={}',
1488 source,
1489 destination,
1490 exc,
1491 )
1492 return
1493 logger.info(
1494 'slskd: removed source after successful import source={!r} '
1495 'destination={!r}',
1496 source,
1497 destination,
1498 )
1500 def _finalize_local_source(
1501 self,
1502 song: dict[str, Any],
1503 source_path: Path,
1504 provider: Optional[str],
1505 progress_cb: Optional[ProgressCallback],
1506 subdir: Optional[str],
1507 ) -> tuple[str, Optional[Path]]:
1508 """Tag a file a provider already downloaded (slskd) and return
1509 ``(its library path, the file it was copied from or None)``.
1511 With slskd's ``leave_in_place`` the file stays under the slskd
1512 folder and is returned with the virtual ``slskd/`` prefix (served
1513 via ``/media/slskd/...``); otherwise it's copied into the same
1514 destination a YouTube download would use, keeping its original
1515 format — slskd files are not transcoded — and the source is removed
1516 only after the imported file has been tagged and verified.
1517 """
1519 if provider == 'deemix':
1520 stored = library_stored_path(
1521 source_path,
1522 self.download_dir,
1523 slskd_dir_from_downloader(self),
1524 [Path(self.deemix_settings['source_dir'])],
1525 )
1526 _report(progress_cb, 100.0, 'Done', provider)
1527 return stored, None
1529 copied_from: Optional[Path] = None
1530 cleanup_source: Optional[Path] = None
1531 lookups = _MetadataLookups(self, song)
1532 try:
1533 if provider == 'slskd' and self.slskd_settings.get(
1534 'leave_in_place', True
1535 ):
1536 final_path = source_path
1537 target_dir = source_path.parent
1538 stored = library_stored_path(
1539 final_path,
1540 self.download_dir,
1541 slskd_dir_from_downloader(self),
1542 )
1543 in_place = True
1544 else:
1545 target_dir, rel_prefix, basename = self._target_location(
1546 song, subdir
1547 )
1548 suffix = source_path.suffix or f'.{self.audio_format}'
1549 final_path = target_dir / f'{basename}{suffix}'
1550 if source_path.resolve() != final_path.resolve():
1551 shutil.copy2(source_path, final_path)
1552 copied_from = source_path.resolve()
1553 if provider == 'slskd':
1554 cleanup_source = source_path
1555 stored = f'{rel_prefix}{final_path.name}'
1556 in_place = False
1557 found = lookups.collect()
1558 finally:
1559 lookups.cancel()
1560 if provider == 'slskd' and self.slskd_settings.get('keep_file_cover'):
1561 found = _keep_larger_file_cover(found, source_path)
1563 # Never drop a cover.jpg into slskd's own folders.
1564 tagged = self._tag_file(
1565 final_path,
1566 target_dir,
1567 song,
1568 found,
1569 save_album_cover=not in_place,
1570 )
1571 if cleanup_source is not None:
1572 if not tagged:
1573 logger.warning(
1574 'slskd: retaining source because tagging failed '
1575 'source={!r} destination={!r}',
1576 cleanup_source,
1577 final_path,
1578 )
1579 raise RuntimeError(
1580 'Could not tag imported slskd file; source retained'
1581 )
1582 if not _valid_imported_audio(final_path):
1583 logger.warning(
1584 'slskd: retaining source because imported file failed '
1585 'audio verification source={!r} destination={!r}',
1586 cleanup_source,
1587 final_path,
1588 )
1589 raise RuntimeError(
1590 'Imported slskd file failed verification; source retained'
1591 )
1592 self._remove_slskd_source_after_import(cleanup_source, final_path)
1593 _report(progress_cb, 100.0, 'Done', provider)
1594 return stored, copied_from
1596 @staticmethod
1597 def _progress_hook(
1598 progress_cb: Optional[ProgressCallback], provider: Optional[str]
1599 ) -> Callable[[dict[str, Any]], None]:
1600 """A yt-dlp progress hook forwarding to *progress_cb*.
1602 yt-dlp calls the hook for every downloaded chunk, and each report
1603 becomes a WebSocket broadcast scheduled on the event loop — only
1604 forward it when the whole-number percentage actually changes.
1605 """
1607 last_reported_pct = -1
1609 def hook(data: dict[str, Any]) -> None:
1610 nonlocal last_reported_pct
1611 if progress_cb is None:
1612 return
1613 try:
1614 status = data.get('status')
1615 if status == 'downloading':
1616 total = (
1617 data.get('total_bytes')
1618 or data.get('total_bytes_estimate')
1619 or 0
1620 )
1621 downloaded = data.get('downloaded_bytes') or 0
1622 if total:
1623 pct = min(95.0, downloaded / total * 95.0)
1624 if int(pct) != last_reported_pct:
1625 last_reported_pct = int(pct)
1626 progress_cb(pct, 'Downloading', provider)
1627 elif status == 'finished':
1628 progress_cb(96.0, 'Converting', provider)
1629 except Exception:
1630 logger.opt(exception=True).debug('progress hook error')
1632 return hook
1634 def _ydl_options(
1635 self, out_template: str, hook: Callable[[dict[str, Any]], None]
1636 ) -> tuple[dict[str, Any], bool]:
1637 """``(yt-dlp options, whether cookies are used)`` for one audio
1638 download to *out_template*."""
1640 ydl_opts = {
1641 'format': 'bestaudio/best',
1642 'outtmpl': out_template,
1643 'quiet': True,
1644 'noprogress': True,
1645 'logger': _YtdlpLogger(),
1646 'noplaylist': True,
1647 'nocheckcertificate': True,
1648 'overwrites': True,
1649 'progress_hooks': [hook],
1650 # Resilience against flaky DNS/network in containers.
1651 # googlevideo.com CDN hosts are short-lived shards and a single
1652 # transient EAI_AGAIN/timeout used to abort the whole download.
1653 'retries': 10,
1654 'fragment_retries': 10,
1655 'extractor_retries': 3,
1656 'socket_timeout': 30,
1657 # The default `web` player_client is the one most aggressively
1658 # gated by YouTube's "Sign in to confirm you're not a bot"
1659 # check on datacenter IPs. `tv` and `mweb` almost always
1660 # bypass it. Order matters — yt-dlp tries them in sequence.
1661 'extractor_args': {
1662 'youtube': {'player_client': _yt_player_clients()}
1663 },
1664 # `web`/`web_embedded` need a JS runtime to solve YouTube's
1665 # signature/n-challenges (see the comment on
1666 # _DEFAULT_YT_PLAYER_CLIENTS above) — that's the only real
1667 # fallback once `ios`/`android` get SABR-gated. yt-dlp
1668 # refuses to fetch its EJS challenge-solver script by
1669 # default, so without this the JS runtime never actually
1670 # gets used and those two clients silently yield no audio
1671 # (see henriquesebastiao/downtify#247).
1672 'remote_components': ['ejs:github'],
1673 # Light pacing so we don't trigger 429 rate limits when the
1674 # user fires off multiple downloads back-to-back.
1675 'sleep_interval_requests': 1,
1676 }
1677 # Many container setups have IPv6 advertised but unroutable for
1678 # googlevideo.com, which surfaces as EAI_AGAIN on the AAAA lookup.
1679 # Setting DOWNTIFY_FORCE_IPV4=1 binds yt-dlp to IPv4 only.
1680 if os.getenv('DOWNTIFY_FORCE_IPV4', '').strip() in {
1681 '1',
1682 'true',
1683 'yes',
1684 }:
1685 ydl_opts['source_address'] = '0.0.0.0'
1687 # Cookies authenticate yt-dlp as a real browser session — needed
1688 # for age-restricted (explicit) tracks and whenever YouTube
1689 # challenges the download. The file comes either from
1690 # DOWNTIFY_COOKIES_FILE or from the settings UI upload, resolved
1691 # by the store in that order; DOWNTIFY_COOKIES_FROM_BROWSER takes
1692 # "<browser>" or "<browser>:<profile>" (e.g. "firefox" or
1693 # "chrome:Default").
1694 cookies_file = self._resolve_cookies_file()
1695 if cookies_file:
1696 ydl_opts['cookiefile'] = cookies_file
1697 cookies_browser = os.getenv(
1698 'DOWNTIFY_COOKIES_FROM_BROWSER', ''
1699 ).strip()
1700 if cookies_browser:
1701 parts = cookies_browser.split(':', 1)
1702 ydl_opts['cookiesfrombrowser'] = (
1703 (parts[0],) if len(parts) == 1 else (parts[0], parts[1])
1704 )
1706 return ydl_opts, bool(cookies_file or cookies_browser)
1708 @staticmethod
1709 def _private_cookiefile(
1710 ydl_opts: dict[str, Any],
1711 ) -> contextlib.AbstractContextManager[None]:
1712 """Per-download copy of the cookie jar, rotated cookies merged
1713 back afterwards - see :func:`downtify.cookies.private_cookiefile`."""
1715 return private_cookiefile(ydl_opts)
1717 def fetch_audio(
1718 self,
1719 video_id: str,
1720 target_dir: Path,
1721 basename: str,
1722 *,
1723 song: dict[str, Any],
1724 progress_cb: Optional[ProgressCallback] = None,
1725 provider: Optional[str] = None,
1726 audio_format: Optional[str] = None,
1727 ) -> Path:
1728 """Download YouTube video *video_id*'s audio - or, when it is a
1729 SoundCloud track URL, that track's - as
1730 ``<target_dir>/<basename>.<audio_format>`` (default: the chosen
1731 format) and return the file - untagged. *song* only words the
1732 error when it fails."""
1734 fmt = audio_format or self.audio_format
1735 extract_format = 'best' if fmt == 'original' else fmt
1736 extract_bitrate = '' if fmt == 'original' else self.audio_bitrate
1737 out_template = str(target_dir / f'{basename}.%(ext)s')
1738 hook = self._progress_hook(progress_cb, provider)
1739 ydl_opts, has_cookies = self._ydl_options(out_template, hook)
1740 soundcloud = parse_soundcloud_url(video_id)
1741 if soundcloud is not None:
1742 url = soundcloud[1]
1743 # Without SoundCloud Go+ a Go+ track only streams a 30-second
1744 # preview: fail rather than file the preview as the song.
1745 ydl_opts['format'] = (
1746 'bestaudio[format_id!*=preview]/best[format_id!*=preview]'
1747 )
1748 else:
1749 url = f'https://music.youtube.com/watch?v={video_id}'
1750 try:
1751 with (
1752 self._private_cookiefile(ydl_opts),
1753 yt_dlp.YoutubeDL(ydl_opts) as ydl,
1754 ):
1755 # Registered directly: `ydl_opts['postprocessors']` only
1756 # accepts yt-dlp's built-in postprocessors by name.
1757 ydl.add_post_processor(
1758 _ExtractAudioPP(ydl, extract_format, extract_bitrate),
1759 when='post_process',
1760 )
1761 ydl.download([url])
1762 except Exception as exc:
1763 if soundcloud is not None and _is_missing_format_error(exc):
1764 raise RuntimeError(
1765 f'"{song.get("name") or "This track"}" only streams a '
1766 '30-second preview on SoundCloud (a SoundCloud Go+ '
1767 'track).'
1768 ) from exc
1769 raise _translate_download_error(
1770 exc, song, has_cookies=has_cookies
1771 ) from exc
1773 final_path = target_dir / f'{basename}.{fmt}'
1774 if not final_path.exists():
1775 # yt-dlp sometimes uses the upstream extension for opus/m4a
1776 for candidate in target_dir.glob(f'{basename}.*'):
1777 if candidate.is_file():
1778 final_path = candidate
1779 break
1780 return final_path
1782 def _fetch_and_tag(
1783 self,
1784 song: dict[str, Any],
1785 video_id: str,
1786 progress_cb: Optional[ProgressCallback],
1787 subdir: Optional[str],
1788 provider: Optional[str] = None,
1789 ) -> str:
1790 target_dir, rel_prefix, basename = self._target_location(song, subdir)
1792 # Genre, cover art and lyrics are looked up while yt-dlp downloads,
1793 # instead of one after another once it's done.
1794 lookups = _MetadataLookups(self, song)
1795 try:
1796 final_path = self.fetch_audio(
1797 video_id,
1798 target_dir,
1799 basename,
1800 song=song,
1801 progress_cb=progress_cb,
1802 provider=provider,
1803 )
1804 found = lookups.collect()
1805 finally:
1806 # A failed download shouldn't wait on lookups nobody will use.
1807 lookups.cancel()
1809 self._tag_file(final_path, target_dir, song, found)
1810 _report(progress_cb, 100.0, 'Done', provider)
1811 return f'{rel_prefix}{final_path.name}'
1814class _LookupResults:
1815 """What :class:`_MetadataLookups` found (``None`` = nothing / failed)."""
1817 def __init__(
1818 self,
1819 genre: Optional[str],
1820 cover_bytes: Optional[bytes],
1821 lyrics: Any,
1822 isrc: Optional[str] = None,
1823 ) -> None:
1824 self.genre = genre
1825 self.cover_bytes = cover_bytes
1826 self.lyrics = lyrics
1827 self.isrc = isrc
1830class _MetadataLookups:
1831 """Genre, cover-art, lyrics and ISRC lookups for one song, run in
1832 parallel.
1834 They depend only on the song's metadata, not on the audio file, so
1835 callers start them while the audio itself is being fetched. The cover
1836 is fetched once and reused for the embedded art and cover.jpg.
1837 """
1839 def __init__(self, downloader: Downloader, song: dict[str, Any]) -> None:
1840 self._song = song
1841 self._pool = ThreadPoolExecutor(
1842 max_workers=4, thread_name_prefix='downtify-metadata'
1843 )
1844 self._isrc = (
1845 self._pool.submit(isrc_mod.lookup, song)
1846 if getattr(downloader, 'write_isrc', True)
1847 else None
1848 )
1849 self._genre = (
1850 None
1851 if song.get('genre')
1852 else self._pool.submit(_fetch_itunes_genre, song)
1853 )
1854 self._cover = (
1855 self._pool.submit(
1856 _download_cover_meeting,
1857 song,
1858 cover_goal(getattr(downloader, 'cover_min_resolution', 0)),
1859 )
1860 if downloader.download_cover_art
1861 else None
1862 )
1863 self._lyrics = (
1864 self._pool.submit(
1865 # The keyword only when it's on: the call stays as it was.
1866 functools.partial(lyrics_mod.fetch, synced_only=True)
1867 if getattr(downloader, 'lyrics_synced_only', False)
1868 else lyrics_mod.fetch,
1869 song,
1870 downloader.lyrics_providers,
1871 downloader.lyrics_cache,
1872 )
1873 if downloader.lyrics_providers
1874 else None
1875 )
1877 def collect(self) -> _LookupResults:
1878 return _LookupResults(
1879 genre=_lookup_result(
1880 self._genre, 'iTunes genre lookup', self._song
1881 ),
1882 cover_bytes=_lookup_result(
1883 self._cover, 'Cover download', self._song
1884 ),
1885 lyrics=_lookup_result(
1886 self._lyrics, 'Lyrics fetch', self._song, error=True
1887 ),
1888 isrc=_lookup_result(self._isrc, 'ISRC lookup', self._song),
1889 )
1891 def cancel(self) -> None:
1892 self._pool.shutdown(wait=False, cancel_futures=True)
1895def _report(
1896 progress_cb: Optional[ProgressCallback],
1897 pct: float,
1898 message: str,
1899 provider: Optional[str] = None,
1900) -> None:
1901 if progress_cb is not None:
1902 progress_cb(pct, message, provider)
1905def _lookup_result(
1906 future: Optional[Future],
1907 label: str,
1908 song: dict[str, Any],
1909 *,
1910 error: bool = False,
1911) -> Any:
1912 """Result of one of ``_fetch_and_tag``'s concurrent metadata lookups.
1914 A failed lookup never fails the download: it's logged (at ERROR when
1915 ``error`` is set, DEBUG otherwise) and treated as "nothing found".
1916 """
1918 if future is None:
1919 return None
1920 try:
1921 return future.result()
1922 except Exception:
1923 log = logger.opt(exception=True)
1924 (log.error if error else log.debug)(
1925 '{} failed for {!r}', label, song.get('name')
1926 )
1927 return None
1930def _download_cover(url: str) -> Optional[bytes]:
1931 """The cover at *url*. A small Spotify cover (300 or 64 px) is asked
1932 for at Spotify's 640 px size first, and at *url* as given only when
1933 that fails."""
1935 if not url:
1936 return None
1937 largest = spotify_mod.largest_cover_url(url)
1938 if largest != url:
1939 data = _fetch_cover(largest)
1940 if data:
1941 return data
1942 return _fetch_cover(url)
1945def _fetch_cover(url: str) -> Optional[bytes]:
1946 try:
1947 response = httpx.get(url, timeout=15)
1948 response.raise_for_status()
1949 except Exception:
1950 logger.opt(exception=True).warning('Failed to fetch cover art {}', url)
1951 return None
1952 return response.content
1955def cover_goal(minimum: int) -> int:
1956 """The cover size a download aims for: the "Cover size" setting
1957 (1200px by default), or the "Minimum cover size" when larger. A
1958 track's own cover below it is swapped for the largest the cover
1959 sources have (:func:`_download_cover_meeting`) - Spotify's largest
1960 is 640px, so a Spotify track's cover is always looked for."""
1962 return max(int(minimum or 0), cover_resolution())
1965def _download_cover_meeting(
1966 song: dict[str, Any], minimum: int, *, largest: bool = False
1967) -> Optional[bytes]:
1968 """The track's cover, or a larger one when it is under *minimum*.
1970 The cover the song's source offers (``song['cover_url']``) is used
1971 as long as its short side reaches *minimum* pixels (0 = always).
1972 Below that, Spotify, iTunes and YouTube Music are asked for the same
1973 release's art (:func:`downtify.cover_sources.best_cover`) and the
1974 largest image wins - the minimum is a floor, not a fixed size. When
1975 no source reaches it, the largest found is still embedded rather
1976 than none.
1978 With *largest* (the "Always use the largest cover" setting) those
1979 sources are asked on every download, whatever the minimum: a YouTube
1980 Music cover tops out around 1200 px and a Spotify one at 640 px,
1981 while iTunes usually has the same release at 1200 px.
1982 """
1984 data = _download_cover(song.get('cover_url', ''))
1985 if minimum <= 0 and not largest:
1986 return data
1987 have = image_short_side(data) if data else 0
1988 if have >= minimum and not largest:
1989 return data
1990 current = (
1991 cover_sources.CoverCandidate(source='download', data=data, width=have)
1992 if data
1993 else None
1994 )
1995 try:
1996 better = cover_sources.best_cover(song, current=current)
1997 except Exception:
1998 logger.opt(exception=True).info('Larger cover lookup failed')
1999 better = None
2000 if better is not None:
2001 data, have = better.data, better.width
2002 if have < minimum:
2003 logger.info(
2004 'No cover of at least {}px for {} - {}; using {}px',
2005 minimum,
2006 ', '.join(song.get('artists') or []),
2007 song.get('name', ''),
2008 have,
2009 )
2010 return data
2013def save_playlist_cover(cover_url: str, m3u_path: Path) -> Optional[Path]:
2014 """Save a playlist's cover art next to its M3U file.
2016 Writes ``<playlist-name>.jpg`` beside *m3u_path*, reusing whatever
2017 URL the caller resolved as the largest cover art the source
2018 (Spotify or YouTube Music) offers for that playlist — both always
2019 serve JPEG for these URLs, so the extension is fixed rather than
2020 sniffed from the response. Returns the written path, or ``None``
2021 when there's no cover URL, the fetch fails, or the file can't be
2022 written.
2023 """
2025 data = _download_cover(cover_url)
2026 if not data:
2027 return None
2028 cover_path = m3u_path.with_suffix('.jpg')
2029 try:
2030 # The cover can land before the first track, so the playlist's
2031 # folder may not exist yet.
2032 cover_path.parent.mkdir(parents=True, exist_ok=True)
2033 cover_path.write_bytes(data)
2034 except OSError:
2035 logger.opt(exception=True).warning(
2036 'Could not write playlist cover {}', cover_path
2037 )
2038 return None
2039 return cover_path
2042def _album_track_index_for_tags(
2043 song: dict[str, Any],
2044) -> tuple[Optional[int], Optional[int]]:
2045 """Normalize ``track_number`` / ``album_track_total`` for tagging frames."""
2046 raw_n = song.get('track_number')
2047 raw_tot = song.get('album_track_total')
2048 try:
2049 n = int(raw_n)
2050 except (TypeError, ValueError):
2051 return None, None
2052 if n <= 0:
2053 return None, None
2054 tot: Optional[int] = None
2055 if raw_tot is not None and raw_tot != '':
2056 try:
2057 t = int(raw_tot)
2058 except (TypeError, ValueError):
2059 pass
2060 else:
2061 if t > 0:
2062 tot = t
2063 return n, tot
2066def _recording_date_for_tags(song: dict[str, Any]) -> str:
2067 """Prefer full ``YYYY-MM-DD`` from Spotify; fall back to year-only."""
2069 rd = str(song.get('release_date') or '').strip()
2070 if rd:
2071 return rd
2072 return str(song.get('year') or '').strip()
2075def _album_artist_for_tags(song: dict[str, Any]) -> Optional[str]:
2076 """The album-artist tag: the album artist the source declares (the
2077 same on every track of an album, which media servers group albums by),
2078 else the track's first artist. Never inferred from how many artists a
2079 track has - a guest on one track doesn't make an album a compilation;
2080 "Various Artists" only comes from the source (see ``album_artist.py``).
2081 """
2083 if song.get('compilation'):
2084 return VARIOUS_ARTISTS
2085 album_artist = str(song.get('album_artist') or '').strip()
2086 if album_artist:
2087 return album_artist
2088 artists = song.get('artists') or []
2089 first = str(artists[0]).strip() if artists else ''
2090 return first or None
2093def _release_type_for_tags(song: dict[str, Any]) -> str:
2094 """YouTube Music's release classification ('album'/'single'/'ep'),
2095 lower-cased to match the MusicBrainz Picard tagging convention used
2096 by most taggers/media servers (``MusicBrainz Album Type`` / Vorbis
2097 ``RELEASETYPE`` / the MP4 freeform equivalent)."""
2099 return str(song.get('release_type') or '').strip().lower()
2102def _keep_larger_file_cover(
2103 found: '_LookupResults', source: Path
2104) -> '_LookupResults':
2105 """*found* with the cover the downloaded file came with, when that
2106 one is larger than the fetched cover (the slskd ``keep_file_cover``
2107 setting): shared files often carry the release's full-size artwork,
2108 larger than YouTube Music's or Spotify's. A file without a cover, or
2109 with a smaller or unreadable one, changes nothing."""
2111 try:
2112 data, _mime = extract_embedded_cover(source)
2113 except Exception:
2114 logger.opt(exception=True).debug(
2115 'Could not read the cover of {}', source
2116 )
2117 return found
2118 if not data:
2119 return found
2120 own = image_short_side(data)
2121 fetched = image_short_side(found.cover_bytes) if found.cover_bytes else 0
2122 if own <= fetched:
2123 return found
2124 logger.info(
2125 "Keeping the file's own {}px cover over the fetched {}px one: {}",
2126 own,
2127 fetched,
2128 source.name,
2129 )
2130 found.cover_bytes = data
2131 return found
2134def embed_metadata(
2135 path: Path,
2136 song: dict[str, Any],
2137 *,
2138 download_cover: bool = True,
2139 cover_bytes: Optional[bytes] = None,
2140) -> None:
2141 if not path.exists():
2142 return
2144 title = song.get('name', '')
2145 artists = song.get('artists') or []
2146 album_artist = _album_artist_for_tags(song)
2147 # Only ever added, never removed: a file re-tagged by the library
2148 # upgrade (whose song has no `compilation`) keeps its flag.
2149 compilation = bool(song.get('compilation'))
2150 album = song.get('album_name', '') or ''
2151 recording_date = _recording_date_for_tags(song)
2152 genre = (song.get('genre') or '').strip()
2153 release_type = _release_type_for_tags(song)
2154 # Only ever added: a file is never un-flagged by a song that doesn't say.
2155 explicit = bool(song.get('explicit'))
2156 if not download_cover:
2157 cover_bytes = None
2158 elif cover_bytes is None:
2159 cover_bytes = _download_cover(song.get('cover_url', ''))
2160 track_number, album_track_total = _album_track_index_for_tags(song)
2161 if track_number is None:
2162 logger.info(
2163 'Tag embed: no track_number/disc position for file={} '
2164 'song_id={} title={!r} raw_track_number={!r} raw_total={!r}',
2165 path.name,
2166 song.get('song_id'),
2167 title,
2168 song.get('track_number'),
2169 song.get('album_track_total'),
2170 )
2171 if not recording_date:
2172 logger.info(
2173 'Tag embed: no recording date (year/release_date) for file={} '
2174 'song_id={} title={!r} raw_year={!r} raw_release_date={!r}',
2175 path.name,
2176 song.get('song_id'),
2177 title,
2178 song.get('year'),
2179 song.get('release_date'),
2180 )
2181 logger.debug(
2182 'Tag embed summary: {} track={}/{} date={!r}',
2183 path.name,
2184 track_number,
2185 album_track_total,
2186 recording_date,
2187 )
2189 suffix = path.suffix.lower().lstrip('.')
2191 if suffix == 'mp3':
2192 _tag_mp3(
2193 path,
2194 title,
2195 artists,
2196 album_artist,
2197 album,
2198 recording_date,
2199 genre,
2200 cover_bytes,
2201 track_number,
2202 album_track_total,
2203 release_type,
2204 explicit,
2205 compilation=compilation,
2206 )
2207 elif suffix in {'m4a', 'mp4', 'aac'}:
2208 _tag_mp4(
2209 path,
2210 title,
2211 artists,
2212 album_artist,
2213 album,
2214 recording_date,
2215 genre,
2216 cover_bytes,
2217 track_number,
2218 album_track_total,
2219 release_type,
2220 explicit,
2221 compilation=compilation,
2222 )
2223 elif suffix == 'flac':
2224 _tag_flac(
2225 path,
2226 title,
2227 artists,
2228 album_artist,
2229 album,
2230 recording_date,
2231 genre,
2232 cover_bytes,
2233 track_number,
2234 album_track_total,
2235 release_type,
2236 explicit,
2237 compilation=compilation,
2238 )
2239 elif suffix in {'ogg', 'oga'}:
2240 _tag_ogg_vorbis(
2241 path,
2242 title,
2243 artists,
2244 album_artist,
2245 album,
2246 recording_date,
2247 genre,
2248 cover_bytes,
2249 track_number,
2250 album_track_total,
2251 release_type,
2252 explicit,
2253 compilation=compilation,
2254 )
2255 elif suffix == 'opus':
2256 _tag_opus(
2257 path,
2258 title,
2259 artists,
2260 album_artist,
2261 album,
2262 recording_date,
2263 genre,
2264 cover_bytes,
2265 track_number,
2266 album_track_total,
2267 release_type,
2268 explicit,
2269 compilation=compilation,
2270 )
2273def _tag_mp3(
2274 path: Path,
2275 title: str,
2276 artists: list[str],
2277 album_artist: Optional[str],
2278 album: str,
2279 year: str,
2280 genre: str,
2281 cover_bytes: Optional[bytes],
2282 track_number: Optional[int],
2283 album_track_total: Optional[int],
2284 release_type: str = '',
2285 explicit: bool = False,
2286 *,
2287 compilation: bool = False,
2288) -> None:
2289 audio = MP3(str(path), ID3=ID3)
2290 if audio.tags is None:
2291 audio.add_tags()
2292 audio.tags.delall('APIC')
2293 audio.tags.add(TIT2(encoding=3, text=title))
2294 if artists:
2295 audio.tags.add(TPE1(encoding=3, text='; '.join(artists)))
2296 # One value per artist (MusicBrainz Picard's ARTISTS), so readers
2297 # never have to split "Earth, Wind & Fire" on its comma.
2298 audio.tags.add(TXXX(encoding=3, desc='ARTISTS', text=list(artists)))
2299 if album_artist:
2300 audio.tags.add(TPE2(encoding=3, text=album_artist))
2301 if compilation:
2302 audio.tags.add(TCMP(encoding=3, text='1'))
2303 if album:
2304 audio.tags.add(TALB(encoding=3, text=album))
2305 if track_number is not None:
2306 trck = (
2307 f'{track_number}/{album_track_total}'
2308 if album_track_total is not None
2309 else str(track_number)
2310 )
2311 audio.tags.add(TRCK(encoding=3, text=trck))
2312 # Downtify never handles multi-disc releases, so this is always "1" -
2313 # but writing it is not just cosmetic: media servers/taggers that
2314 # read a missing disc tag as 0 (rather than defaulting to 1) will
2315 # otherwise fail to match this track against the same recording
2316 # tagged by another source, since disc position is compared
2317 # alongside track position.
2318 audio.tags.add(TPOS(encoding=3, text='1'))
2319 if year:
2320 audio.tags.add(TDRC(encoding=3, text=year))
2321 if genre:
2322 audio.tags.add(TCON(encoding=3, text=genre))
2323 if release_type:
2324 audio.tags.delall('TXXX:MusicBrainz Album Type')
2325 audio.tags.add(
2326 TXXX(
2327 encoding=3,
2328 desc='MusicBrainz Album Type',
2329 text=release_type,
2330 )
2331 )
2332 if explicit:
2333 audio.tags.delall(f'TXXX:{ADVISORY_TAG}')
2334 audio.tags.add(
2335 TXXX(encoding=3, desc=ADVISORY_TAG, text=ADVISORY_EXPLICIT)
2336 )
2337 if cover_bytes:
2338 audio.tags.add(
2339 APIC(
2340 encoding=3,
2341 mime='image/jpeg',
2342 type=3,
2343 desc='Cover',
2344 data=cover_bytes,
2345 )
2346 )
2347 # ID3v2.3 has no UTF-8 text encoding (only Latin-1/UTF-16), so saving
2348 # v2.3 frames created with encoding=3 forces mutagen to transcode them
2349 # to UTF-16 on write. That conversion corrupts the tail of the frame
2350 # (observed: the last 1-2 characters replaced with garbage code
2351 # points), which then desyncs the ID3 reader for every frame that
2352 # follows TIT2 — silently dropping artist/album/genre/date/track
2353 # number/cover on read. v2.4 supports UTF-8 natively, so no forced
2354 # transcode happens.
2355 audio.save(v2_version=4)
2358def _tag_mp4(
2359 path: Path,
2360 title: str,
2361 artists: list[str],
2362 album_artist: Optional[str],
2363 album: str,
2364 year: str,
2365 genre: str,
2366 cover_bytes: Optional[bytes],
2367 track_number: Optional[int],
2368 album_track_total: Optional[int],
2369 release_type: str = '',
2370 explicit: bool = False,
2371 *,
2372 compilation: bool = False,
2373) -> None:
2374 audio = MP4(str(path))
2375 audio['\xa9nam'] = title
2376 if artists:
2377 audio['\xa9ART'] = artists
2378 audio['----:com.apple.iTunes:ARTISTS'] = [
2379 MP4FreeForm(a.encode('utf-8')) for a in artists
2380 ]
2381 if album_artist:
2382 audio['aART'] = [album_artist]
2383 if compilation:
2384 audio['cpil'] = True
2385 if album:
2386 audio['\xa9alb'] = album
2387 if track_number is not None:
2388 total = album_track_total if album_track_total is not None else 0
2389 audio['trkn'] = [(track_number, total)]
2390 # see the matching comment in _tag_mp3 for why disc=1 is always written
2391 audio['disk'] = [(1, 0)]
2392 if year:
2393 audio['\xa9day'] = year
2394 if genre:
2395 audio['\xa9gen'] = genre
2396 if release_type:
2397 audio['----:com.apple.iTunes:MusicBrainz Album Type'] = [
2398 MP4FreeForm(release_type.encode('utf-8'))
2399 ]
2400 if explicit:
2401 audio[MP4_ADVISORY_ATOM] = [int(ADVISORY_EXPLICIT)]
2402 if cover_bytes:
2403 audio['covr'] = [
2404 MP4Cover(cover_bytes, imageformat=MP4Cover.FORMAT_JPEG)
2405 ]
2406 audio.save()
2409def _tag_flac(
2410 path: Path,
2411 title: str,
2412 artists: list[str],
2413 album_artist: Optional[str],
2414 album: str,
2415 year: str,
2416 genre: str,
2417 cover_bytes: Optional[bytes],
2418 track_number: Optional[int],
2419 album_track_total: Optional[int],
2420 release_type: str = '',
2421 explicit: bool = False,
2422 *,
2423 compilation: bool = False,
2424) -> None:
2425 audio = FLAC(str(path))
2426 audio['title'] = title
2427 if artists:
2428 audio['artist'] = artists
2429 audio['artists'] = artists
2430 if album_artist:
2431 audio['albumartist'] = album_artist
2432 if compilation:
2433 audio['compilation'] = '1'
2434 if album:
2435 audio['album'] = album
2436 if track_number is not None:
2437 audio['tracknumber'] = str(track_number)
2438 if album_track_total is not None:
2439 audio['tracktotal'] = str(album_track_total)
2440 # see the matching comment in _tag_mp3 for why disc=1 is always written
2441 audio['discnumber'] = '1'
2442 if year:
2443 audio['date'] = year
2444 if genre:
2445 audio['genre'] = genre
2446 if release_type:
2447 audio['releasetype'] = release_type
2448 if explicit:
2449 audio[ADVISORY_TAG] = ADVISORY_EXPLICIT
2450 if cover_bytes:
2451 picture = Picture()
2452 picture.data = cover_bytes
2453 picture.type = 3
2454 picture.mime = 'image/jpeg'
2455 audio.clear_pictures()
2456 audio.add_picture(picture)
2457 audio.save()
2460def _tag_ogg_vorbis(
2461 path: Path,
2462 title: str,
2463 artists: list[str],
2464 album_artist: Optional[str],
2465 album: str,
2466 year: str,
2467 genre: str,
2468 cover_bytes: Optional[bytes],
2469 track_number: Optional[int],
2470 album_track_total: Optional[int],
2471 release_type: str = '',
2472 explicit: bool = False,
2473 *,
2474 compilation: bool = False,
2475) -> None:
2476 audio = OggVorbis(str(path))
2477 _apply_vorbis_comments(
2478 audio,
2479 title,
2480 artists,
2481 album_artist,
2482 album,
2483 year,
2484 genre,
2485 cover_bytes,
2486 track_number,
2487 album_track_total,
2488 release_type,
2489 explicit,
2490 compilation=compilation,
2491 )
2492 audio.save()
2495def _tag_opus(
2496 path: Path,
2497 title: str,
2498 artists: list[str],
2499 album_artist: Optional[str],
2500 album: str,
2501 year: str,
2502 genre: str,
2503 cover_bytes: Optional[bytes],
2504 track_number: Optional[int],
2505 album_track_total: Optional[int],
2506 release_type: str = '',
2507 explicit: bool = False,
2508 *,
2509 compilation: bool = False,
2510) -> None:
2511 audio = OggOpus(str(path))
2512 _apply_vorbis_comments(
2513 audio,
2514 title,
2515 artists,
2516 album_artist,
2517 album,
2518 year,
2519 genre,
2520 cover_bytes,
2521 track_number,
2522 album_track_total,
2523 release_type,
2524 explicit,
2525 compilation=compilation,
2526 )
2527 audio.save()
2530def _apply_vorbis_comments(
2531 audio,
2532 title,
2533 artists,
2534 album_artist,
2535 album,
2536 year,
2537 genre,
2538 cover_bytes: Optional[bytes],
2539 track_number: Optional[int],
2540 album_track_total: Optional[int],
2541 release_type: str = '',
2542 explicit: bool = False,
2543 *,
2544 compilation: bool = False,
2545):
2546 audio['title'] = title
2547 if artists:
2548 audio['artist'] = artists
2549 audio['artists'] = artists
2550 if album_artist:
2551 audio['albumartist'] = album_artist
2552 if compilation:
2553 audio['compilation'] = '1'
2554 if album:
2555 audio['album'] = album
2556 if track_number is not None:
2557 audio['TRACKNUMBER'] = str(track_number)
2558 if album_track_total is not None:
2559 audio['TRACKTOTAL'] = str(album_track_total)
2560 # see the matching comment in _tag_mp3 for why disc=1 is always written
2561 audio['DISCNUMBER'] = '1'
2562 if year:
2563 audio['date'] = year
2564 if release_type:
2565 audio['RELEASETYPE'] = release_type
2566 if explicit:
2567 audio[ADVISORY_TAG] = ADVISORY_EXPLICIT
2568 if genre:
2569 audio['genre'] = genre
2570 if cover_bytes:
2571 picture = Picture()
2572 picture.data = cover_bytes
2573 picture.type = 3
2574 picture.mime = 'image/jpeg'
2575 picture_data = picture.write()
2576 encoded_data = base64.b64encode(picture_data).decode('ascii')
2577 audio['metadata_block_picture'] = [encoded_data]
2580def embed_lyrics(
2581 path: Path,
2582 lyrics: 'lyrics_mod.Lyrics',
2583 *,
2584 sidecar: Optional[Path] = None,
2585 embed_synced: bool = False,
2586) -> None:
2587 """Embed plain lyrics into the audio tag and write a .lrc sidecar
2588 when synced lyrics are available; with *embed_synced*, the tag gets
2589 the synced text too."""
2591 lyrics_mod.write_to_file(
2592 path, lyrics, sidecar=sidecar, embed_synced=embed_synced
2593 )