Coverage for downtify/monitor.py: 93%
1019 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Periodic playlist monitoring and automatic downloading."""
3from __future__ import annotations
5import asyncio
6import contextlib
7import functools
8import os
9import re
10import sqlite3
11from collections.abc import AsyncIterator, Iterator
12from dataclasses import asdict, dataclass
13from datetime import datetime, time, timedelta, timezone
14from pathlib import Path
15from typing import Any, Callable, Optional
17from loguru import logger
19from . import deezer, m3u, musicbrainz, providers, spotify
20from .artist_catalogue import (
21 ArtistCatalogue,
22 release_day,
23 release_key,
24 upcoming,
25 watch_kinds,
26)
27from .artist_catalogue import wanted as release_wanted
28from .artist_release_store import ArtistReleaseStore
29from .cover_cache import CoverArtCache
30from .db.config import Dialect
31from .db.core import Connection, Database, as_database
32from .db.dialect import insert_returning_id
33from .download_policy import DownloadPolicy, DownloadPolicyBlocked
34from .download_queue import queue_pause
35from .downloader import (
36 DOWNLOAD_EXECUTOR,
37 Downloader,
38 ProgressCallback,
39 save_playlist_cover,
40)
41from .external_library import extra_dirs_from_settings
42from .file_naming import file_name_key
43from .library_cache_keys import file_content_key
44from .library_metadata_cache import LibraryMetadataCache
45from .library_only import enabled, policy
46from .library_paths import locate_library_file, slskd_dir_from_downloader
47from .library_paths_cache import invalidate_library_paths_cache
48from .library_songs import LibrarySongs, same_recording
49from .navidrome import (
50 _effective_navidrome_settings,
51 cache_navidrome_song_id,
52 enrich_song_from_library_file,
53 sync_playlist_to_navidrome,
54)
55from .navidrome_index import NavidromeIndex
56from .playlist_catalog import PlaylistCatalog
57from .playlist_hidden import HiddenSongs, PlaylistHiddenSongs
58from .playlist_spotify_cache import PlaylistSpotifyCache
59from .podcasts import PodcastStore, sync_show
60from .release_files import matches_release
61from .track_index import TrackIndex, normalize_spotify_track_id
62from .track_owners import TrackOwners
63from .watch_history import (
64 OUTCOME_DOWNLOADED,
65 OUTCOME_LINKED,
66 WatchHistoryStore,
67)
69MONITOR_LOOP_INTERVAL = 60 # seconds between loop sweeps
70# Seconds between filesystem reconciliation sweeps (see reconcile_loop).
71RECONCILE_LOOP_INTERVAL = 3600
72MINUTES_PER_DAY = 1440
74SYNC_TIME_ENV_VAR = 'DOWNTIFY_MONITOR_SYNC_TIME'
77@dataclass
78class LibraryStores:
79 """The library stores a sweep keeps up to date (see ``api.state``)."""
81 track_index: Optional[TrackIndex] = None
82 playlist_catalog: Optional[PlaylistCatalog] = None
83 navidrome_index: Optional[NavidromeIndex] = None
84 metadata_cache: Optional[LibraryMetadataCache] = None
85 cover_cache: Optional[CoverArtCache] = None
86 playlist_spotify_cache: Optional[PlaylistSpotifyCache] = None
87 track_owners: Optional[TrackOwners] = None
88 # Songs the owner hid from a playlist: never listed or fetched again.
89 playlist_hidden: Optional[PlaylistHiddenSongs] = None
90 # The library listing rows (file, artist, title, duration): an artist
91 # watch links a song the library already holds under another name
92 # instead of downloading it again (see downtify.library_songs).
93 listing: Optional[Callable[[], list[dict[str, Any]]]] = None
96def _now_iso() -> str:
97 return datetime.now(timezone.utc).isoformat()
100def _sync_anchor_time() -> Optional[time]:
101 """Parse ``DOWNTIFY_MONITOR_SYNC_TIME`` (``HH:MM``, 24h, local time).
103 Returns ``None`` when unset or malformed, in which case scheduling
104 falls back to the plain ``last_checked + interval`` behavior. Only
105 read from the environment at call time (not cached) so it can be
106 changed without a restart of the whole process in tests, and so a
107 typo doesn't get baked in for the process lifetime.
108 """
109 raw = os.getenv(SYNC_TIME_ENV_VAR, '').strip()
110 if not raw:
111 return None
112 hour_str, _, minute_str = raw.partition(':')
113 try:
114 hour = int(hour_str)
115 minute = int(minute_str) if minute_str else 0
116 return time(hour, minute)
117 except ValueError:
118 logger.warning(
119 '{}={!r} is not a valid HH:MM time; ignoring it.',
120 SYNC_TIME_ENV_VAR,
121 raw,
122 )
123 return None
126def _next_due_at(last: datetime, interval_minutes: int) -> datetime:
127 """Compute when *last*'s next check is due.
129 For sub-day intervals this is simply ``last + interval``. For
130 intervals that are a whole number of days (daily, weekly, every 2
131 weeks, monthly), the result is additionally snapped to the
132 :data:`SYNC_TIME_ENV_VAR` time-of-day when it's set, so all
133 day-or-longer playlists sync at the same configured hour (e.g.
134 ``03:00``) instead of at whatever time the playlist happened to be
135 added or last checked. The date component always advances by at
136 least one full interval, so the snap never moves the due time
137 earlier than an unsnapped ``last + interval`` would allow.
138 """
139 due = last + timedelta(minutes=interval_minutes)
140 if interval_minutes % MINUTES_PER_DAY != 0:
141 return due
142 anchor = _sync_anchor_time()
143 if anchor is None:
144 return due
145 local_due = due.astimezone()
146 anchored_local = local_due.replace(
147 hour=anchor.hour, minute=anchor.minute, second=0, microsecond=0
148 )
149 return anchored_local.astimezone(timezone.utc)
152def _is_due(last_checked: Optional[str], interval_minutes: int) -> bool:
153 if last_checked is None:
154 return True
155 try:
156 last = datetime.fromisoformat(last_checked)
157 if last.tzinfo is None:
158 last = last.replace(tzinfo=timezone.utc)
159 return datetime.now(timezone.utc) >= _next_due_at(
160 last, interval_minutes
161 )
162 except ValueError:
163 return True
166KIND_PLAYLIST = 'playlist'
167KIND_ARTIST = 'artist'
168#: A podcast watch. ``spotify_id`` (the watch table's generic unique-key
169#: column) holds the show's RSS feed URL for this kind — there is no
170#: Spotify id involved. See ``downtify.podcasts`` for everything else
171#: about the show (retention, episodes, tags): this table only knows
172#: how to schedule the check.
173KIND_PODCAST = 'podcast'
175SOURCE_SPOTIFY = 'spotify'
176SOURCE_YOUTUBE_MUSIC = 'youtube_music'
177SOURCE_DEEZER = 'deezer'
178DEEZER_PLAYLIST_PREFIX = 'deezer:playlist:'
180#: The kinds of release an artist watch can download (YouTube Music's own
181#: 'Album' / 'Single' / 'EP' split). A watch stores the ones it wants as a
182#: comma-separated list, in this order.
183RELEASE_ALBUM = 'album'
184RELEASE_SINGLE = 'single'
185RELEASE_EP = 'ep'
186RELEASE_TYPES = (RELEASE_ALBUM, RELEASE_SINGLE, RELEASE_EP)
187ALL_RELEASE_TYPES = ','.join(RELEASE_TYPES)
190def normalize_release_types(value: Any) -> str:
191 """*value* (a list or a comma-separated string) as the stored form:
192 the known release types it names, in :data:`RELEASE_TYPES` order.
193 ``""`` when it names none - callers treat that as invalid rather than
194 silently watching nothing."""
196 if isinstance(value, str):
197 items = value.split(',')
198 elif isinstance(value, (list, tuple, set)):
199 items = list(value)
200 else:
201 return ''
202 wanted = {str(item).strip().lower() for item in items}
203 return ','.join(t for t in RELEASE_TYPES if t in wanted)
206def release_type_of(album: dict[str, Any]) -> str:
207 """The :data:`RELEASE_TYPES` entry of a discography release. YouTube
208 Music labels each one 'Album', 'Single' or 'EP'; anything else (or no
209 label) counts as an album, which is what the albums shelf defaults
210 to."""
212 label = str(album.get('release_type') or '').strip().lower()
213 return label if label in {RELEASE_SINGLE, RELEASE_EP} else RELEASE_ALBUM
216def watch_source(url: str) -> str:
217 """The service a watch was added from, read off the URL it was added with.
219 Anything that isn't a YouTube Music or Deezer URL is Spotify, which is
220 what every watch was before the other services existed, so older rows
221 need no migration.
222 """
223 if providers.parse_youtube_url(url or '') is not None:
224 return SOURCE_YOUTUBE_MUSIC
225 if deezer.parse_deezer_url(url or '') is not None:
226 return SOURCE_DEEZER
227 return SOURCE_SPOTIFY
230def parse_playlist_url(url: str) -> Optional[tuple[str, str]]:
231 """``(source, playlist_id)`` for a supported public playlist URL."""
232 parsed = spotify.parse_spotify_url(url or '')
233 if parsed is not None and parsed[0] == 'playlist':
234 return SOURCE_SPOTIFY, parsed[1]
235 youtube_parsed = providers.parse_youtube_url(url or '')
236 if youtube_parsed is not None and youtube_parsed[0] == 'playlist':
237 return SOURCE_YOUTUBE_MUSIC, youtube_parsed[1]
238 deezer_parsed = deezer.parse_deezer_url(url or '')
239 if deezer_parsed is not None and deezer_parsed[0] == 'playlist':
240 return SOURCE_DEEZER, deezer_parsed[1]
241 return None
244def playlist_watch_key(source: str, playlist_id: str) -> str:
245 """The key (``spotify_id`` column) a playlist watch is stored under:
246 the playlist's own id, namespaced for Deezer, whose ids are bare
247 numbers. :func:`fetch_playlist` and friends accept either form."""
249 if source == SOURCE_DEEZER:
250 return DEEZER_PLAYLIST_PREFIX + playlist_id
251 return playlist_id
254def fetch_playlist(
255 source: str, playlist_id: str
256) -> tuple[str, list[dict[str, Any]]]:
257 """``(name, tracks)`` of a public playlist (blocking)."""
258 if source == SOURCE_YOUTUBE_MUSIC:
259 return providers.playlist_info_and_tracks_from_id(playlist_id)
260 if source == SOURCE_DEEZER:
261 return deezer.playlist_info_and_tracks(
262 playlist_id.removeprefix(DEEZER_PLAYLIST_PREFIX)
263 )
264 return spotify.playlist_info_and_tracks(playlist_id)
267def fetch_playlist_tracks(
268 source: str, playlist_id: str
269) -> list[dict[str, Any]]:
270 """Every track of a public playlist (blocking)."""
271 if source == SOURCE_SPOTIFY:
272 return spotify.playlist_tracks_from_id(playlist_id)
273 if source == SOURCE_DEEZER:
274 return fetch_playlist(source, playlist_id)[1]
275 return providers.playlist_tracks_from_id(playlist_id)
278def fetch_playlist_cover_url(source: str, playlist_id: str) -> str:
279 """Largest playlist cover art available on its service (blocking)."""
280 if source == SOURCE_YOUTUBE_MUSIC:
281 return providers.playlist_cover_url_from_id(playlist_id)
282 if source == SOURCE_DEEZER:
283 return deezer.playlist_cover_url_from_id(
284 playlist_id.removeprefix(DEEZER_PLAYLIST_PREFIX)
285 )
286 return spotify.playlist_cover_url_from_id(playlist_id)
289def download_playlist_cover(
290 source: str,
291 playlist_id: str,
292 m3u_path: Path,
293 settings: dict[str, Any],
294) -> Optional[Path]:
295 """Save the playlist's cover art beside *m3u_path*, when enabled.
297 No-ops when ``download_cover_art_playlists`` is off. A cover
298 fetch/write failure is logged and swallowed — it must never fail
299 the playlist download itself, which has already succeeded by the
300 time this runs.
301 """
302 if not settings.get('download_cover_art_playlists'):
303 return None
304 try:
305 cover_url = fetch_playlist_cover_url(source, playlist_id)
306 except Exception:
307 logger.opt(exception=True).warning(
308 'Failed to resolve playlist cover art for {} ({})',
309 playlist_id,
310 source,
311 )
312 return None
313 if not cover_url:
314 return None
315 return save_playlist_cover(cover_url, m3u_path)
318@dataclass
319class MonitoredPlaylist:
320 id: int
321 spotify_id: str
322 name: str
323 url: str
324 interval_minutes: int
325 enabled: bool
326 last_checked: Optional[str]
327 last_track_count: int
328 created_at: str
329 # 'playlist' watches a playlist's tracks; 'artist' watches a YouTube
330 # Music artist's discography for new releases. ``spotify_id`` is just
331 # the unique key a watch is addressed by: the Spotify or YouTube Music
332 # playlist id, or for an artist the YouTube Music channel id.
333 # Who watches (``0``: before accounts could own anything).
334 user_id: int = 0
335 kind: str = KIND_PLAYLIST
336 # Artist watches only: which release types to download (see
337 # :data:`RELEASE_TYPES`), and whether to skip everything the artist
338 # had already released when the watch started ("new releases only").
339 release_types: str = ALL_RELEASE_TYPES
340 new_only: bool = False
341 # Set while a "new releases only" watch still has to record the
342 # artist's current discography as already there (see check_artist).
343 baseline_pending: bool = False
344 # Artist watches: the artist's Deezer id, whose discography is read
345 # alongside YouTube Music's ('' until found).
346 deezer_id: str = ''
347 # Artist watches: the artist's MusicBrainz id, whose release groups
348 # fill in what the other catalogues miss ('' until found).
349 musicbrainz_id: str = ''
350 # Artist watches: the MusicBrainz secondary types this watch lets in
351 # (``musicbrainz.KINDS`` names, comma-separated); ``None`` follows
352 # the ``artist_watch_musicbrainz_kinds`` setting.
353 musicbrainz_kinds: Optional[str] = None
355 @property
356 def source(self) -> str:
357 return watch_source(self.url)
359 def to_dict(self) -> dict[str, Any]:
360 data = asdict(self)
361 data.pop('baseline_pending')
362 data['release_types'] = [t for t in self.release_types.split(',') if t]
363 return {**data, 'source': self.source}
366class PlaylistMonitorDB:
367 def __init__(self, path_or_db: str | Path | Database) -> None:
368 self.db = as_database(path_or_db, 'monitor')
369 self._path = None if self.db.path is None else str(self.db.path)
370 self._init_db()
371 self.releases = ArtistReleaseStore(self._connect)
372 # Each artist's whole catalogue, and the user's per-release choice.
373 self.catalogue = ArtistCatalogue(self._connect)
374 self.download_policy = DownloadPolicy(self._connect)
375 # What each watch added and failed to (provenance and statistics).
376 self.history = WatchHistoryStore(self._connect)
378 @contextlib.contextmanager
379 def _connect(self) -> Iterator[Connection]:
380 with self.db.connect() as conn:
381 # SQLite needs this per connection for ON DELETE CASCADE.
382 # PostgreSQL always enforces foreign keys.
383 if conn.dialect is Dialect.SQLITE:
384 conn.execute('PRAGMA foreign_keys = ON')
385 yield conn
387 def _init_db(self) -> None:
388 # PostgreSQL gets the final schema from the migrations.
389 if self.db.dialect is not Dialect.SQLITE:
390 return
391 with self._connect() as conn:
392 conn.executescript("""
393 CREATE TABLE IF NOT EXISTS monitored_playlists (
394 id INTEGER PRIMARY KEY AUTOINCREMENT,
395 spotify_id TEXT NOT NULL,
396 name TEXT NOT NULL,
397 url TEXT NOT NULL,
398 interval_minutes INTEGER NOT NULL DEFAULT 60,
399 enabled INTEGER NOT NULL DEFAULT 1,
400 last_checked TEXT,
401 last_track_count INTEGER NOT NULL DEFAULT 0,
402 created_at TEXT NOT NULL,
403 user_id INTEGER NOT NULL DEFAULT 0,
404 UNIQUE(spotify_id, user_id)
405 );
406 CREATE TABLE IF NOT EXISTS downloaded_tracks (
407 id INTEGER PRIMARY KEY AUTOINCREMENT,
408 playlist_id INTEGER NOT NULL,
409 track_spotify_id TEXT NOT NULL,
410 downloaded_at TEXT NOT NULL,
411 FOREIGN KEY (playlist_id) REFERENCES monitored_playlists(id)
412 ON DELETE CASCADE,
413 UNIQUE(playlist_id, track_spotify_id)
414 );
415 CREATE TABLE IF NOT EXISTS seen_albums (
416 id INTEGER PRIMARY KEY AUTOINCREMENT,
417 playlist_id INTEGER NOT NULL,
418 album_id TEXT NOT NULL,
419 name TEXT,
420 seen_at TEXT NOT NULL,
421 FOREIGN KEY (playlist_id) REFERENCES monitored_playlists(id)
422 ON DELETE CASCADE,
423 UNIQUE(playlist_id, album_id)
424 );
425 """)
426 # Migrations: artist watch options, and releases a "new
427 # releases only" watch skipped (as opposed to downloaded).
428 for statement in (
429 'ALTER TABLE monitored_playlists ADD COLUMN release_types '
430 f"TEXT NOT NULL DEFAULT '{ALL_RELEASE_TYPES}'",
431 'ALTER TABLE monitored_playlists ADD COLUMN new_only '
432 'INTEGER NOT NULL DEFAULT 0',
433 'ALTER TABLE monitored_playlists ADD COLUMN baseline_pending '
434 'INTEGER NOT NULL DEFAULT 0',
435 'ALTER TABLE seen_albums ADD COLUMN skipped '
436 'INTEGER NOT NULL DEFAULT 0',
437 ):
438 try:
439 conn.execute(statement)
440 except sqlite3.OperationalError:
441 pass
442 # Migration: add filename column if it doesn't exist yet
443 try:
444 conn.execute(
445 'ALTER TABLE downloaded_tracks ADD COLUMN filename TEXT'
446 )
447 except Exception:
448 pass
449 # Migration: watches used to be playlists only.
450 try:
451 conn.execute(
452 'ALTER TABLE monitored_playlists ADD COLUMN kind TEXT '
453 f"NOT NULL DEFAULT '{KIND_PLAYLIST}'"
454 )
455 except Exception:
456 pass
457 # Migration: one watch per URL became one watch per URL and
458 # owner, so two accounts can follow the same playlist.
459 columns = {
460 row[1]
461 for row in conn.execute(
462 'PRAGMA table_info(monitored_playlists)'
463 )
464 }
465 if 'user_id' not in columns:
466 conn.execute('PRAGMA foreign_keys = OFF')
467 try:
468 # Same shape as above, plus the owner (keep in sync;
469 # 'album,single,ep' is ALL_RELEASE_TYPES).
470 conn.execute("""
471 CREATE TABLE monitored_playlists_new (
472 id INTEGER PRIMARY KEY AUTOINCREMENT,
473 spotify_id TEXT NOT NULL,
474 name TEXT NOT NULL,
475 url TEXT NOT NULL,
476 interval_minutes INTEGER NOT NULL DEFAULT 60,
477 enabled INTEGER NOT NULL DEFAULT 1,
478 last_checked TEXT,
479 last_track_count INTEGER NOT NULL DEFAULT 0,
480 created_at TEXT NOT NULL,
481 kind TEXT NOT NULL DEFAULT 'playlist',
482 release_types TEXT NOT NULL
483 DEFAULT 'album,single,ep',
484 new_only INTEGER NOT NULL DEFAULT 0,
485 baseline_pending INTEGER NOT NULL DEFAULT 0,
486 user_id INTEGER NOT NULL DEFAULT 0,
487 UNIQUE(spotify_id, user_id)
488 )
489 """)
490 conn.execute("""
491 INSERT INTO monitored_playlists_new
492 (id, spotify_id, name, url, interval_minutes,
493 enabled, last_checked, last_track_count,
494 created_at, kind, release_types, new_only,
495 baseline_pending, user_id)
496 SELECT id, spotify_id, name, url, interval_minutes,
497 enabled, last_checked, last_track_count,
498 created_at, kind, release_types, new_only,
499 baseline_pending, 0
500 FROM monitored_playlists ORDER BY id
501 """)
502 conn.execute('DROP TABLE monitored_playlists')
503 conn.execute(
504 'ALTER TABLE monitored_playlists_new '
505 'RENAME TO monitored_playlists'
506 )
507 finally:
508 conn.execute('PRAGMA foreign_keys = ON')
509 # Migration: an artist watch's same artist on Deezer (see
510 # _deezer_releases), found by name on its first check. After
511 # the rebuild above, which only knows the older columns.
512 try:
513 conn.execute(
514 'ALTER TABLE monitored_playlists ADD COLUMN deezer_id '
515 "TEXT NOT NULL DEFAULT ''"
516 )
517 except sqlite3.OperationalError:
518 pass
519 # Migration: the same artist on MusicBrainz (see
520 # _musicbrainz_releases).
521 try:
522 conn.execute(
523 'ALTER TABLE monitored_playlists ADD COLUMN '
524 "musicbrainz_id TEXT NOT NULL DEFAULT ''"
525 )
526 except sqlite3.OperationalError:
527 pass
528 # Migration: a watch's own MusicBrainz release kinds (NULL:
529 # the setting's).
530 try:
531 conn.execute(
532 'ALTER TABLE monitored_playlists ADD COLUMN '
533 'musicbrainz_kinds TEXT'
534 )
535 except sqlite3.OperationalError:
536 pass
538 def add_playlist(
539 self,
540 spotify_id: str,
541 name: str,
542 url: str,
543 interval_minutes: int = 60,
544 kind: str = KIND_PLAYLIST,
545 release_types: str = ALL_RELEASE_TYPES,
546 new_only: bool = False,
547 user_id: int = 0,
548 ) -> MonitoredPlaylist:
549 with self._connect() as conn:
550 new_id = insert_returning_id(
551 conn,
552 """INSERT INTO monitored_playlists
553 (spotify_id, name, url, interval_minutes, enabled,
554 created_at, kind, release_types, new_only,
555 baseline_pending, user_id)
556 VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?)""",
557 (
558 spotify_id,
559 name,
560 url,
561 interval_minutes,
562 _now_iso(),
563 kind,
564 release_types or ALL_RELEASE_TYPES,
565 int(new_only),
566 int(new_only),
567 int(user_id or 0),
568 ),
569 )
570 row = conn.execute(
571 'SELECT * FROM monitored_playlists WHERE id = ?',
572 (new_id,),
573 ).fetchone()
574 return _row_to_playlist(row)
576 def reassign_legacy(self, user_id: int) -> int:
577 """Give pre-feature watches (``user_id`` ``0``) to *user_id*."""
579 if not user_id:
580 return 0
581 with self._connect() as conn:
582 cur = conn.execute(
583 'UPDATE monitored_playlists SET user_id = ? WHERE user_id = 0',
584 (int(user_id),),
585 )
586 return cur.rowcount
588 def list_playlists(
589 self, user_id: Optional[int] = None
590 ) -> list[MonitoredPlaylist]:
591 with self._connect() as conn:
592 if user_id is None:
593 rows = conn.execute(
594 'SELECT * FROM monitored_playlists '
595 'ORDER BY created_at DESC'
596 ).fetchall()
597 else:
598 rows = conn.execute(
599 'SELECT * FROM monitored_playlists WHERE user_id = ? '
600 'ORDER BY created_at DESC',
601 (int(user_id),),
602 ).fetchall()
603 return [_row_to_playlist(r) for r in rows]
605 def get_playlist(self, playlist_id: int) -> Optional[MonitoredPlaylist]:
606 with self._connect() as conn:
607 row = conn.execute(
608 'SELECT * FROM monitored_playlists WHERE id = ?',
609 (playlist_id,),
610 ).fetchone()
611 return _row_to_playlist(row) if row else None
613 def get_by_spotify_id(
614 self, spotify_id: str, user_id: Optional[int] = None
615 ) -> Optional[MonitoredPlaylist]:
616 with self._connect() as conn:
617 if user_id is None:
618 row = conn.execute(
619 'SELECT * FROM monitored_playlists WHERE spotify_id = ? '
620 'ORDER BY id LIMIT 1',
621 (spotify_id,),
622 ).fetchone()
623 else:
624 row = conn.execute(
625 'SELECT * FROM monitored_playlists WHERE spotify_id = ? '
626 'AND user_id = ?',
627 (spotify_id, int(user_id)),
628 ).fetchone()
629 return _row_to_playlist(row) if row else None
631 def delete_playlist(self, playlist_id: int) -> bool:
632 with self._connect() as conn:
633 cur = conn.execute(
634 'DELETE FROM monitored_playlists WHERE id = ?',
635 (playlist_id,),
636 )
637 return cur.rowcount > 0
639 def update_playlist(
640 self, playlist_id: int, **kwargs: Any
641 ) -> Optional[MonitoredPlaylist]:
642 allowed = {
643 'interval_minutes',
644 'enabled',
645 'last_checked',
646 'last_track_count',
647 'name',
648 'url',
649 'release_types',
650 'new_only',
651 'baseline_pending',
652 'deezer_id',
653 'musicbrainz_id',
654 'musicbrainz_kinds',
655 }
656 updates = {k: v for k, v in kwargs.items() if k in allowed}
657 if not updates:
658 return self.get_playlist(playlist_id)
659 set_clause = ', '.join(f'{k} = ?' for k in updates)
660 # Flags are BIGINT columns; PostgreSQL rejects a bool for them.
661 values = [
662 int(v) if isinstance(v, bool) else v for v in updates.values()
663 ]
664 values.append(playlist_id)
665 with self._connect() as conn:
666 conn.execute(
667 f'UPDATE monitored_playlists SET {set_clause} WHERE id = ?',
668 values,
669 )
670 row = conn.execute(
671 'SELECT * FROM monitored_playlists WHERE id = ?',
672 (playlist_id,),
673 ).fetchone()
674 return _row_to_playlist(row) if row else None
676 def retarget_playlist(
677 self, playlist_id: int, spotify_id: str, name: str, url: str
678 ) -> Optional[MonitoredPlaylist]:
679 """Point a watch at a different playlist or artist.
681 Keeps its id, interval and enabled state, but starts over like a
682 new watch: the name follows the new target, and the download and
683 release history of the old one is dropped. Files already in the
684 library stay; ones the new target shares are reused when it's
685 checked. Its ``watch_events`` stay too, naming the old target, so
686 the songs it brought in keep their origin.
687 """
688 with self._connect() as conn:
689 cur = conn.execute(
690 """UPDATE monitored_playlists
691 SET spotify_id = ?, name = ?, url = ?,
692 last_checked = NULL, last_track_count = 0,
693 baseline_pending = new_only, deezer_id = '',
694 musicbrainz_id = ''
695 WHERE id = ?""",
696 (spotify_id, name, url, playlist_id),
697 )
698 if cur.rowcount == 0:
699 return None
700 # The old target's failures don't concern the new one.
701 conn.execute(
702 'DELETE FROM watch_failures WHERE playlist_id = ?',
703 (playlist_id,),
704 )
705 conn.execute(
706 'DELETE FROM downloaded_tracks WHERE playlist_id = ?',
707 (playlist_id,),
708 )
709 conn.execute(
710 'DELETE FROM seen_albums WHERE playlist_id = ?',
711 (playlist_id,),
712 )
713 conn.execute(
714 'DELETE FROM artist_release_tracks WHERE playlist_id = ?',
715 (playlist_id,),
716 )
717 conn.execute(
718 'DELETE FROM artist_catalogue WHERE playlist_id = ?',
719 (playlist_id,),
720 )
721 conn.execute(
722 'DELETE FROM artist_catalogue_tracks WHERE playlist_id = ?',
723 (playlist_id,),
724 )
725 row = conn.execute(
726 'SELECT * FROM monitored_playlists WHERE id = ?',
727 (playlist_id,),
728 ).fetchone()
729 return _row_to_playlist(row)
731 def get_track_filenames(
732 self, playlist_id: int
733 ) -> dict[str, Optional[str]]:
734 """Return ``{track_spotify_id: filename}`` for all known tracks."""
735 with self._connect() as conn:
736 rows = conn.execute(
737 'SELECT track_spotify_id, filename FROM downloaded_tracks WHERE playlist_id = ?',
738 (playlist_id,),
739 ).fetchall()
740 return {r['track_spotify_id']: r['filename'] for r in rows}
742 def get_seen_album_ids(self, playlist_id: int) -> set[str]:
743 """Return the album ids already processed for an artist watch."""
744 with self._connect() as conn:
745 rows = conn.execute(
746 'SELECT album_id FROM seen_albums WHERE playlist_id = ?',
747 (playlist_id,),
748 ).fetchall()
749 return {r['album_id'] for r in rows}
751 def mark_album_seen(
752 self,
753 playlist_id: int,
754 album_id: str,
755 name: Optional[str] = None,
756 ) -> None:
757 with self._connect() as conn:
758 conn.execute(
759 """INSERT INTO seen_albums
760 (playlist_id, album_id, name, seen_at, last_attempt)
761 VALUES (?, ?, ?, ?, ?)
762 ON CONFLICT(playlist_id, album_id) DO UPDATE SET
763 name=excluded.name, skipped=0,last_attempt=excluded.last_attempt""",
764 (playlist_id, album_id, name, _now_iso(), _now_iso()),
765 )
767 def mark_albums_skipped(
768 self, playlist_id: int, albums: list[dict[str, Any]]
769 ) -> int:
770 """Record *albums* as already there when a "new releases only"
771 watch starts, without downloading them. A release that was
772 downloaded before keeps its row as is. Returns how many were
773 added."""
775 now = _now_iso()
776 added = 0
777 with self._connect() as conn:
778 for album in albums:
779 album_id = album.get('album_id')
780 if not album_id:
781 continue
782 cur = conn.execute(
783 """INSERT INTO seen_albums
784 (playlist_id, album_id, name, seen_at, skipped)
785 VALUES (?, ?, ?, ?, 1)
786 ON CONFLICT(playlist_id, album_id) DO NOTHING""",
787 (playlist_id, album_id, album.get('name'), now),
788 )
789 added += cur.rowcount
790 return added
792 def forget_skipped_albums(self, playlist_id: int) -> int:
793 """Drop the releases a "new releases only" watch skipped, so the
794 next check downloads them - turning the option off. Returns how
795 many."""
797 with self._connect() as conn:
798 cur = conn.execute(
799 'DELETE FROM seen_albums WHERE playlist_id = ? AND skipped = 1',
800 (playlist_id,),
801 )
802 return cur.rowcount
804 def set_new_only(
805 self, playlist_id: int, new_only: bool
806 ) -> Optional[MonitoredPlaylist]:
807 """Turn "new releases only" on or off for an artist watch.
809 On: the next check records whatever the artist has released by
810 then as already there (see :func:`check_artist`), so only later
811 releases download. Off: the releases it skipped are forgotten, so
812 the next check downloads the back catalog after all.
813 """
815 current = self.get_playlist(playlist_id)
816 if current is None or current.new_only == new_only:
817 return current
818 if new_only:
819 return self.update_playlist(
820 playlist_id, new_only=1, baseline_pending=1
821 )
822 self.forget_skipped_albums(playlist_id)
823 return self.update_playlist(
824 playlist_id, new_only=0, baseline_pending=0
825 )
827 def playlists_for_track(self, track_spotify_id: str) -> set[str]:
828 """Names of the watched playlists that downloaded this track."""
830 tid = str(track_spotify_id or '').strip()
831 if not tid:
832 return set()
833 with self._connect() as conn:
834 rows = conn.execute(
835 """SELECT DISTINCT p.name FROM downloaded_tracks dt
836 JOIN monitored_playlists p ON p.id = dt.playlist_id
837 WHERE dt.track_spotify_id = ?""",
838 (tid,),
839 ).fetchall()
840 return {str(row['name']) for row in rows}
842 def update_filename_for_spotify(
843 self, track_spotify_id: str, filename: str
844 ) -> set[str]:
845 """Point every watch's record of this track at ``filename``.
847 Used when the track is downloaded again elsewhere (another
848 playlist, a single download), so a watch doesn't keep a stale
849 path. Returns the playlists that have the track.
850 """
852 tid = str(track_spotify_id or '').strip()
853 name = str(filename or '').strip().replace('\\', '/')
854 if not tid or not name:
855 return set()
856 with self._connect() as conn:
857 conn.execute(
858 """UPDATE downloaded_tracks SET filename = ?
859 WHERE track_spotify_id = ?
860 AND (filename IS NULL OR filename != ?)""",
861 (name, tid, name),
862 )
863 self.history.rename_file(tid, name)
864 return self.playlists_for_track(tid)
866 def mark_track_downloaded(
867 self,
868 playlist_id: int,
869 track_spotify_id: str,
870 filename: Optional[str] = None,
871 *,
872 outcome: Optional[str] = None,
873 provider: str = '',
874 initial: bool = False,
875 ) -> None:
876 """Record that the watch has the track (in *filename*).
878 With an *outcome* (see ``OUTCOME_*``) the addition is also logged
879 in the watch's history (:mod:`downtify.watch_history`) - which
880 provider fetched it, and whether it came with the watch's first
881 check (*initial*, its backfill) - and any failure recorded for
882 the track is cleared.
883 """
884 with self._connect() as conn:
885 conn.execute(
886 """INSERT INTO downloaded_tracks
887 (playlist_id, track_spotify_id, downloaded_at, filename)
888 VALUES (?, ?, ?, ?)
889 ON CONFLICT(playlist_id, track_spotify_id) DO UPDATE SET
890 downloaded_at=excluded.downloaded_at,
891 filename=excluded.filename""",
892 (playlist_id, track_spotify_id, _now_iso(), filename),
893 )
894 if outcome:
895 self.history.record_added(
896 playlist_id,
897 track_spotify_id,
898 filename,
899 outcome=outcome,
900 provider=provider,
901 initial=initial,
902 )
904 def list_all_downloaded_tracks(self) -> list[dict[str, Any]]:
905 """Every ``downloaded_tracks`` row that has a filename, across
906 every watch.
908 Used by the hourly reconciliation sweep (see
909 :func:`reconcile_downloaded_tracks`), which checks all of them
910 against the filesystem in one pass instead of one watch at a
911 time, and by the per-watch statistics.
912 """
913 with self._connect() as conn:
914 rows = conn.execute(
915 """SELECT playlist_id, track_spotify_id, filename
916 FROM downloaded_tracks
917 WHERE filename IS NOT NULL"""
918 ).fetchall()
919 return [dict(r) for r in rows]
921 def remove_downloaded_tracks(self, pairs: list[tuple[int, str]]) -> int:
922 """Delete the given ``(playlist_id, track_spotify_id)`` rows.
924 Called when the hourly reconciliation sweep finds a downloaded
925 track's file is no longer under the downloads directory — the
926 track goes back to being "not downloaded" and is re-fetched on
927 the watch's next check.
928 """
929 if not pairs:
930 return 0
931 with self._connect() as conn:
932 conn.executemany(
933 """DELETE FROM downloaded_tracks
934 WHERE playlist_id = ? AND track_spotify_id = ?""",
935 pairs,
936 )
937 return len(pairs)
940def delete_user_watches(db: PlaylistMonitorDB, user_id: int) -> None:
941 """Drop a deleted account's watches (downloaded files stay put)."""
943 for watch in db.list_playlists(user_id):
944 db.delete_playlist(watch.id)
947def repoint_downloaded_file(db: PlaylistMonitorDB, old: str, new: str) -> int:
948 """Point every record naming the file *old* at *new*: the tracks
949 watches downloaded, the artist releases they recorded and the watch
950 history. For a merged duplicate (see downtify.duplicate_merge), so a
951 watch doesn't take its track for missing and download it again.
952 Returns how many downloaded tracks moved."""
954 old_name = str(old or '').strip().replace('\\', '/')
955 new_name = str(new or '').strip().replace('\\', '/')
956 if not old_name or not new_name or old_name == new_name:
957 return 0
958 with db._connect() as conn:
959 moved = conn.execute(
960 'UPDATE downloaded_tracks SET filename = ? WHERE filename = ?',
961 (new_name, old_name),
962 ).rowcount
963 conn.execute(
964 'UPDATE artist_release_tracks SET filename = ? WHERE filename = ?',
965 (new_name, old_name),
966 )
967 conn.execute(
968 'UPDATE watch_events SET filename = ? WHERE filename = ?',
969 (new_name, old_name),
970 )
971 return moved
974def _row_to_playlist(row: Any) -> MonitoredPlaylist:
975 keys = row.keys()
976 return MonitoredPlaylist(
977 id=row['id'],
978 spotify_id=row['spotify_id'],
979 name=row['name'],
980 url=row['url'],
981 interval_minutes=row['interval_minutes'],
982 enabled=bool(row['enabled']),
983 last_checked=row['last_checked'],
984 last_track_count=row['last_track_count'],
985 created_at=row['created_at'],
986 # Rows written before the kind migration have no column at all
987 # when reading from a stale connection/schema cache.
988 kind=(row['kind'] if 'kind' in keys else KIND_PLAYLIST)
989 or KIND_PLAYLIST,
990 user_id=int(row['user_id']) if 'user_id' in keys else 0,
991 release_types=(row['release_types'] if 'release_types' in keys else '')
992 or ALL_RELEASE_TYPES,
993 new_only=bool(row['new_only']) if 'new_only' in keys else False,
994 baseline_pending=(
995 bool(row['baseline_pending'])
996 if 'baseline_pending' in keys
997 else False
998 ),
999 deezer_id=str(row['deezer_id'] or '') if 'deezer_id' in keys else '',
1000 musicbrainz_id=(
1001 str(row['musicbrainz_id'] or '')
1002 if 'musicbrainz_id' in keys
1003 else ''
1004 ),
1005 musicbrainz_kinds=(
1006 row['musicbrainz_kinds'] if 'musicbrainz_kinds' in keys else None
1007 ),
1008 )
1011async def _fill_from_spotify_track(song: dict[str, Any]) -> None:
1012 """Top up a Spotify playlist row from the track's own embed.
1014 Playlist embed entries are missing release year and use the playlist
1015 cover instead of the album cover; the per-track embed has both. The
1016 playlist values stay as a fallback if the per-track fetch fails.
1017 """
1018 try:
1019 full = await asyncio.to_thread(spotify.track_from_id, song['song_id'])
1020 except Exception:
1021 logger.opt(exception=True).warning(
1022 'Per-track Spotify fetch failed for {}; '
1023 'falling back to playlist data',
1024 song['song_id'],
1025 )
1026 return
1027 for key in ('cover_url', 'year', 'release_date', 'album_name', 'artists'):
1028 value = full.get(key)
1029 if value:
1030 song[key] = value
1031 for key in ('track_number', 'album_track_total'):
1032 if not song.get(key) and full.get(key):
1033 song[key] = full[key]
1034 # The track's own page is the surer word on explicit lyrics.
1035 if full.get('explicit'):
1036 song['explicit'] = True
1039async def check_playlist(
1040 playlist: MonitoredPlaylist,
1041 db: PlaylistMonitorDB,
1042 downloader: Downloader,
1043 broadcast: Callable[[dict[str, Any]], Any],
1044 loop: asyncio.AbstractEventLoop,
1045 settings: Optional[dict[str, Any]] = None,
1046 library: Optional[LibraryStores] = None,
1047) -> int:
1048 """Fetch playlist, detect new tracks, download them. Returns count downloaded.
1050 With ``library`` stores, new tracks already in the library (by Spotify
1051 id, when *Overwrite existing files* is off) are linked instead of
1052 downloaded, every downloaded track is registered in them, and the
1053 playlist's catalog and Navidrome playlist are synced after the sweep.
1054 """
1055 logger.info(
1056 'Checking monitored playlist "{}" ({})',
1057 playlist.name,
1058 playlist.spotify_id,
1059 )
1061 from_spotify = playlist.source == SOURCE_SPOTIFY
1062 # The first check backfills whatever the playlist already had; the
1063 # statistics keep that apart from what it adds over time.
1064 initial = playlist.last_checked is None
1065 try:
1066 tracks = await asyncio.to_thread(
1067 fetch_playlist_tracks, playlist.source, playlist.spotify_id
1068 )
1069 except Exception:
1070 logger.exception('Failed to fetch playlist {}', playlist.spotify_id)
1071 await asyncio.to_thread(
1072 db.update_playlist, playlist.id, last_checked=_now_iso()
1073 )
1074 return 0
1075 library = library or LibraryStores()
1076 if from_spotify and library.playlist_spotify_cache is not None:
1077 await asyncio.to_thread(
1078 library.playlist_spotify_cache.store,
1079 playlist.spotify_id,
1080 playlist.name,
1081 tracks,
1082 )
1083 if library.playlist_catalog is not None:
1084 await asyncio.to_thread(
1085 library.playlist_catalog.ensure_playlist,
1086 playlist.name,
1087 spotify_id=_catalog_spotify_id(playlist),
1088 user_id=playlist.user_id or None,
1089 )
1091 known_tracks = await asyncio.to_thread(db.get_track_filenames, playlist.id)
1092 # Songs the owner hid from the playlist are left out from here on:
1093 # not downloaded (even once their file is gone), not in the M3U.
1094 tracks = _without_hidden(
1095 tracks, _hidden_songs(library, playlist.name), known_tracks
1096 )
1098 pl_subdir = m3u.sanitize_playlist_name(playlist.name)
1100 # Filenames already known from earlier sweeps, keyed by track id,
1101 # topped up as each new track lands. Lets the M3U be rewritten after
1102 # every single download without re-resolving the whole playlist
1103 # against the filesystem each time (which is O(tracks) globs per
1104 # write, and quadratic over a large sweep).
1105 #
1106 # A track already in `downloaded_tracks` is trusted as downloaded
1107 # without checking whether its file is still on disk — the file may
1108 # have been moved elsewhere in the filesystem outside the downloads
1109 # directory (e.g. into a separate media library), which must not
1110 # cause a re-download on every single sweep. A file that's actually
1111 # gone (deleted, not just moved) is instead noticed and forgotten by
1112 # the hourly :func:`reconcile_downloaded_tracks` sweep, which is what
1113 # makes such a track eligible for re-download again.
1114 resolved: dict[str, str] = {}
1116 new_tracks, linked_from_library = await _split_new_tracks(
1117 playlist,
1118 tracks,
1119 db,
1120 downloader,
1121 library,
1122 known_tracks,
1123 resolved,
1124 initial=initial,
1125 settings=settings or {},
1126 )
1128 if new_tracks:
1129 logger.info(
1130 'Found {} track(s) to download in playlist "{}"',
1131 len(new_tracks),
1132 playlist.name,
1133 )
1134 if settings is None or settings.get('generate_m3u', True):
1135 # Once per sweep, and before the tracks — a sweep that finds
1136 # nothing new never re-fetches it.
1137 await asyncio.to_thread(
1138 _download_cover_for_watch, playlist, downloader, settings or {}
1139 )
1141 delay_seconds = (settings or {}).get('download_delay_seconds', 0) or 0
1142 track_positions = {id(t): i for i, t in enumerate(tracks)}
1144 downloaded = 0
1145 for index, song in enumerate(new_tracks):
1146 track_id = song['song_id']
1147 # A paused download queue holds the sweep before each track.
1148 await queue_pause.wait()
1150 # YouTube Music rows already carry their own video's metadata.
1151 if from_spotify:
1152 await _fill_from_spotify_track(song)
1154 filename = None
1155 try:
1156 filename = await _fetch_watch_track(
1157 song,
1158 playlist,
1159 db,
1160 downloader,
1161 broadcast,
1162 loop,
1163 subdir=pl_subdir,
1164 initial=initial,
1165 )
1166 invalidate_library_paths_cache()
1167 downloaded += 1
1168 if filename:
1169 resolved[track_id] = filename
1170 await asyncio.to_thread(
1171 _register_monitored_download,
1172 song,
1173 filename,
1174 downloader,
1175 library,
1176 settings or {},
1177 playlist_name=playlist.name,
1178 track_order=track_positions[id(song)],
1179 owner_id=playlist.user_id,
1180 )
1181 if settings is None or settings.get('generate_m3u', True):
1182 # Rewrite the M3U after every track rather than once the
1183 # whole sweep finishes, so the playlist grows as it
1184 # downloads and a slow/hung track further down the list
1185 # never holds up what's already on disk.
1186 await asyncio.to_thread(
1187 _regenerate_m3u, playlist, tracks, downloader, resolved
1188 )
1189 if delay_seconds > 0 and index != len(new_tracks) - 1:
1190 await asyncio.sleep(delay_seconds)
1191 except Exception as exc:
1192 logger.exception('Failed to auto-download track {}', track_id)
1193 # Only the download itself failing makes it a failed track.
1194 if filename is None:
1195 await asyncio.to_thread(
1196 _record_failure, db, playlist.id, song, exc
1197 )
1199 await asyncio.to_thread(
1200 db.update_playlist,
1201 playlist.id,
1202 last_checked=_now_iso(),
1203 last_track_count=len(tracks),
1204 )
1205 await asyncio.to_thread(
1206 _forget_stale_failures,
1207 db,
1208 playlist.id,
1209 {str(t.get('song_id')) for t in tracks if t.get('song_id')},
1210 )
1212 if downloaded > 0 or linked_from_library > 0:
1213 known_tracks = await asyncio.to_thread(
1214 db.get_track_filenames, playlist.id
1215 )
1216 if settings is None or settings.get('generate_m3u', True):
1217 await asyncio.to_thread(
1218 _regenerate_m3u,
1219 playlist,
1220 tracks,
1221 downloader,
1222 None,
1223 known_tracks,
1224 )
1225 await asyncio.to_thread(
1226 _sync_library_playlist,
1227 playlist,
1228 tracks,
1229 downloader,
1230 known_tracks,
1231 library,
1232 settings,
1233 )
1234 return downloaded
1237def _hidden_songs(library: LibraryStores, name: str) -> HiddenSongs:
1238 if library.playlist_hidden is None:
1239 return HiddenSongs()
1240 try:
1241 return library.playlist_hidden.songs(name)
1242 except Exception:
1243 logger.opt(exception=True).warning(
1244 'Could not read the hidden songs of "{}"', name
1245 )
1246 return HiddenSongs()
1249def _without_hidden(
1250 tracks: list[dict[str, Any]],
1251 hidden: HiddenSongs,
1252 known_tracks: dict[str, Optional[str]],
1253) -> list[dict[str, Any]]:
1254 """*tracks* minus the songs hidden from the playlist - by source id,
1255 or by the file the watch recorded for them."""
1257 if not hidden:
1258 return tracks
1259 return [
1260 track
1261 for track in tracks
1262 if not hidden.hides_song(track)
1263 and not hidden.hides_file(known_tracks.get(track.get('song_id') or ''))
1264 ]
1267def _catalog_spotify_id(playlist: MonitoredPlaylist) -> Optional[str]:
1268 """The playlist catalog keys playlists by Spotify id; a YouTube Music
1269 watch has none."""
1271 return playlist.spotify_id if playlist.source == SOURCE_SPOTIFY else None
1274async def _split_new_tracks(
1275 playlist: MonitoredPlaylist,
1276 tracks: list[dict[str, Any]],
1277 db: PlaylistMonitorDB,
1278 downloader: Downloader,
1279 library: LibraryStores,
1280 known_tracks: dict[str, Optional[str]],
1281 resolved: dict[str, str],
1282 *,
1283 initial: bool = False,
1284 settings: Optional[dict[str, Any]] = None,
1285) -> tuple[list[dict[str, Any]], int]:
1286 """``(tracks to download, number linked from the library)``.
1288 Tracks the watch already recorded are skipped (and their filenames
1289 added to ``resolved``). With *Overwrite existing files* off, a new
1290 track the library already holds is recorded for the watch instead of
1291 being downloaded again (as :data:`OUTCOME_LINKED`): by its Spotify id
1292 in the track index, else as the same song (:func:`_same_song_in_library`).
1294 The second lookup is what stops a long-finished playlist from growing
1295 (issue #573): Spotify swaps a playlist entry for the same song under
1296 a new id (a re-release, a remaster, another market's copy), the new
1297 id is unknown to the watch and to the index, and the name-based check
1298 misses a title that changed (``… - Remastered 2011``).
1299 """
1301 new_tracks: list[dict[str, Any]] = []
1302 linked = 0
1303 link_existing = not getattr(downloader, 'overwrite_existing_files', True)
1304 settings = settings or {}
1305 # The library by song, read once per sweep and only when needed.
1306 songs: list[Optional[LibrarySongs]] = []
1307 for track in tracks:
1308 tid = track.get('song_id')
1309 if not tid:
1310 continue
1311 if tid in known_tracks:
1312 stored = known_tracks[tid]
1313 if stored is not None:
1314 resolved[tid] = stored
1315 continue
1316 existing = (
1317 await asyncio.to_thread(
1318 _library_file_for, track, downloader, library, settings
1319 )
1320 if link_existing
1321 else None
1322 )
1323 if not existing and link_existing:
1324 if not songs:
1325 songs.append(await asyncio.to_thread(_library_songs, library))
1326 existing = await asyncio.to_thread(
1327 _same_song_in_library,
1328 track,
1329 songs[0],
1330 downloader,
1331 library,
1332 settings,
1333 )
1334 if not existing:
1335 logger.debug(
1336 'Watch "{}": {} ({}) is new to the watch and not in the '
1337 'library; downloading it',
1338 playlist.name,
1339 track.get('name'),
1340 tid,
1341 )
1342 new_tracks.append(track)
1343 continue
1344 await asyncio.to_thread(
1345 functools.partial(
1346 db.mark_track_downloaded,
1347 playlist.id,
1348 tid,
1349 existing,
1350 outcome=OUTCOME_LINKED,
1351 initial=initial,
1352 )
1353 )
1354 resolved[tid] = existing
1355 linked += 1
1356 if linked:
1357 logger.info(
1358 'Linked {} track(s) already in the library into playlist "{}"',
1359 linked,
1360 playlist.name,
1361 )
1362 return new_tracks, linked
1365async def _fetch_watch_track(
1366 song: dict[str, Any],
1367 playlist: MonitoredPlaylist,
1368 db: PlaylistMonitorDB,
1369 downloader: Downloader,
1370 broadcast: Callable[[dict[str, Any]], Any],
1371 loop: asyncio.AbstractEventLoop,
1372 *,
1373 subdir: str,
1374 initial: bool,
1375) -> str:
1376 """Download one new track of a playlist watch and record it, with
1377 how it went (see :func:`_outcome_of`). Raises when it failed."""
1379 observed: dict[str, Any] = {}
1380 progress = _progress_cb(song, playlist.name, broadcast, loop, observed)
1381 async with (
1382 _download_slot(song, playlist.name, broadcast),
1383 queue_pause.running(),
1384 ):
1385 filename = await loop.run_in_executor(
1386 DOWNLOAD_EXECUTOR,
1387 lambda: downloader.download(song, progress, subdir=subdir),
1388 )
1389 await asyncio.to_thread(
1390 functools.partial(
1391 db.mark_track_downloaded,
1392 playlist.id,
1393 song['song_id'],
1394 filename,
1395 outcome=_outcome_of(observed),
1396 provider=observed.get('provider', ''),
1397 initial=initial,
1398 )
1399 )
1400 return filename
1403def _outcome_of(observed: dict[str, Any]) -> str:
1404 """What a finished download was, from its progress messages: the
1405 downloader found the song already on disk, or fetched it."""
1407 return OUTCOME_LINKED if observed.get('reused') else OUTCOME_DOWNLOADED
1410def _record_failure(
1411 db: PlaylistMonitorDB,
1412 playlist_id: int,
1413 song: dict[str, Any],
1414 error: BaseException,
1415) -> None:
1416 """Best effort: a failure that can't be recorded is only logged."""
1418 try:
1419 db.history.record_failure(playlist_id, song, str(error))
1420 except Exception:
1421 logger.opt(exception=True).debug(
1422 'Could not record the failure of {}', song.get('song_id')
1423 )
1426def _forget_stale_failures(
1427 db: PlaylistMonitorDB, playlist_id: int, track_ids: set[str]
1428) -> None:
1429 try:
1430 db.history.forget_failures_except(playlist_id, track_ids)
1431 except Exception:
1432 logger.opt(exception=True).debug(
1433 'Could not prune the failures of watch {}', playlist_id
1434 )
1437def _library_file_for(
1438 song: dict[str, Any],
1439 downloader: Downloader,
1440 library: LibraryStores,
1441 settings: Optional[dict[str, Any]] = None,
1442) -> Optional[str]:
1443 """The library path already holding ``song`` per the track index.
1445 The configured external music folders count (an ``ext/…`` entry is
1446 found there, not forgotten as missing); an entry whose file is gone
1447 everywhere is forgotten."""
1449 tid = normalize_spotify_track_id(song)
1450 if not tid or library.track_index is None:
1451 return None
1452 stored = library.track_index.lookup(tid)
1453 if not stored:
1454 return None
1455 if _locate(stored, downloader, settings or {}):
1456 return stored
1457 library.track_index.forget(tid)
1458 return None
1461def _same_song_in_library(
1462 song: dict[str, Any],
1463 songs: Optional[LibrarySongs],
1464 downloader: Downloader,
1465 library: LibraryStores,
1466 settings: Optional[dict[str, Any]] = None,
1467) -> Optional[str]:
1468 """The library file holding ``song`` under another id: the same ISRC,
1469 else the same song key and length (:mod:`downtify.library_songs`).
1471 The file is then indexed under ``song``'s Spotify id too, so the next
1472 sweep (and the playlist catalog) finds it straight away."""
1474 if songs is None:
1475 return None
1476 found = songs.find(song)
1477 if not found:
1478 return None
1479 full = _locate(found, downloader, settings or {})
1480 if full is None:
1481 return None
1482 logger.info(
1483 'Linking {!r} ({}) to {} already in the library (same song under '
1484 'another id)',
1485 song.get('name'),
1486 song.get('song_id'),
1487 found,
1488 )
1489 if library.track_index is not None:
1490 try:
1491 library.track_index.register_song(song, found, full_path=full)
1492 except Exception:
1493 logger.opt(exception=True).debug(
1494 'Could not index {} for {}', found, song.get('song_id')
1495 )
1496 return found
1499def _record_sweep_owner(
1500 library: LibraryStores,
1501 full: Path,
1502 owner_id: int,
1503 stored_path: str = '',
1504) -> None:
1505 """Attribute a sweep download to the watch owner (best effort)."""
1507 if not owner_id or library.track_owners is None:
1508 return
1509 try:
1510 key = file_content_key(full)
1511 if key:
1512 library.track_owners.add(key, owner_id, stored_path)
1513 except Exception:
1514 logger.opt(exception=True).debug(
1515 'Could not record sweep owner for {}', full
1516 )
1519def _record_artist_sweep_owner(
1520 library: LibraryStores,
1521 filename: str,
1522 downloader: Downloader,
1523 owner_id: int,
1524) -> None:
1525 """Attribute an artist-watch download (best effort)."""
1527 if not owner_id or library.track_owners is None:
1528 return
1529 try:
1530 full = locate_library_file(
1531 filename,
1532 Path(downloader.download_dir),
1533 slskd_dir_from_downloader(downloader),
1534 )
1535 if full is None:
1536 return
1537 key = file_content_key(full)
1538 if key:
1539 library.track_owners.add(key, owner_id, filename)
1540 except Exception:
1541 logger.opt(exception=True).debug(
1542 'Could not record sweep owner for {}', filename
1543 )
1546def _register_monitored_download(
1547 song: dict[str, Any],
1548 filename: str,
1549 downloader: Downloader,
1550 library: LibraryStores,
1551 settings: dict[str, Any],
1552 *,
1553 playlist_name: str,
1554 track_order: int,
1555 owner_id: int = 0,
1556) -> None:
1557 """Record a track a sweep downloaded in the library stores."""
1559 download_dir = Path(downloader.download_dir)
1560 slskd_dir = slskd_dir_from_downloader(downloader)
1561 full = locate_library_file(filename, download_dir, slskd_dir)
1562 if full is None:
1563 return
1564 _record_sweep_owner(library, full, owner_id, filename)
1565 try:
1566 if library.track_index is not None:
1567 library.track_index.register_song(song, filename, full_path=full)
1568 if library.playlist_catalog is not None:
1569 library.playlist_catalog.upsert_track(
1570 playlist_name, song, filename, full, track_order=track_order
1571 )
1572 if library.navidrome_index is not None:
1573 cache_navidrome_song_id(
1574 settings,
1575 song,
1576 filename,
1577 library.navidrome_index,
1578 download_dir=download_dir,
1579 slskd_dir=slskd_dir,
1580 )
1581 if library.metadata_cache is not None:
1582 library.metadata_cache.refresh_stored_path(
1583 filename, download_dir=download_dir, slskd_dir=slskd_dir
1584 )
1585 if settings.get('cache_cover_art') and library.cover_cache:
1586 library.cover_cache.refresh_stored_path(
1587 filename, download_dir=download_dir, slskd_dir=slskd_dir
1588 )
1589 except Exception:
1590 logger.exception('Could not register {} in the library', filename)
1593def _sync_library_playlist(
1594 playlist: MonitoredPlaylist,
1595 tracks: list[dict[str, Any]],
1596 downloader: Downloader,
1597 known_tracks: dict[str, Optional[str]],
1598 library: LibraryStores,
1599 settings: Optional[dict[str, Any]],
1600) -> None:
1601 """After a sweep: mirror the playlist's on-disk tracks into the
1602 catalog and, when Navidrome sync is on, update its Navidrome playlist."""
1604 download_dir = Path(downloader.download_dir)
1605 slskd_dir = slskd_dir_from_downloader(downloader)
1606 rows: list[tuple[dict[str, Any], str, Path]] = []
1607 for song in tracks:
1608 filename = known_tracks.get(song.get('song_id') or '')
1609 full = (
1610 locate_library_file(filename, download_dir, slskd_dir)
1611 if filename
1612 else None
1613 )
1614 if full is not None:
1615 rows.append((song, filename, full))
1617 if library.playlist_catalog is not None and rows:
1618 try:
1619 library.playlist_catalog.replace_playlist_tracks(
1620 playlist.name,
1621 rows,
1622 spotify_id=_catalog_spotify_id(playlist),
1623 user_id=playlist.user_id or None,
1624 )
1625 except Exception:
1626 logger.exception(
1627 'Playlist catalog sync failed for "{}"', playlist.name
1628 )
1630 if (
1631 not settings
1632 or settings.get('sync_navidrome', True) is False
1633 or not _effective_navidrome_settings(settings).get('enabled')
1634 ):
1635 return
1636 songs = [
1637 enrich_song_from_library_file(
1638 {**song, 'filename': filename}, download_dir, slskd_dir
1639 )
1640 for song, filename, _full in rows
1641 ]
1642 if not songs:
1643 logger.warning(
1644 'Navidrome sync skip for "{}": no tracks on disk', playlist.name
1645 )
1646 return
1647 try:
1648 sync_playlist_to_navidrome(
1649 playlist.name,
1650 songs,
1651 settings,
1652 navidrome_index=library.navidrome_index,
1653 download_dir=download_dir,
1654 )
1655 except Exception:
1656 logger.exception('Navidrome sync failed for "{}"', playlist.name)
1659#: Where the "Parallel downloads" limit lives (``api.state
1660#: .download_semaphore``, replaced when the setting changes), set by
1661#: :func:`share_download_slots`. A watch's downloads take a slot from it
1662#: like every other download, so a burst of checks - a hundred watches
1663#: added at once each starting its first check, "Check now", "Download
1664#: all missing" - never runs more downloads than the setting allows.
1665def _no_download_limit() -> Optional[asyncio.Semaphore]:
1666 return None
1669_download_slots: Callable[[], Optional[asyncio.Semaphore]] = _no_download_limit
1672def share_download_slots(
1673 get: Callable[[], Optional[asyncio.Semaphore]],
1674) -> None:
1675 """Make watch downloads wait for the limit *get* returns."""
1677 global _download_slots
1678 _download_slots = get
1681@contextlib.asynccontextmanager
1682async def _download_slot(
1683 song: dict[str, Any],
1684 label: str,
1685 broadcast: Callable[[dict[str, Any]], Any],
1686) -> AsyncIterator[None]:
1687 """Hold one "Parallel downloads" slot for a watch's download.
1689 While every slot is taken the song shows in the queue as waiting
1690 (``queued``), as a pasted link would.
1691 """
1693 sem = _download_slots()
1694 if sem is None:
1695 yield
1696 return
1697 if sem.locked():
1698 await broadcast({
1699 'song': song,
1700 'progress': 0,
1701 'message': '',
1702 'status': 'queued',
1703 'playlist_name': label,
1704 })
1705 async with sem:
1706 yield
1709def _progress_cb(
1710 song: dict[str, Any],
1711 label: str,
1712 broadcast: Callable[[dict[str, Any]], Any],
1713 loop: asyncio.AbstractEventLoop,
1714 observed: Optional[dict[str, Any]] = None,
1715) -> ProgressCallback:
1716 """Build a downloader progress callback that broadcasts over the WS."""
1718 def _cb(pct: float, message: str, provider: Optional[str] = None) -> None:
1719 if observed is not None:
1720 observed['message'] = message
1721 observed['reused'] = message == 'Already downloaded'
1722 if provider:
1723 observed['provider'] = provider
1724 asyncio.run_coroutine_threadsafe(
1725 broadcast({
1726 'song': song,
1727 'progress': pct,
1728 'message': message,
1729 'provider': provider or '',
1730 'playlist_name': label,
1731 }),
1732 loop,
1733 )
1735 return _cb
1738#: Prefix of a release id from Deezer, telling it apart from a YouTube
1739#: Music browse id in ``seen_albums`` and ``artist_release_tracks``.
1740DEEZER_RELEASE = 'deezer:'
1741#: Prefix of a release id from MusicBrainz (a release group's id).
1742MUSICBRAINZ_RELEASE = 'mb:'
1745def _artist_discography(
1746 playlist: MonitoredPlaylist,
1747 db: PlaylistMonitorDB,
1748 settings: dict[str, Any],
1749) -> list[dict[str, Any]]:
1750 """Every release of a watched artist that a catalogue lists.
1752 YouTube Music's artist page doesn't always list everything - it
1753 leaves out Drake's *Views* - so, with ``artist_watch_deezer`` on,
1754 Deezer's discography of the same artist is read too, and with
1755 ``artist_watch_musicbrainz`` on, MusicBrainz's release groups that
1756 neither of them lists (mixtapes, older or rarer releases - see
1757 :func:`_musicbrainz_releases`). A release two catalogues list is
1758 processed twice, but its songs are downloaded once: the sweep links
1759 a song the library (or this sweep) already has. Raises only when no
1760 source answered.
1761 """
1763 found: list[dict[str, Any]] = []
1764 asked = failed = 0
1765 sources = [
1766 (
1767 'YouTube Music',
1768 lambda: providers.artist_albums_from_channel_id(
1769 playlist.spotify_id
1770 ),
1771 )
1772 ]
1773 if settings.get('artist_watch_deezer'):
1774 sources.append(('Deezer', lambda: _deezer_releases(playlist, db)))
1775 if settings.get('artist_watch_musicbrainz'):
1776 sources.append((
1777 'MusicBrainz',
1778 lambda: _musicbrainz_releases(playlist, db, settings, found),
1779 ))
1780 for name, read in sources:
1781 asked += 1
1782 try:
1783 found += read()
1784 except Exception:
1785 failed += 1
1786 logger.opt(exception=True).warning(
1787 '{} discography failed for artist "{}"', name, playlist.name
1788 )
1789 if failed == asked:
1790 raise RuntimeError('No discography source answered')
1791 return found
1794def _deezer_releases(
1795 playlist: MonitoredPlaylist, db: PlaylistMonitorDB
1796) -> list[dict[str, Any]]:
1797 """The watched artist's Deezer discography, as release summaries.
1799 The Deezer artist is found by name on the first check (the most
1800 followed artist of exactly that name) and kept on the watch. When a
1801 "new releases only" watch gets it, everything Deezer lists at that
1802 point is recorded as already there - it was out before the watch
1803 started, like the YouTube Music releases its baseline recorded.
1804 """
1806 artist_id = playlist.deezer_id
1807 linked = False
1808 if not artist_id:
1809 artist_id = deezer.lookup_artist_id(playlist.name) or ''
1810 if not artist_id:
1811 logger.info('No Deezer artist named "{}"', playlist.name)
1812 return []
1813 linked = True
1814 _name, _cover, rows = deezer.artist_page_from_id(artist_id)
1815 releases = [
1816 {**row, 'album_id': DEEZER_RELEASE + str(row['album_id'])}
1817 for row in rows
1818 if row.get('album_id')
1819 ]
1820 if linked:
1821 db.update_playlist(playlist.id, deezer_id=artist_id)
1822 playlist.deezer_id = artist_id
1823 logger.info(
1824 'Artist "{}" is Deezer artist {} ({} releases)',
1825 playlist.name,
1826 artist_id,
1827 len(releases),
1828 )
1829 if playlist.new_only and not playlist.baseline_pending:
1830 db.mark_albums_skipped(playlist.id, out_already(releases))
1831 return releases
1834def out_already(albums: list[dict[str, Any]]) -> list[dict[str, Any]]:
1835 """*albums* without the announced ones: a "new releases only" watch
1836 records what is out as already there, never what is still to come."""
1838 return [a for a in albums if not upcoming(a)]
1841def spread_release_dates(albums: list[dict[str, Any]]) -> None:
1842 """Give every listing of a release the date one catalogue has for
1843 it (YouTube Music gives a year at most), so an announced album that
1844 another catalogue lists already waits for its release day too."""
1846 dates: dict[tuple[str, str], str] = {}
1847 for album in albums:
1848 day = release_day(album)
1849 key = (
1850 release_key(str(album.get('name') or '')),
1851 release_type_of(album),
1852 )
1853 if len(day) > len(dates.get(key, '')):
1854 dates[key] = day
1855 for album in albums:
1856 key = (
1857 release_key(str(album.get('name') or '')),
1858 release_type_of(album),
1859 )
1860 if len(dates.get(key, '')) > len(release_day(album)):
1861 album['release_date'] = dates[key]
1864def _musicbrainz_releases(
1865 playlist: MonitoredPlaylist,
1866 db: PlaylistMonitorDB,
1867 settings: dict[str, Any],
1868 listed: list[dict[str, Any]],
1869) -> list[dict[str, Any]]:
1870 """The watched artist's MusicBrainz release groups that the other
1871 catalogues (*listed*) don't have, as release summaries.
1873 The MusicBrainz artist is found on the first check - through the
1874 Deezer artist found just before, the YouTube Music channel, or an
1875 unambiguous name (:func:`musicbrainz.artist_id_for`) - and kept on
1876 the watch. Only the artist's own groups are kept, not the ones they
1877 are a guest on. Groups of every kind are listed; the sweep downloads
1878 those whose secondary types the watch lets in (``musicbrainz_kinds``,
1879 else ``artist_watch_musicbrainz_kinds``: mixtapes and soundtracks by
1880 default) or the user switched on (see :mod:`downtify.artist_catalogue`).
1882 A group another catalogue lists under the same name and type is left
1883 to that catalogue: MusicBrainz fills gaps, it doesn't fetch every
1884 release a third time. When a "new releases only" watch first finds
1885 the artist, every group is recorded as already there, as for Deezer.
1886 """
1888 artist_id = playlist.musicbrainz_id
1889 linked = False
1890 if not artist_id:
1891 artist_id = musicbrainz.artist_id_for(
1892 playlist.name,
1893 deezer_id=playlist.deezer_id,
1894 channel_id=playlist.spotify_id,
1895 )
1896 if not artist_id:
1897 logger.info(
1898 'No MusicBrainz artist settled for "{}" (none, or several '
1899 'of that name)',
1900 playlist.name,
1901 )
1902 return []
1903 linked = True
1904 # Every kind is listed (the watch's page shows them all); which ones
1905 # download is the sweep's call (artist_catalogue.wanted).
1906 every_kind = set(musicbrainz.KINDS.values())
1907 releases = [
1908 {
1909 **musicbrainz.release_summary(group),
1910 'album_id': MUSICBRAINZ_RELEASE + group['id'],
1911 }
1912 for group in musicbrainz.release_groups(artist_id)
1913 if musicbrainz.wanted(
1914 group, artist_id, musicbrainz.PRIMARY_TYPES, every_kind
1915 )
1916 ]
1917 if linked:
1918 db.update_playlist(playlist.id, musicbrainz_id=artist_id)
1919 playlist.musicbrainz_id = artist_id
1920 logger.info(
1921 'Artist "{}" is MusicBrainz artist {} ({} release groups)',
1922 playlist.name,
1923 artist_id,
1924 len(releases),
1925 )
1926 if playlist.new_only and not playlist.baseline_pending:
1927 db.mark_albums_skipped(playlist.id, out_already(releases))
1928 known = {
1929 (release_key(str(a.get('name') or '')), release_type_of(a))
1930 for a in listed
1931 }
1932 return [
1933 r
1934 for r in releases
1935 if (release_key(r['name']), release_type_of(r)) not in known
1936 ]
1939def _release_tracks(
1940 album: dict[str, Any], artist_name: str
1941) -> Optional[list[dict[str, Any]]]:
1942 """A release's tracks, from the catalogue that listed it.
1944 ``None`` for a release not to download, recorded as skipped: a
1945 Deezer release that isn't this artist's (Deezer lists the singles an
1946 artist is a guest on among their own - see :func:`_credited_release`),
1947 or a MusicBrainz group without an official edition (a bootleg).
1948 MusicBrainz groups were already narrowed to the artist's own
1949 (:func:`musicbrainz.wanted`).
1950 """
1952 album_id = str(album['album_id'])
1953 if album_id.startswith(MUSICBRAINZ_RELEASE):
1954 edition, songs = musicbrainz.release_group_tracks(
1955 album_id[len(MUSICBRAINZ_RELEASE) :]
1956 )
1957 return songs if edition else None
1958 if not album_id.startswith(DEEZER_RELEASE):
1959 return providers.album_tracks_from_browse_id(album_id)
1960 owner, main, songs = deezer.album_credits_from_id(
1961 album_id[len(DEEZER_RELEASE) :]
1962 )
1963 name = str(album.get('name') or '')
1964 titles = [name, *(str(s.get('name') or '') for s in songs)]
1965 if not _credited_release(artist_name, owner, main, titles):
1966 return None
1967 return [
1968 {**song, 'album_name': song.get('album_name') or name}
1969 for song in songs
1970 ]
1973#: The catalogue whose tracklist is read first for a release several of
1974#: them list: Deezer's API is the quickest, MusicBrainz's the slowest
1975#: (one request a second).
1976_TRACKLIST_ORDER = ('deezer', 'youtube', 'musicbrainz')
1979def take_inventory(
1980 playlist: MonitoredPlaylist,
1981 db: PlaylistMonitorDB,
1982 *,
1983 progress: Optional[Callable[[int, int], None]] = None,
1984 stop: Optional[Callable[[], bool]] = None,
1985) -> dict[str, int]:
1986 """Read the tracklist of every release in an artist watch's catalogue
1987 that has none kept yet ("Check what I have"), so its page can count
1988 what the library has of each - downloading nothing.
1990 A release two catalogues list is read once, from the quickest
1991 (:data:`_TRACKLIST_ORDER`). One that turns out not to be the
1992 artist's own (a guest spot, a bootleg) is marked so. A release a
1993 catalogue fails on is left for the next check. ``{checked, failed,
1994 total}``; *progress* hears ``(done, total)`` after each release and
1995 *stop* can end it early."""
1997 kept = set(db.catalogue.tracks(playlist.id))
1998 groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
1999 for row in db.catalogue.rows(playlist.id):
2000 key = (release_key(row['name']), release_type_of(row))
2001 groups.setdefault(key, []).append(row)
2002 todo = []
2003 for rows in groups.values():
2004 if any(r['album_id'] in kept or r['not_own'] for r in rows):
2005 continue
2006 rows.sort(
2007 key=lambda r: (
2008 _TRACKLIST_ORDER.index(r['source'])
2009 if r['source'] in _TRACKLIST_ORDER
2010 else len(_TRACKLIST_ORDER)
2011 )
2012 )
2013 todo.append(rows[0])
2014 checked = failed = 0
2015 for row in todo:
2016 if stop is not None and stop():
2017 break
2018 album = {'album_id': row['album_id'], 'name': row['name']}
2019 try:
2020 tracks = _release_tracks(album, playlist.name)
2021 except Exception:
2022 failed += 1
2023 logger.opt(exception=True).info(
2024 'Could not read the tracks of {} ({})',
2025 row['name'],
2026 row['album_id'],
2027 )
2028 else:
2029 if tracks is None:
2030 db.catalogue.set_not_own(playlist.id, row['album_id'])
2031 else:
2032 db.catalogue.store_tracks(playlist.id, row['album_id'], tracks)
2033 checked += 1
2034 if progress is not None:
2035 progress(checked + failed, len(todo))
2036 logger.info(
2037 'Artist "{}": read the tracks of {} release(s), {} failed',
2038 playlist.name,
2039 checked,
2040 failed,
2041 )
2042 return {'checked': checked, 'failed': failed, 'total': len(todo)}
2045def _credited_release(
2046 artist: str, owner: str, main: list[str], titles: list[str]
2047) -> bool:
2048 """Whether a Deezer release is *artist*'s own: credited to them, or
2049 a joint release (*$ome $exy $ongs 4 U* is PARTYNEXTDOOR's and
2050 Drake's) - they are one of its ``Main`` contributors and no title
2051 names them as a guest. Deezer gives a guest the ``Main`` role too,
2052 so a ``(feat. Drake)`` single is told apart by its title alone."""
2054 wanted = file_name_key(artist)
2055 if not owner or file_name_key(owner) == wanted:
2056 return True
2057 if wanted not in {file_name_key(m) for m in main}:
2058 return False
2059 guest = re.compile(
2060 r'\b(?:feat|ft|featuring|with)\b\.?[^)\]]*?(?<!\w)'
2061 + re.escape(artist.casefold())
2062 + r'(?!\w)'
2063 )
2064 return not any(guest.search(title.casefold()) for title in titles)
2067def _release_count(albums: list[dict[str, Any]]) -> int:
2068 """Distinct releases, by name: one listed by both catalogues counts
2069 once."""
2071 return len({
2072 file_name_key(str(a.get('name') or '')) or str(a.get('album_id'))
2073 for a in albums
2074 })
2077def _locate(
2078 filename: str, downloader: Downloader, settings: dict[str, Any]
2079) -> Optional[Path]:
2080 root = Path(downloader.download_dir)
2081 slskd = slskd_dir_from_downloader(downloader)
2082 extras = extra_dirs_from_settings(settings, root, slskd)
2083 return locate_library_file(filename, root, slskd, extras)
2086def _release_file_present(
2087 filename: str,
2088 song: dict[str, Any],
2089 downloader: Downloader,
2090 settings: dict[str, Any],
2091) -> bool:
2092 path = _locate(filename, downloader, settings)
2093 return path is not None and matches_release(path, song)
2096def _library_file_exists(
2097 filename: str, downloader: Downloader, settings: dict[str, Any]
2098) -> bool:
2099 return _locate(filename, downloader, settings) is not None
2102def _library_songs(library: Optional[LibraryStores]) -> Optional[LibrarySongs]:
2103 """The library by song, for an artist sweep; ``None`` without it."""
2105 if library is None or library.listing is None:
2106 return None
2107 try:
2108 return LibrarySongs(library.listing())
2109 except Exception:
2110 logger.opt(exception=True).warning(
2111 'Could not read the library to match artist watch tracks'
2112 )
2113 return None
2116def _download_release_track(
2117 downloader: Downloader,
2118 song: dict[str, Any],
2119 album: dict[str, Any],
2120 progress: Optional[ProgressCallback],
2121 observed: dict[str, Any],
2122 *,
2123 complete_albums: bool = False,
2124) -> str:
2125 """Download one track of a watched artist's release.
2127 Its file name can already be taken by another release's file, which
2128 the downloader refuses rather than overwrite. When that file holds
2129 this very song (a single that is also on the album, a deluxe
2130 edition) it is linked: *observed* gets ``reused`` and ``other_release``
2131 and its name is returned. When it is a *different* recording of the
2132 same name - another album's ``Intro``, with files organized by
2133 artist but not by album - the track is saved as ``<name> (<album>)``
2134 instead, so both songs are kept and the release can be completed.
2135 With *complete_albums* the same recording is saved that way too, so
2136 every release keeps a copy of its own (issue #546).
2137 """
2139 try:
2140 return downloader.download(song, progress)
2141 except FileExistsError:
2142 taken = downloader.existing_filename_for(song)
2143 if (
2144 not complete_albums
2145 and taken
2146 and same_recording(Path(downloader.download_dir) / taken, song)
2147 ):
2148 observed.update(reused=True, other_release=True)
2149 return taken
2150 suffix = str(album.get('name') or song.get('album_name') or '').strip()
2151 if not suffix:
2152 raise
2153 logger.info(
2154 'The file name of "{}" is taken by another recording; '
2155 'saving this one with "({})" added',
2156 song.get('name', ''),
2157 suffix,
2158 )
2159 return downloader.download(
2160 {**song, '_output_suffix': suffix}, progress
2161 )
2164async def check_artist( # noqa: PLR0914 # one sweep: discography, releases, tracks
2165 playlist: MonitoredPlaylist,
2166 db: PlaylistMonitorDB,
2167 downloader: Downloader,
2168 broadcast: Callable[[dict[str, Any]], Any],
2169 loop: asyncio.AbstractEventLoop,
2170 settings: Optional[dict[str, Any]] = None,
2171 library: Optional[LibraryStores] = None,
2172 *,
2173 fill_missing: bool = False,
2174) -> int:
2175 """Download every release of a watched artist that isn't known yet.
2177 With *fill_missing* ("Download missing" on the watch's page), every
2178 release the watch downloads is gone through again, known or not, so
2179 the songs missing from it - deleted since, or never fetched - are
2180 downloaded (or linked when the library has them under another name).
2181 Releases out before a "new releases only" watch stay skipped unless
2182 switched on.
2184 ``playlist.spotify_id`` is the artist's YouTube Music channel id. The
2185 discography is listed once per sweep - YouTube Music's, plus Deezer's
2186 with ``artist_watch_deezer`` on (see :func:`_artist_discography`);
2187 only releases missing from ``seen_albums`` have their tracklists
2188 fetched, so a steady-state sweep costs a request per source. A song
2189 the library already has, from another release or another source, is
2190 linked rather than downloaded again. Returns the number of tracks
2191 downloaded.
2193 Only releases of the watch's ``release_types`` are downloaded; one of
2194 another type is left unmarked, so turning that type on later picks it
2195 up. A "new releases only" watch first records everything already out
2196 as skipped (``baseline_pending``) and downloads nothing that sweep -
2197 but only once the discography actually came back non-empty, since an
2198 empty answer is as likely a YouTube Music hiccup as an artist with no
2199 releases, and taking it at face value would download the whole back
2200 catalog on the next sweep.
2201 """
2202 logger.info(
2203 'Checking monitored artist "{}" ({})',
2204 playlist.name,
2205 playlist.spotify_id,
2206 )
2208 try:
2209 albums = await asyncio.to_thread(
2210 _artist_discography, playlist, db, settings or {}
2211 )
2212 except Exception:
2213 logger.exception(
2214 'Failed to fetch discography for artist {}', playlist.spotify_id
2215 )
2216 await asyncio.to_thread(
2217 db.update_playlist, playlist.id, last_checked=_now_iso()
2218 )
2219 return 0
2220 spread_release_dates(albums)
2221 # The watch's page lists the whole catalogue from this listing.
2222 await asyncio.to_thread(db.catalogue.record, playlist.id, albums)
2224 if playlist.baseline_pending:
2225 if not albums:
2226 logger.info(
2227 'Artist "{}": no releases listed yet; will record the '
2228 'existing ones on the next check',
2229 playlist.name,
2230 )
2231 await asyncio.to_thread(
2232 db.update_playlist, playlist.id, last_checked=_now_iso()
2233 )
2234 return 0
2235 skipped = await asyncio.to_thread(
2236 db.mark_albums_skipped, playlist.id, out_already(albums)
2237 )
2238 logger.info(
2239 'Artist "{}" watches new releases only: {} existing release(s) '
2240 'skipped',
2241 playlist.name,
2242 skipped,
2243 )
2244 await asyncio.to_thread(
2245 db.update_playlist,
2246 playlist.id,
2247 baseline_pending=0,
2248 last_checked=_now_iso(),
2249 last_track_count=_release_count(albums),
2250 )
2251 return 0
2253 wanted_types = set(
2254 (playlist.release_types or ALL_RELEASE_TYPES).split(',')
2255 )
2256 seen = await asyncio.to_thread(db.get_seen_album_ids, playlist.id)
2257 seen.discard(
2258 await asyncio.to_thread(db.releases.next_release, playlist.id)
2259 )
2260 if fill_missing:
2261 processed = await asyncio.to_thread(db.catalogue.seen, playlist.id)
2262 seen = {album_id for album_id, skipped in processed.items() if skipped}
2263 # The watch's types (and MusicBrainz kinds), unless the user switched
2264 # a release on or off on the watch's page.
2265 kinds = watch_kinds(playlist, settings or {})
2266 choices = await asyncio.to_thread(db.catalogue.choices, playlist.id)
2267 new_albums = [
2268 a
2269 for a in albums
2270 if a.get('album_id')
2271 and a['album_id'] not in seen
2272 and release_wanted(a, wanted_types, kinds, choices)
2273 # Announced, not out yet: downloaded on its release day.
2274 and not upcoming(a)
2275 ]
2276 if new_albums:
2277 logger.info(
2278 'Found {} new release(s) for artist "{}"',
2279 len(new_albums),
2280 playlist.name,
2281 )
2283 delay_seconds = (settings or {}).get('download_delay_seconds', 0) or 0
2284 complete_albums = bool(
2285 (settings or {}).get('artist_watch_complete_albums')
2286 )
2287 known_tracks = await asyncio.to_thread(db.get_track_filenames, playlist.id)
2288 # Songs the library already holds under another name - a playlist
2289 # download named from Spotify, another release of this artist - are
2290 # linked rather than downloaded again.
2291 songs = (
2292 await asyncio.to_thread(_library_songs, library)
2293 if new_albums
2294 else None
2295 )
2297 downloaded = 0
2298 for album in new_albums:
2299 album_id = album['album_id']
2300 await asyncio.to_thread(db.releases.attempt, playlist.id, album_id)
2301 try:
2302 tracks = await asyncio.to_thread(
2303 _release_tracks, album, playlist.name
2304 )
2305 except Exception:
2306 logger.exception(
2307 'Failed to fetch tracks for release {} ({})',
2308 album.get('name'),
2309 album_id,
2310 )
2311 continue
2312 if tracks is None:
2313 # Another artist's release this one is only featured on, or
2314 # a bootleg.
2315 await asyncio.to_thread(
2316 db.mark_albums_skipped, playlist.id, [album]
2317 )
2318 await asyncio.to_thread(
2319 db.catalogue.set_not_own, playlist.id, album_id
2320 )
2321 continue
2322 # The watch's page counts what the library has of it from this.
2323 await asyncio.to_thread(
2324 db.catalogue.store_tracks, playlist.id, album_id, tracks
2325 )
2327 complete = bool(tracks) and max(
2328 (int(s.get('album_track_total') or 0) for s in tracks), default=0
2329 ) <= len(tracks)
2330 release_tracks = await asyncio.to_thread(
2331 db.releases.tracks, playlist.id, album_id
2332 )
2333 for position, song in enumerate(tracks, start=1):
2334 track_id = song.get('song_id')
2335 if not track_id:
2336 complete = False
2337 continue
2338 filename = release_tracks.get(
2339 (position, track_id), known_tracks.get(track_id)
2340 )
2341 if filename and await asyncio.to_thread(
2342 _release_file_present,
2343 filename,
2344 song,
2345 downloader,
2346 settings or {},
2347 ):
2348 await asyncio.to_thread(
2349 db.releases.record,
2350 playlist.id,
2351 album_id,
2352 position,
2353 track_id,
2354 filename,
2355 )
2356 continue
2357 # "Keep every album complete": each release gets its own copy
2358 # of a song it shares with another (a greatest hits, a deluxe
2359 # edition, a single) instead of linking the one already there.
2360 same = (
2361 songs.find(song)
2362 if songs is not None and not complete_albums
2363 else None
2364 )
2365 if same and await asyncio.to_thread(
2366 _library_file_exists, same, downloader, settings or {}
2367 ):
2368 await asyncio.to_thread(
2369 functools.partial(
2370 db.mark_track_downloaded,
2371 playlist.id,
2372 track_id,
2373 same,
2374 outcome=OUTCOME_LINKED,
2375 initial=playlist.last_checked is None,
2376 )
2377 )
2378 await asyncio.to_thread(
2379 db.releases.record,
2380 playlist.id,
2381 album_id,
2382 position,
2383 track_id,
2384 same,
2385 )
2386 known_tracks[track_id] = same
2387 continue
2388 if await asyncio.to_thread(db.download_policy.blocked, song):
2389 complete = False
2390 continue
2391 await queue_pause.wait()
2392 try:
2393 observed: dict[str, Any] = {}
2394 async with (
2395 _download_slot(song, playlist.name, broadcast),
2396 queue_pause.running(),
2397 ):
2398 filename = await loop.run_in_executor(
2399 DOWNLOAD_EXECUTOR,
2400 lambda s=song, a=album, progress_state=observed: (
2401 _download_release_track(
2402 downloader,
2403 {**s, '_monitor_release': a['album_id']},
2404 a,
2405 _progress_cb(
2406 s,
2407 playlist.name,
2408 broadcast,
2409 loop,
2410 progress_state,
2411 ),
2412 progress_state,
2413 complete_albums=complete_albums,
2414 )
2415 ),
2416 )
2417 if not filename or not (
2418 observed.get('other_release')
2419 or await asyncio.to_thread(
2420 _release_file_present,
2421 filename,
2422 song,
2423 downloader,
2424 settings or {},
2425 )
2426 ):
2427 raise ValueError(
2428 'Downloaded file does not validate against the requested release'
2429 )
2430 await asyncio.to_thread(
2431 functools.partial(
2432 db.mark_track_downloaded,
2433 playlist.id,
2434 track_id,
2435 filename,
2436 outcome=_outcome_of(observed),
2437 provider=observed.get('provider', ''),
2438 # The first check downloads the back catalog.
2439 initial=playlist.last_checked is None,
2440 )
2441 )
2442 await asyncio.to_thread(
2443 db.releases.record,
2444 playlist.id,
2445 album_id,
2446 position,
2447 track_id,
2448 filename,
2449 )
2450 await asyncio.to_thread(db.download_policy.succeeded, song)
2451 invalidate_library_paths_cache()
2452 known_tracks[track_id] = filename
2453 if songs is not None:
2454 songs.add(filename, song)
2455 downloaded += int(not observed.get('reused'))
2456 if library is not None and filename:
2457 await asyncio.to_thread(
2458 _record_artist_sweep_owner,
2459 library,
2460 filename,
2461 downloader,
2462 playlist.user_id,
2463 )
2464 await broadcast({
2465 'song': song,
2466 'progress': 100,
2467 'message': 'Already downloaded'
2468 if observed.get('reused')
2469 else 'Done',
2470 'status': 'done',
2471 'filename': filename,
2472 'playlist_name': playlist.name,
2473 'reused': bool(observed.get('reused')),
2474 'provider': observed.get('provider', ''),
2475 })
2476 if delay_seconds > 0 and not observed.get('reused'):
2477 await asyncio.sleep(delay_seconds)
2478 except DownloadPolicyBlocked:
2479 complete = False
2480 await broadcast({
2481 'song': song,
2482 'status': 'failed',
2483 'progress': 0,
2484 'message': 'download_excluded',
2485 'playlist_name': playlist.name,
2486 })
2487 except Exception as exc:
2488 complete = False
2489 logger.exception(
2490 'Failed to auto-download track {} of release {}',
2491 track_id,
2492 album.get('name'),
2493 )
2494 await asyncio.to_thread(
2495 _record_failure, db, playlist.id, song, exc
2496 )
2497 await broadcast({
2498 'song': song,
2499 'status': 'error',
2500 'message': 'Artist release download or validation failed',
2501 'playlist_name': playlist.name,
2502 })
2504 # Only remember the release once every track is accounted for, so
2505 # a transient failure is retried on the next sweep instead of
2506 # being silently skipped forever. Tracks that already succeeded
2507 # are cheap to skip via `known_tracks`.
2508 if complete:
2509 await asyncio.to_thread(
2510 db.mark_album_seen, playlist.id, album_id, album.get('name')
2511 )
2513 await asyncio.to_thread(
2514 db.update_playlist,
2515 playlist.id,
2516 last_checked=_now_iso(),
2517 last_track_count=_release_count(albums),
2518 )
2519 return downloaded
2522def _regenerate_m3u(
2523 playlist: MonitoredPlaylist,
2524 tracks: list[dict[str, Any]],
2525 downloader: Downloader,
2526 resolved: Optional[dict[str, str]] = None,
2527 known_tracks: Optional[dict[str, Optional[str]]] = None,
2528) -> Optional[Path]:
2529 """Rewrite the playlist's M3U, in playlist order. Returns its path,
2530 or ``None`` when nothing was written.
2532 Walks the full ordered track list (not just the freshly downloaded
2533 ones) and hands the entries to :func:`m3u.write_m3u`. Tracks with no
2534 file are dropped, so a partially-downloaded playlist still yields a
2535 valid, correctly-ordered M3U.
2537 With *resolved* (``{track_id: filename}``) filenames are taken from
2538 that map alone — the cheap path used for the rewrite after each
2539 individual download. Without it every track is resolved against the
2540 filesystem instead, which is the authoritative view used for the
2541 final rewrite at the end of a sweep: the watch's recorded filename
2542 (``known_tracks``) when that file exists — it may be a track linked
2543 from elsewhere in the library, e.g. an slskd download — else the
2544 track's expected path in the playlist folder.
2545 """
2547 pl_subdir = m3u.sanitize_playlist_name(playlist.name)
2548 download_dir = Path(downloader.download_dir)
2549 slskd_dir = slskd_dir_from_downloader(downloader)
2550 entries: list[dict[str, Any]] = []
2551 for song in tracks:
2552 if resolved is not None:
2553 filename = resolved.get(song.get('song_id') or '')
2554 else:
2555 filename = (known_tracks or {}).get(song.get('song_id') or '')
2556 if not (
2557 filename
2558 and locate_library_file(filename, download_dir, slskd_dir)
2559 ):
2560 filename = downloader.existing_filename_for(
2561 song, subdir=pl_subdir
2562 )
2563 if not filename:
2564 continue
2565 entries.append({
2566 'filename': filename,
2567 'title': song.get('name', ''),
2568 'artist': ', '.join(song.get('artists') or []),
2569 'duration': song.get('duration', 0),
2570 })
2571 if not entries:
2572 logger.warning(
2573 'M3U skip for monitored playlist "{}": no tracks on disk',
2574 playlist.name,
2575 )
2576 return None
2577 # When organize-by-artist/album is on the tracks live in those folders
2578 # rather than the per-playlist subfolder, so the M3U goes to the legacy
2579 # Playlists/ directory where its relative paths still resolve.
2580 organize = downloader.organize_by_artist or downloader.organize_by_album
2581 path, _kept = m3u.write_m3u(
2582 downloader.download_dir,
2583 playlist.name,
2584 entries,
2585 playlist_subdir=None if organize else pl_subdir,
2586 slskd_dir=slskd_dir,
2587 )
2588 return path
2591def _download_cover_for_watch(
2592 playlist: MonitoredPlaylist,
2593 downloader: Downloader,
2594 settings: dict[str, Any],
2595) -> Optional[Path]:
2596 """Save a watched playlist's cover where its M3U lives.
2598 Called once per sweep that has tracks to fetch, before the first of
2599 them — the path is resolved from the playlist's name rather than
2600 read back from a written M3U, so the artwork is in place while the
2601 folder fills up.
2602 """
2604 organize = downloader.organize_by_artist or downloader.organize_by_album
2605 pl_subdir = m3u.sanitize_playlist_name(playlist.name)
2606 m3u_path = m3u.m3u_path_for(
2607 downloader.download_dir,
2608 playlist.name,
2609 playlist_subdir=None if organize else pl_subdir,
2610 )
2611 return download_playlist_cover(
2612 playlist.source, playlist.spotify_id, m3u_path, settings
2613 )
2616# Ids of watches with a check running right now. Adding a watch starts its
2617# first check immediately, and the background sweep would otherwise start
2618# a second one for the same never-checked watch while the first is still
2619# downloading — both then fetch the same tracks into the same files.
2620_checks_running: set[int] = set()
2623def check_running(watch_id: int) -> bool:
2624 """Whether a check of the watch is running right now."""
2626 return watch_id in _checks_running
2629async def check_podcast_watch(
2630 playlist: MonitoredPlaylist,
2631 db: PlaylistMonitorDB,
2632 podcasts: PodcastStore,
2633 download_dir: Path,
2634 broadcast: Callable[[dict[str, Any]], Any],
2635 loop: asyncio.AbstractEventLoop,
2636) -> int:
2637 """Sync one podcast watch: ``playlist.spotify_id`` is its feed URL.
2639 Only the scheduling row lives here; the show itself, its retention
2640 policy and its episodes are ``podcasts.py``'s job — this is just the
2641 glue ``check_watch`` needs to treat a podcast like any other kind of
2642 watch. Returns the number of episodes downloaded.
2643 """
2645 show = await asyncio.to_thread(
2646 podcasts.get_show_by_feed_url, playlist.spotify_id
2647 )
2648 if show is None:
2649 logger.warning(
2650 'Podcast watch "{}" has no matching show row (feed {})',
2651 playlist.name,
2652 playlist.spotify_id,
2653 )
2654 await asyncio.to_thread(
2655 db.update_playlist, playlist.id, last_checked=_now_iso()
2656 )
2657 return 0
2659 def _progress(info: dict[str, Any]) -> None:
2660 asyncio.run_coroutine_threadsafe(
2661 broadcast({'type': 'podcast_progress', **info}), loop
2662 )
2664 try:
2665 async with queue_pause.running():
2666 count = await asyncio.to_thread(
2667 sync_show,
2668 podcasts,
2669 show,
2670 download_dir,
2671 progress=_progress,
2672 should_continue=lambda: not queue_pause.paused,
2673 )
2674 except Exception:
2675 logger.exception('Podcast sync failed for "{}"', playlist.name)
2676 count = 0
2677 await asyncio.to_thread(
2678 db.update_playlist, playlist.id, last_checked=_now_iso()
2679 )
2680 if count:
2681 asyncio.run_coroutine_threadsafe(broadcast({'type': 'podcasts'}), loop)
2682 return count
2685async def check_watch(
2686 playlist: MonitoredPlaylist,
2687 db: PlaylistMonitorDB,
2688 downloader: Downloader,
2689 broadcast: Callable[[dict[str, Any]], Any],
2690 loop: asyncio.AbstractEventLoop,
2691 settings: Optional[dict[str, Any]] = None,
2692 library: Optional[LibraryStores] = None,
2693 podcasts: Optional[PodcastStore] = None,
2694 *,
2695 fill_missing: bool = False,
2696) -> int:
2697 saved = settings if settings is not None else {}
2698 if enabled(saved):
2699 return 0
2700 await queue_pause.wait()
2701 # Only a "Download missing" check carries the option.
2702 extra = {'fill_missing': True} if fill_missing else {}
2703 with policy.external(saved):
2704 return await _check_watch_online(
2705 playlist,
2706 db,
2707 downloader,
2708 broadcast,
2709 loop,
2710 settings,
2711 library,
2712 podcasts,
2713 **extra,
2714 )
2717def _watch_cover_file(
2718 playlist: MonitoredPlaylist, downloader: Downloader
2719) -> str:
2720 """Where a watched playlist's cover is saved (see
2721 :func:`_download_cover_for_watch`), relative to the download folder -
2722 what ``/playlist-cover?file=`` takes. ``''`` for an artist (the page
2723 shows the artist's photo) or when the path can't be worked out."""
2725 if playlist.kind != KIND_PLAYLIST:
2726 return ''
2727 try:
2728 organize = (
2729 downloader.organize_by_artist or downloader.organize_by_album
2730 )
2731 root = Path(downloader.download_dir)
2732 m3u_path = m3u.m3u_path_for(
2733 root,
2734 playlist.name,
2735 playlist_subdir=None
2736 if organize
2737 else m3u.sanitize_playlist_name(playlist.name),
2738 )
2739 return Path(m3u_path).with_suffix('.jpg').relative_to(root).as_posix()
2740 except Exception:
2741 return ''
2744def queue_watch(
2745 playlist: MonitoredPlaylist, downloader: Downloader
2746) -> dict[str, Any]:
2747 """The watch a download came from, as the queue shows it."""
2749 return {
2750 'id': playlist.id,
2751 'name': playlist.name,
2752 'kind': playlist.kind,
2753 'url': playlist.url,
2754 'source': playlist.source,
2755 'cover_file': _watch_cover_file(playlist, downloader),
2756 }
2759def with_watch(
2760 broadcast: Callable[[dict[str, Any]], Any], watch: dict[str, Any]
2761) -> Callable[[dict[str, Any]], Any]:
2762 """*broadcast*, adding ``watch`` to every song event, so the queue can
2763 say which watch is downloading each song."""
2765 def send(message: dict[str, Any]) -> Any:
2766 if isinstance(message, dict) and 'song' in message:
2767 message = {**message, 'watch': watch}
2768 return broadcast(message)
2770 return send
2773async def _check_watch_online(
2774 playlist: MonitoredPlaylist,
2775 db: PlaylistMonitorDB,
2776 downloader: Downloader,
2777 broadcast: Callable[[dict[str, Any]], Any],
2778 loop: asyncio.AbstractEventLoop,
2779 settings: Optional[dict[str, Any]] = None,
2780 library: Optional[LibraryStores] = None,
2781 podcasts: Optional[PodcastStore] = None,
2782 *,
2783 fill_missing: bool = False,
2784) -> int:
2785 """Run the right check for a watch by kind, unless one is already running."""
2786 if playlist.id in _checks_running:
2787 logger.info('Watch "{}" is already being checked', playlist.name)
2788 return 0
2789 _checks_running.add(playlist.id)
2790 try:
2791 if playlist.kind != KIND_PODCAST:
2792 broadcast = with_watch(
2793 broadcast, queue_watch(playlist, downloader)
2794 )
2795 if playlist.kind == KIND_PODCAST:
2796 if podcasts is None:
2797 logger.warning(
2798 'Podcast watch "{}" but podcast store not ready',
2799 playlist.name,
2800 )
2801 return 0
2802 return await check_podcast_watch(
2803 playlist,
2804 db,
2805 podcasts,
2806 Path(downloader.download_dir),
2807 broadcast,
2808 loop,
2809 )
2810 if playlist.kind == KIND_ARTIST:
2811 return await check_artist(
2812 playlist,
2813 db,
2814 downloader,
2815 broadcast,
2816 loop,
2817 settings,
2818 library,
2819 fill_missing=fill_missing,
2820 )
2821 return await check_playlist(
2822 playlist, db, downloader, broadcast, loop, settings, library
2823 )
2824 finally:
2825 _checks_running.discard(playlist.id)
2828async def monitor_loop(
2829 db: PlaylistMonitorDB,
2830 get_downloader: Callable[[], Optional[Downloader]],
2831 broadcast: Callable[[dict[str, Any]], Any],
2832 loop: asyncio.AbstractEventLoop,
2833 settings: Optional[dict[str, Any]] = None,
2834 get_library: Optional[Callable[[], LibraryStores]] = None,
2835 get_podcasts: Optional[Callable[[], Optional[PodcastStore]]] = None,
2836) -> None:
2837 """Background task: sweep all enabled playlists that are due for checking."""
2838 while True:
2839 try:
2840 await queue_pause.wait()
2841 playlists = await asyncio.to_thread(db.list_playlists)
2842 for pl in playlists:
2843 await queue_pause.wait()
2844 if not pl.enabled:
2845 continue
2846 if not _is_due(pl.last_checked, pl.interval_minutes) and not (
2847 pl.kind == KIND_ARTIST
2848 and await asyncio.to_thread(
2849 db.catalogue.release_day_arrived,
2850 pl.id,
2851 pl.last_checked,
2852 )
2853 ):
2854 continue
2855 downloader = get_downloader()
2856 if downloader is None:
2857 continue
2858 try:
2859 count = await check_watch(
2860 pl,
2861 db,
2862 downloader,
2863 broadcast,
2864 loop,
2865 settings,
2866 get_library() if get_library else None,
2867 get_podcasts() if get_podcasts else None,
2868 )
2869 if count > 0:
2870 logger.info(
2871 'Auto-downloaded {} new track(s) from "{}"',
2872 count,
2873 pl.name,
2874 )
2875 except Exception:
2876 logger.exception(
2877 'Error while checking watch "{}"', pl.name
2878 )
2879 except Exception:
2880 logger.exception('Unexpected error in monitor loop')
2881 await asyncio.sleep(MONITOR_LOOP_INTERVAL)
2884async def reconcile_downloaded_tracks(
2885 db: PlaylistMonitorDB,
2886 downloader: Downloader,
2887 *,
2888 settings: Optional[dict[str, Any]] = None,
2889) -> int:
2890 """Drop ``downloaded_tracks`` rows absent from the registered library.
2892 ``check_playlist`` trusts a row in ``downloaded_tracks`` as proof a
2893 track was already downloaded without re-checking the filesystem (see
2894 its docstring) — otherwise a file moved elsewhere on disk would be
2895 re-downloaded on every single sweep. This is the other half of that
2896 trade-off: a track whose file has genuinely disappeared (deleted,
2897 not just moved) is still noticed here and forgotten, so it becomes
2898 eligible for re-download again on the watch's next check. Returns
2899 the number of rows removed. Configured external music folders are
2900 checked alongside downloads and the slskd source folder.
2901 """
2902 rows = await asyncio.to_thread(db.list_all_downloaded_tracks)
2903 slskd_dir = slskd_dir_from_downloader(downloader)
2904 extra_dirs = await asyncio.to_thread(
2905 extra_dirs_from_settings,
2906 settings or {},
2907 Path(downloader.download_dir),
2908 slskd_dir,
2909 )
2910 # One stat() per downloaded track — thousands of them on a slow disk
2911 # or network mount — so off the event loop.
2912 missing = await asyncio.to_thread(
2913 lambda: [
2914 (row['playlist_id'], row['track_spotify_id'])
2915 for row in rows
2916 if locate_library_file(
2917 row['filename'],
2918 downloader.download_dir,
2919 slskd_dir,
2920 extra_dirs,
2921 )
2922 is None
2923 ]
2924 )
2925 if missing:
2926 await asyncio.to_thread(db.remove_downloaded_tracks, missing)
2927 return len(missing)
2930async def reconcile_loop(
2931 db: PlaylistMonitorDB,
2932 get_downloader: Callable[[], Optional[Downloader]],
2933 interval_seconds: int = RECONCILE_LOOP_INTERVAL,
2934 *,
2935 settings: Optional[dict[str, Any]] = None,
2936) -> None:
2937 """Background task: hourly, prune downloaded-track records for files
2938 that are no longer in any registered library folder.
2940 Runs independently of :func:`monitor_loop` and its per-watch
2941 schedule — this sweep always checks every known downloaded track
2942 across every watch, on its own fixed cadence. External folder settings
2943 are resolved again each sweep, so changes need no restart.
2944 """
2945 while True:
2946 try:
2947 await queue_pause.wait()
2948 downloader = get_downloader()
2949 if downloader is not None:
2950 removed = await reconcile_downloaded_tracks(
2951 db, downloader, settings=settings
2952 )
2953 if removed:
2954 logger.info(
2955 '{} downloaded-track record(s) removed: file no '
2956 'longer found in the registered library folders',
2957 removed,
2958 )
2959 except Exception:
2960 logger.exception('Unexpected error in reconcile loop')
2961 await asyncio.sleep(interval_seconds)