Coverage for downtify/artist_catalogue.py: 95%
216 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""What a watched artist has released, and what the library has of it
2(issue #581, phase 2: the catalogue view).
4Every sweep of an artist watch lists the artist's discography from its
5catalogues (YouTube Music, Deezer, MusicBrainz - ``monitor.
6_artist_discography``). :class:`ArtistCatalogue` keeps the last listing
7of each release, so the watch's page can show the whole catalogue without
8asking the catalogues again, plus two things learnt along the way: how
9many tracks a release has (once its tracklist was fetched) and whether it
10turned out not to be the artist's own (a guest spot, a bootleg).
12It also keeps the user's **choice** per release, like Lidarr's per-album
13monitoring: ``on`` downloads it even when its type isn't watched or it
14came out before a "new releases only" watch started, ``off`` never
15downloads it (what is already there stays), ``''`` follows the watch.
17:func:`catalogue_view` puts it together for ``GET
18/api/monitor/playlists/{id}/releases``: one row per release - the same
19album listed by two catalogues is one row - with its state:
21``complete`` every track is in the library
22``partly`` some are
23``missing`` it is wanted and none are there yet (the next check tries)
24``upcoming`` it is wanted but announced, not out yet (``release_date`` is
25 in the future): the first check on or after its release
26 day downloads it
27``skipped`` out before a "new releases only" watch started
28``not_watched`` a type or kind the watch doesn't download
29``off`` switched off
30``not_own`` a guest spot or a bootleg: never downloaded
32**What the library has** of a release is counted against its tracklist
33(:meth:`ArtistCatalogue.store_tracks`): every track the library holds -
34the same artist and title, the same length, as :mod:`downtify.
35library_songs` matches them - whoever downloaded it. A sweep keeps the
36tracklists of the releases it processes; ``monitor.take_inventory``
37reads the others on request ("Check what I have"), downloading nothing.
38So an album the user had before the watch started shows as complete,
39not just "out before the watch".
40"""
42from __future__ import annotations
44import json
45import re
46from datetime import date, datetime, timezone
47from typing import Any, Callable, ContextManager, Iterable, Optional
49from . import musicbrainz
50from .db.config import Dialect
51from .db.core import Connection
52from .file_naming import file_name_key
53from .library_catalog import song_key
55CHOICES = ('', 'on', 'off')
57#: Row order of the states when one release is listed twice: the best
58#: news wins.
59_STATE_ORDER = (
60 'complete',
61 'partly',
62 'upcoming',
63 'missing',
64 'skipped',
65 'not_watched',
66 'off',
67 'not_own',
68)
70#: MusicBrainz's secondary types back to the names settings use.
71_KIND_NAMES = {mb: name for name, mb in musicbrainz.KINDS.items()}
74def _now() -> str:
75 return datetime.now(timezone.utc).isoformat()
78def _source_of(album_id: str, source: Any) -> str:
79 """The catalogue a release came from: its own ``source``, else read
80 off its id (``youtube``, as YouTube Music's own rows say)."""
82 if source:
83 return str(source)
84 if album_id.startswith('deezer:'):
85 return 'deezer'
86 if album_id.startswith('mb:'):
87 return 'musicbrainz'
88 return 'youtube'
91class ArtistCatalogue:
92 """The ``artist_catalogue`` table of the monitor database."""
94 def __init__(
95 self, connect: Callable[[], ContextManager[Connection]]
96 ) -> None:
97 self._connect = connect
98 with self._connect() as conn:
99 if getattr(conn, 'dialect', Dialect.SQLITE) is Dialect.SQLITE:
100 self._init_sqlite(conn)
102 @staticmethod
103 def _init_sqlite(conn: Connection) -> None:
104 """Create the SQLite-only schema; PostgreSQL migrations own it."""
105 conn.execute("""CREATE TABLE IF NOT EXISTS artist_catalogue (
106 playlist_id INTEGER NOT NULL,
107 album_id TEXT NOT NULL,
108 name TEXT NOT NULL,
109 release_type TEXT NOT NULL DEFAULT '',
110 kinds TEXT NOT NULL DEFAULT '',
111 year TEXT NOT NULL DEFAULT '',
112 source TEXT NOT NULL DEFAULT '',
113 cover_url TEXT NOT NULL DEFAULT '',
114 url TEXT NOT NULL DEFAULT '',
115 position INTEGER NOT NULL DEFAULT 0,
116 track_total INTEGER NOT NULL DEFAULT 0,
117 not_own INTEGER NOT NULL DEFAULT 0,
118 choice TEXT NOT NULL DEFAULT '',
119 listed_at TEXT NOT NULL,
120 release_date TEXT NOT NULL DEFAULT '',
121 PRIMARY KEY (playlist_id, album_id),
122 FOREIGN KEY (playlist_id) REFERENCES monitored_playlists(id)
123 ON DELETE CASCADE
124 )""")
125 columns = {
126 r[1] for r in conn.execute('PRAGMA table_info(artist_catalogue)')
127 }
128 if 'release_date' not in columns:
129 conn.execute(
130 'ALTER TABLE artist_catalogue ADD COLUMN release_date '
131 "TEXT NOT NULL DEFAULT ''"
132 )
133 # Each release's tracklist, once a sweep or "Check what I
134 # have" read it.
135 conn.execute("""CREATE TABLE IF NOT EXISTS artist_catalogue_tracks (
136 playlist_id INTEGER NOT NULL,
137 album_id TEXT NOT NULL,
138 position INTEGER NOT NULL,
139 name TEXT NOT NULL,
140 artists TEXT NOT NULL DEFAULT '[]',
141 duration REAL NOT NULL DEFAULT 0,
142 PRIMARY KEY (playlist_id, album_id, position),
143 FOREIGN KEY (playlist_id) REFERENCES monitored_playlists(id)
144 ON DELETE CASCADE
145 )""")
147 def record(
148 self, playlist_id: int, albums: Iterable[dict[str, Any]]
149 ) -> None:
150 """Keep a sweep's listing (what a release is called, its type,
151 cover...). What was learnt about a release and the user's choice
152 are kept as they are."""
154 now = _now()
155 with self._connect() as conn:
156 for position, album in enumerate(albums):
157 album_id = str(album.get('album_id') or '')
158 if not album_id:
159 continue
160 conn.execute(
161 """INSERT INTO artist_catalogue
162 (playlist_id, album_id, name, release_type, kinds,
163 year, source, cover_url, url, position, listed_at,
164 release_date)
165 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
166 ON CONFLICT(playlist_id, album_id) DO UPDATE SET
167 name = excluded.name,
168 release_type = excluded.release_type,
169 kinds = excluded.kinds,
170 year = excluded.year,
171 release_date = excluded.release_date,
172 source = excluded.source,
173 cover_url = excluded.cover_url,
174 url = excluded.url,
175 position = excluded.position,
176 listed_at = excluded.listed_at""",
177 (
178 playlist_id,
179 album_id,
180 str(album.get('name') or album_id),
181 str(album.get('release_type') or ''),
182 ','.join(album.get('kinds') or []),
183 str(album.get('year') or ''),
184 _source_of(album_id, album.get('source')),
185 str(album.get('cover_url') or ''),
186 str(album.get('url') or ''),
187 position,
188 now,
189 release_day(album),
190 ),
191 )
193 def release_day_arrived(
194 self, playlist_id: int, last_checked: Optional[str]
195 ) -> bool:
196 """Whether one of the watch's announced releases has come out
197 since its last check: the day it is due, whatever the interval."""
199 if not last_checked:
200 return False
201 try:
202 checked = datetime.fromisoformat(last_checked)
203 except ValueError:
204 return False
205 if checked.tzinfo is None:
206 checked = checked.replace(tzinfo=timezone.utc)
207 since = checked.astimezone().date().isoformat()
208 today = date.today().isoformat()
209 with self._connect() as conn:
210 row = conn.execute(
211 'SELECT 1 FROM artist_catalogue WHERE playlist_id = ? '
212 'AND length(release_date) = 10 AND release_date > ? '
213 'AND release_date <= ? LIMIT 1',
214 (playlist_id, since, today),
215 ).fetchone()
216 return row is not None
218 def store_tracks(
219 self, playlist_id: int, album_id: str, songs: list[dict[str, Any]]
220 ) -> None:
221 """Keep a release's tracklist (and its track count)."""
223 with self._connect() as conn:
224 conn.execute(
225 'DELETE FROM artist_catalogue_tracks '
226 'WHERE playlist_id = ? AND album_id = ?',
227 (playlist_id, album_id),
228 )
229 conn.executemany(
230 'INSERT INTO artist_catalogue_tracks VALUES (?,?,?,?,?,?)',
231 [
232 (
233 playlist_id,
234 album_id,
235 position,
236 str(song.get('name') or ''),
237 json.dumps([
238 str(a) for a in song.get('artists') or []
239 ]),
240 float(song.get('duration') or 0),
241 )
242 for position, song in enumerate(songs, start=1)
243 ],
244 )
245 conn.execute(
246 'UPDATE artist_catalogue SET track_total = ? '
247 'WHERE playlist_id = ? AND album_id = ?',
248 (len(songs), playlist_id, album_id),
249 )
251 def tracks(self, playlist_id: int) -> dict[str, list[dict[str, Any]]]:
252 """``{album_id: [{position, name, artists, duration}]}`` for the
253 releases whose tracklist is kept."""
255 found: dict[str, list[dict[str, Any]]] = {}
256 with self._connect() as conn:
257 for r in conn.execute(
258 'SELECT * FROM artist_catalogue_tracks WHERE playlist_id = ? '
259 'ORDER BY album_id, position',
260 (playlist_id,),
261 ):
262 found.setdefault(r['album_id'], []).append({
263 'position': r['position'],
264 'name': r['name'],
265 'artists': json.loads(r['artists'] or '[]'),
266 'duration': r['duration'],
267 })
268 return found
270 def set_total(self, playlist_id: int, album_id: str, total: int) -> None:
271 with self._connect() as conn:
272 conn.execute(
273 'UPDATE artist_catalogue SET track_total = ? '
274 'WHERE playlist_id = ? AND album_id = ?',
275 (int(total), playlist_id, album_id),
276 )
278 def set_not_own(self, playlist_id: int, album_id: str) -> None:
279 with self._connect() as conn:
280 conn.execute(
281 'UPDATE artist_catalogue SET not_own = 1 '
282 'WHERE playlist_id = ? AND album_id = ?',
283 (playlist_id, album_id),
284 )
286 def choices(self, playlist_id: int) -> dict[str, str]:
287 """``{album_id: 'on' | 'off'}`` for releases the user switched."""
289 with self._connect() as conn:
290 return {
291 r['album_id']: r['choice']
292 for r in conn.execute(
293 'SELECT album_id, choice FROM artist_catalogue '
294 "WHERE playlist_id = ? AND choice != ''",
295 (playlist_id,),
296 )
297 }
299 def set_choice(
300 self, playlist_id: int, album_ids: Iterable[str], choice: str
301 ) -> int:
302 """Switch releases ``on``, ``off`` or back to the watch's rules
303 (``''``). Returns how many of *album_ids* the watch lists."""
305 if choice not in CHOICES:
306 raise ValueError(f'unknown choice {choice!r}')
307 changed = 0
308 with self._connect() as conn:
309 for album_id in album_ids:
310 changed += conn.execute(
311 'UPDATE artist_catalogue SET choice = ? '
312 'WHERE playlist_id = ? AND album_id = ?',
313 (choice, playlist_id, str(album_id)),
314 ).rowcount
315 return changed
317 def rows(self, playlist_id: int) -> list[dict[str, Any]]:
318 with self._connect() as conn:
319 return [
320 dict(r)
321 for r in conn.execute(
322 'SELECT * FROM artist_catalogue WHERE playlist_id = ? '
323 'ORDER BY position, album_id',
324 (playlist_id,),
325 )
326 ]
328 def seen(self, playlist_id: int) -> dict[str, bool]:
329 """``{album_id: skipped}`` for the releases a sweep processed
330 (``seen_albums``): ``True`` for one recorded without downloading
331 it."""
333 with self._connect() as conn:
334 return {
335 r['album_id']: bool(r['skipped'])
336 for r in conn.execute(
337 'SELECT album_id, skipped FROM seen_albums '
338 'WHERE playlist_id = ?',
339 (playlist_id,),
340 )
341 }
343 def forget_skipped(
344 self, playlist_id: int, album_ids: Iterable[str]
345 ) -> int:
346 """Drop *album_ids* from the releases a watch skipped, so the next
347 check downloads them (the user switched them on)."""
349 with self._connect() as conn:
350 return sum(
351 conn.execute(
352 'DELETE FROM seen_albums WHERE playlist_id = ? '
353 'AND album_id = ? AND skipped = 1',
354 (playlist_id, str(album_id)),
355 ).rowcount
356 for album_id in album_ids
357 )
359 def clear(self, playlist_id: int) -> None:
360 """Forget the listing - the watch now follows another artist."""
362 with self._connect() as conn:
363 for table in ('artist_catalogue', 'artist_catalogue_tracks'):
364 conn.execute(
365 f'DELETE FROM {table} WHERE playlist_id = ?', # noqa: S608
366 (playlist_id,),
367 )
370def release_key(name: str) -> str:
371 """A release's name as two catalogues are matched by: without a
372 ``(feat. …)`` / ``(with …)`` or remaster tag, which one catalogue
373 adds and another leaves out (*Laugh Now Cry Later (feat. Lil Durk)*
374 on YouTube Music is Deezer's *Laugh Now Cry Later*)."""
376 return song_key('', str(name or '')).split('|', 1)[1] or file_name_key(
377 str(name or '')
378 )
381# ── Announced releases ──────────────────────────────────────────────────
383_DATE = re.compile(r'\d{4}(-\d{2}(-\d{2})?)?')
386def release_day(album: dict[str, Any]) -> str:
387 """A release's date as a catalogue gave it (``YYYY-MM-DD``, or only
388 ``YYYY-MM`` / ``YYYY`` for some MusicBrainz groups), ``''`` without
389 one."""
391 text = str(album.get('release_date') or '').strip()[:10]
392 return text if _DATE.fullmatch(text) else ''
395def upcoming(album: dict[str, Any], today: Optional[date] = None) -> bool:
396 """Whether *album* is announced but not out yet: its date is after
397 *today* (a month or a year alone: after this month or year). Out on
398 its release day itself."""
400 day = release_day(album)
401 if not day:
402 return False
403 now = (today or date.today()).isoformat()
404 return day > now[: len(day)]
407# ── What the watch downloads ────────────────────────────────────────────
410def release_type_of(album: dict[str, Any]) -> str:
411 """``album``, ``single`` or ``ep``, like ``monitor.release_type_of``:
412 a catalogue's label; anything else counts as an album."""
414 label = str(album.get('release_type') or '').strip().lower()
415 return label if label in {'single', 'ep'} else 'album'
418def watch_kinds(playlist: Any, settings: dict[str, Any]) -> set[str]:
419 """The MusicBrainz secondary types a watch lets in: its own choice,
420 else the ``artist_watch_musicbrainz_kinds`` setting."""
422 own = getattr(playlist, 'musicbrainz_kinds', None)
423 if own is not None:
424 return musicbrainz.kinds_from_setting([k for k in own.split(',') if k])
425 return musicbrainz.kinds_from_setting(
426 settings.get('artist_watch_musicbrainz_kinds')
427 )
430def follows_watch(
431 album: dict[str, Any], types: set[str], kinds: set[str]
432) -> bool:
433 """Whether the watch's own rules download *album*: one of its release
434 *types*, and only secondary types it lets in (YouTube Music and
435 Deezer releases have none)."""
437 return release_type_of(album) in types and all(
438 kind in kinds for kind in album.get('kinds') or []
439 )
442def wanted(
443 album: dict[str, Any],
444 types: set[str],
445 kinds: set[str],
446 choices: dict[str, str],
447) -> bool:
448 """Whether a sweep downloads *album*: the user's choice, else the
449 watch's rules."""
451 choice = choices.get(str(album.get('album_id') or ''), '')
452 if choice:
453 return choice == 'on'
454 return follows_watch(album, types, kinds)
457# ── What the library has ────────────────────────────────────────────────
460def library_have(
461 tracks: dict[str, list[dict[str, Any]]],
462 songs: Any,
463 recorded: dict[str, int],
464) -> dict[str, int]:
465 """How many tracks of each release the library holds: of its kept
466 tracklist, those *songs* (a :class:`~downtify.library_songs.
467 LibrarySongs`) finds; for a release with none kept, the tracks the
468 watch recorded (*recorded*)."""
470 have = dict(recorded)
471 if songs is None:
472 return have
473 for album_id, rows in tracks.items():
474 have[album_id] = sum(1 for row in rows if songs.find(row))
475 return have
478def track_list(
479 rows: list[dict[str, Any]],
480 songs: Any,
481 recorded: dict[int, str],
482) -> list[dict[str, Any]]:
483 """A release's tracklist with the library file holding each track
484 (``file``, ``None`` when missing): the one *songs* finds, else the one
485 the watch recorded at that position."""
487 return [
488 {
489 **row,
490 'file': (songs.find(row) if songs is not None else None)
491 or recorded.get(row['position']),
492 }
493 for row in rows
494 ]
497# ── The view ────────────────────────────────────────────────────────────
500def _state( # noqa: PLR0911 # one answer per state
501 row: dict[str, Any],
502 have: int,
503 skipped: Optional[bool],
504 follows: bool,
505 today: Optional[date] = None,
506) -> str:
507 total = int(row['track_total'] or 0)
508 if row['choice'] == 'off':
509 return 'off'
510 if row['not_own']:
511 return 'not_own'
512 if have and total and have >= total:
513 return 'complete'
514 if have:
515 return 'partly'
516 if skipped is False and total == 0:
517 # Processed with nothing to download (an empty release).
518 return 'complete'
519 coming = not skipped and upcoming(row, today)
520 if row['choice'] == 'on':
521 return 'upcoming' if coming else 'missing'
522 if not follows:
523 # A type the watch doesn't download: the lasting reason, even for
524 # one that also came out before a "new releases only" watch.
525 return 'not_watched'
526 if coming:
527 return 'upcoming'
528 return 'skipped' if skipped else 'missing'
531def _downloads(
532 row: dict[str, Any], skipped: Optional[bool], follows: bool
533) -> bool:
534 """Whether the watch downloads a release (its switch): the user's
535 choice, else its type and kind - unless it came out before a "new
536 releases only" watch. Never a guest spot or a bootleg."""
538 if row['not_own']:
539 return False
540 if row['choice']:
541 return row['choice'] == 'on'
542 return follows and not skipped
545def section_of(release_type: str, kinds: list[str]) -> str:
546 """Where a release is listed: ``album``, ``mixtape``, ``ep``,
547 ``single``, or ``other`` (live albums, compilations, remixes...)."""
549 if 'mixtape' in kinds:
550 return 'mixtape'
551 if any(kind != 'soundtrack' for kind in kinds):
552 return 'other'
553 return (
554 release_type if release_type in {'album', 'ep', 'single'} else 'album'
555 )
558def catalogue_view(
559 rows: list[dict[str, Any]],
560 *,
561 have: dict[str, int],
562 seen: dict[str, bool],
563 types: set[str],
564 kinds: set[str],
565 today: Optional[date] = None,
566) -> list[dict[str, Any]]:
567 """The watch's releases as the page lists them: one per release (a
568 release two catalogues list is merged: same name, same type), each
569 ``{id, album_ids, name, release_type, kinds, section, year, sources,
570 cover_url, url, release_date, state, have, total, choice, wanted}`` -
571 ``wanted``
572 being whether the watch downloads it (its switch), ``state`` what
573 the library has of it.
575 *have* is how many tracks of each release the library holds, *seen*
576 whether a processed release was skipped (``True``) or downloaded
577 (``False``); *types* and *kinds* are the watch's rules."""
579 merged: dict[tuple[str, str], dict[str, Any]] = {}
580 for row in rows:
581 album = {
582 'album_id': row['album_id'],
583 'release_type': row['release_type'],
584 'kinds': [k for k in row['kinds'].split(',') if k],
585 }
586 release_type = release_type_of(album)
587 kind_names = [_KIND_NAMES.get(k, k) for k in album['kinds']]
588 follows = follows_watch(album, types, kinds)
589 state = _state(
590 row,
591 have.get(row['album_id'], 0),
592 seen.get(row['album_id']),
593 follows,
594 today,
595 )
596 downloads = _downloads(row, seen.get(row['album_id']), follows)
597 key = (release_key(row['name']) or row['album_id'], release_type)
598 entry = merged.get(key)
599 if entry is None:
600 merged[key] = {
601 'id': row['album_id'],
602 'album_ids': [row['album_id']],
603 'name': row['name'],
604 'release_type': release_type,
605 'kinds': kind_names,
606 'section': section_of(release_type, kind_names),
607 'year': row['year'],
608 'release_date': release_day(row),
609 'sources': [row['source']] if row['source'] else [],
610 'cover_url': row['cover_url'],
611 'url': row['url'],
612 'state': state,
613 'have': have.get(row['album_id'], 0),
614 'total': int(row['track_total'] or 0),
615 'choice': row['choice'],
616 'wanted': downloads,
617 }
618 continue
619 entry['album_ids'].append(row['album_id'])
620 if row['source'] and row['source'] not in entry['sources']:
621 entry['sources'].append(row['source'])
622 if _STATE_ORDER.index(state) < _STATE_ORDER.index(entry['state']):
623 entry['state'] = state
624 entry['have'] = max(entry['have'], have.get(row['album_id'], 0))
625 entry['total'] = max(entry['total'], int(row['track_total'] or 0))
626 entry['year'] = entry['year'] or row['year']
627 entry['release_date'] = max(entry['release_date'], release_day(row))
628 entry['cover_url'] = entry['cover_url'] or row['cover_url']
629 entry['choice'] = entry['choice'] or row['choice']
630 entry['wanted'] = entry['wanted'] or downloads
631 return list(merged.values())
634def missing(releases: list[dict[str, Any]]) -> dict[str, int]:
635 """What "Download missing" would fetch: of the releases the watch
636 downloads (``wanted``), how many tracks are missing (``tracks``, in
637 ``releases`` releases), plus the releases whose tracklist isn't read
638 yet (``unread``: their count is unknown)."""
640 found = {'tracks': 0, 'releases': 0, 'unread': 0}
641 for release in releases:
642 if not release.get('wanted') or release['state'] in {
643 'complete',
644 'upcoming',
645 }:
646 continue
647 if not release['total']:
648 found['unread'] += 1
649 continue
650 gap = release['total'] - release['have']
651 if gap > 0:
652 found['tracks'] += gap
653 found['releases'] += 1
654 return found
657def upcoming_releases(
658 releases: list[dict[str, Any]],
659) -> list[dict[str, Any]]:
660 """The announced releases the watch will download on their release
661 day, soonest first."""
663 return sorted(
664 (
665 release
666 for release in releases
667 if release.get('wanted') and release['state'] == 'upcoming'
668 ),
669 key=lambda release: release.get('release_date') or '',
670 )
673def counts(releases: list[dict[str, Any]]) -> dict[str, int]:
674 """How many releases are in each state."""
676 found = dict.fromkeys(_STATE_ORDER, 0)
677 for release in releases:
678 found[release['state']] = found.get(release['state'], 0) + 1
679 return found
682def wanted_releases(releases: list[dict[str, Any]]) -> list[dict[str, Any]]:
683 """The releases "Download missing" would still fetch from: the ones
684 the watch downloads (``wanted``) that the library has only partly
685 (``partly``) or not at all (``missing``), in the order given."""
687 return [
688 release
689 for release in releases
690 if release.get('wanted') and release['state'] in {'partly', 'missing'}
691 ]