Coverage for downtify/discover.py: 95%
507 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Discover: artists the user doesn't have yet, suggested from the ones
2they do.
4Where the suggestions come from
5-------------------------------
7Each *seed* is an artist already in the library. How much a seed counts
8is its :func:`seed_weights`: how many of its tracks are in the library,
9how many of those are liked, and how often (and how recently) the user
10actually listened to it - the player reports a listen once a track has
11played long enough (see :meth:`DiscoverStore.record_listen`). The
12heaviest seeds are then looked up on Deezer, whose public, keyless
13``/artist/{id}/related`` answers "fans of this artist also like" (see
14:func:`downtify.deezer.related_artists`), and every related artist gets
15points from each seed that suggests it (:func:`rank_candidates`). An
16artist several of the user's favourites point to beats one only a single
17seed mentions.
19Last.fm, which other tools use for this, needs an API key; Spotify's own
20"Fans also like" needs a Spotify id per seed, and the embed pages this
21project relies on (no Web API credentials) only hand that out per track.
22Deezer needs neither.
24What is never suggested: an artist already in the library, and an artist
25on the user's block list (:meth:`DiscoverStore.block`) - "Not interested"
26on a suggestion, or a name typed in by hand.
28Albums and playlists
29--------------------
31:func:`collections` builds on the artists above, with the web player's
32own search (anonymous, like everything else here - see
33:func:`downtify.spotify.search`), one search per artist name:
35* **Albums for you** - the top album of each of the best suggested
36 artists.
37* **More from your artists** - the most popular albums of the user's own
38 heaviest artists that aren't in the library yet.
39* **Playlists** - Spotify's own "<artist> Radio" mixes for the user's
40 heaviest artists (similar music around an artist they already love),
41 then its "This Is <artist>" for the best suggested ones.
43Every result is a Spotify album or playlist, so opening one lands on the
44same page a pasted link does, with the tracks' preview clips. Albums
45already in the library and playlists already downloaded are left out.
47What is stored
48--------------
50Four tables in ``downtify_library.db``: the listen counts, the block
51list, a per-seed cache of Deezer's answer (the seed's Deezer id and its
52related artists) and a per-artist cache of the Spotify search, both
53trusted for :data:`RELATED_TTL` so opening the page again doesn't cost a
54round of requests - Deezer allows about 50 per 5 seconds, and a cold page
55is two per seed.
56"""
58from __future__ import annotations
60import json
61import math
62from collections.abc import Callable, Iterable
63from concurrent.futures import ThreadPoolExecutor
64from datetime import datetime, timedelta, timezone
65from pathlib import Path
66from typing import Any, Optional
68from loguru import logger
70from . import deezer, spotify
71from .db.config import Dialect
72from .db.core import Database, as_database
73from .file_naming import file_name_key, title_key
75#: How long a seed's related artists are trusted before asking Deezer again.
76RELATED_TTL = timedelta(days=7)
77#: How many of the heaviest library artists are looked up.
78MAX_SEEDS = 15
79#: How many related artists are asked for per seed.
80RELATED_PER_SEED = 20
81#: How many suggestions are returned, at most.
82RESULT_LIMIT = 48
83#: Listening to an artist counts half as much after this many days.
84LISTEN_HALF_LIFE_DAYS = 45
85#: Deezer requests in flight at once (it rate-limits at ~50 per 5 s).
86_WORKERS = 4
87#: For Deezer lookups that go through its throttle (discographies,
88#: playlists, album matches): more at once is safe, the throttle keeps the
89#: overall rate under Deezer's limit.
90_THROTTLED_WORKERS = 8
91#: How many of the best suggested artists get an album / playlist.
92COLLECTION_SUGGESTED = 12
93#: How many of the heaviest library artists get albums / a radio mix.
94COLLECTION_SEEDS = 6
95#: Albums not in the library, at most, per library artist.
96MORE_FROM_PER_SEED = 2
97#: Spotify's own account, the owner of its editorial playlists.
98SPOTIFY_OWNER = 'spotify'
99#: How long a Spotify album's Deezer match is trusted: an album found on
100#: Deezer stays the same album, one missing may be added any week.
101MATCH_TTL_FOUND = timedelta(days=30)
102MATCH_TTL_MISSING = RELATED_TTL
103#: The per-artist Deezer cache's parts, by name -> column prefix (a fixed
104#: list: the name ends up in the SQL).
105_DEEZER_PARTS = {'albums': 'albums', 'playlist': 'playlist'}
107#: A seed's related artists, best match first (see ``deezer.related_artists``).
108Related = list[dict[str, Any]]
111def _now() -> datetime:
112 return datetime.now(timezone.utc)
115def artist_key(name: str) -> str:
116 """The key two names of the same artist share: case, spaces and the
117 characters a file name can't hold are ignored, so ``AC/DC`` in a tag
118 and ``ACDC`` on Deezer are the same artist (see
119 :func:`downtify.file_naming.file_name_key`)."""
121 return file_name_key(str(name or ''))
124def _parse_time(value: Any) -> Optional[datetime]:
125 try:
126 parsed = datetime.fromisoformat(str(value))
127 except (TypeError, ValueError):
128 return None
129 if parsed.tzinfo is None:
130 parsed = parsed.replace(tzinfo=timezone.utc)
131 return parsed
134# ── Ranking (pure) ───────────────────────────────────────────────────────
137def seed_weights(
138 library: Iterable[dict[str, Any]],
139 listens: Iterable[dict[str, Any]],
140 now: Optional[datetime] = None,
141) -> dict[str, dict[str, Any]]:
142 """How much each library artist says about the user's taste.
144 *library* is ``{name, tracks, liked}`` per artist in the library;
145 *listens* the rows of :meth:`DiscoverStore.listens`. The weight grows
146 with each signal but flattens out (``log1p``), so a 400-track
147 discography doesn't drown everything else, and listening counts most,
148 liking next, owning least. A listen fades with the time since the
149 artist was last played (:data:`LISTEN_HALF_LIFE_DAYS`).
151 Only artists in *library* can be seeds - a listen to an artist whose
152 tracks have since been deleted doesn't count. Returns ``{key: {name,
153 weight}}``, keyed by :func:`artist_key`; artists with no weight at all
154 are left out.
155 """
157 now = now or _now()
158 plays: dict[str, float] = {}
159 for row in listens:
160 key = artist_key(row.get('name'))
161 if not key:
162 continue
163 last = _parse_time(row.get('last_played'))
164 age_days = (
165 max(0.0, (now - last).total_seconds() / 86400) if last else 0
166 )
167 fade = 0.5 ** (age_days / LISTEN_HALF_LIFE_DAYS)
168 plays[key] = (
169 plays.get(key, 0.0) + max(0, int(row.get('plays') or 0)) * fade
170 )
172 weights: dict[str, dict[str, Any]] = {}
173 for artist in library:
174 name = str(artist.get('name') or '').strip()
175 key = artist_key(name)
176 if not key:
177 continue
178 tracks = max(0, int(artist.get('tracks') or 0))
179 liked = max(0, int(artist.get('liked') or 0))
180 weight = (
181 math.log1p(tracks)
182 + 1.5 * math.log1p(liked)
183 + 2.0 * math.log1p(plays.get(key, 0.0))
184 )
185 if weight <= 0:
186 continue
187 entry = weights.setdefault(key, {'name': name, 'weight': 0.0})
188 entry['weight'] += weight
189 return weights
192def pick_seeds(
193 weights: dict[str, dict[str, Any]], limit: int = MAX_SEEDS
194) -> list[tuple[str, str, float]]:
195 """The *limit* heaviest seeds as ``(key, name, weight)``, heaviest
196 first (ties by name, so the pick is stable)."""
198 ranked = sorted(
199 weights.items(),
200 key=lambda item: (-item[1]['weight'], item[1]['name'].casefold()),
201 )
202 return [(key, v['name'], v['weight']) for key, v in ranked[:limit]]
205def rank_candidates(
206 seeds: list[tuple[str, str, float]],
207 related: dict[str, Related],
208 exclude: set[str],
209 limit: int = RESULT_LIMIT,
210) -> list[dict[str, Any]]:
211 """Every related artist, scored by the seeds that suggest it.
213 A seed gives a related artist its own weight, less the further down
214 Deezer's list it sits; an artist several seeds suggest adds up all of
215 theirs. Keys in *exclude* (the library, the block list) are never
216 suggested, and neither is a seed itself.
218 Each result is ``{name, deezer_id, picture_url, fans, score,
219 because}`` - ``because`` the names of up to three seeds that suggested
220 it, the biggest contributor first. Best first; ties go to the artist
221 with more fans.
222 """
224 seed_keys = {key for key, _name, _weight in seeds}
225 found: dict[str, dict[str, Any]] = {}
226 for seed_key, seed_name, weight in seeds:
227 for position, artist in enumerate(related.get(seed_key) or []):
228 key = artist_key(artist.get('name'))
229 if not key or key in exclude or key in seed_keys:
230 continue
231 points = weight / (1 + 0.15 * position)
232 entry = found.get(key)
233 if entry is None:
234 entry = found[key] = {
235 'name': artist['name'],
236 'deezer_id': str(artist.get('deezer_id') or ''),
237 'picture_url': str(artist.get('picture_url') or ''),
238 'fans': int(artist.get('fans') or 0),
239 'score': 0.0,
240 '_by': {},
241 }
242 entry['score'] += points
243 entry['_by'][seed_name] = entry['_by'].get(seed_name, 0) + points
245 ranked = sorted(
246 found.values(),
247 key=lambda e: (-e['score'], -e['fans'], e['name'].casefold()),
248 )[:limit]
249 for entry in ranked:
250 by = entry.pop('_by')
251 entry['because'] = sorted(by, key=lambda n: -by[n])[:3]
252 entry['score'] = round(entry['score'], 3)
253 return ranked
256# ── Storage ──────────────────────────────────────────────────────────────
259class DiscoverStore:
260 """Listen counts, the block list, and the related-artists cache."""
262 def __init__(self, db_path: str | Path | Database) -> None:
263 self.db = as_database(db_path, 'library')
264 if isinstance(db_path, Database):
265 self._path = str(self.db.path) if self.db.path else None
266 else:
267 self._path = str(db_path)
268 self._init_db()
270 def _connect(self):
271 return self.db.connect()
273 def _init_db(self) -> None:
274 # PostgreSQL gets its schema from the migrations.
275 if self.db.dialect is not Dialect.SQLITE:
276 return
277 with self._connect() as conn:
278 conn.execute("""
279 CREATE TABLE IF NOT EXISTS discover_listens (
280 artist_key TEXT NOT NULL,
281 user_id INTEGER NOT NULL DEFAULT 0,
282 name TEXT NOT NULL,
283 plays INTEGER NOT NULL DEFAULT 0,
284 last_played TEXT NOT NULL,
285 PRIMARY KEY (artist_key, user_id)
286 )
287 """)
288 conn.execute("""
289 CREATE TABLE IF NOT EXISTS discover_plays (
290 play_id TEXT PRIMARY KEY,
291 recorded_at TEXT NOT NULL
292 )
293 """)
294 conn.execute("""
295 CREATE TABLE IF NOT EXISTS discover_blocked (
296 artist_key TEXT NOT NULL,
297 user_id INTEGER NOT NULL DEFAULT 0,
298 name TEXT NOT NULL,
299 blocked_at TEXT NOT NULL,
300 PRIMARY KEY (artist_key, user_id)
301 )
302 """)
303 self._migrate(conn)
304 conn.execute("""
305 CREATE TABLE IF NOT EXISTS discover_search (
306 artist_key TEXT PRIMARY KEY,
307 results_json TEXT NOT NULL DEFAULT '{}',
308 fetched_at TEXT NOT NULL
309 )
310 """)
311 conn.execute("""
312 CREATE TABLE IF NOT EXISTS discover_related (
313 artist_key TEXT PRIMARY KEY,
314 deezer_id TEXT NOT NULL DEFAULT '',
315 related_json TEXT NOT NULL DEFAULT '[]',
316 fetched_at TEXT NOT NULL
317 )
318 """)
319 # An artist's Deezer discography and editorial "100%"
320 # playlist, each fetched (and trusted) on its own.
321 conn.execute("""
322 CREATE TABLE IF NOT EXISTS discover_deezer (
323 artist_key TEXT PRIMARY KEY,
324 albums_json TEXT,
325 albums_at TEXT,
326 playlist_json TEXT,
327 playlist_at TEXT
328 )
329 """)
330 # A Spotify album's match on Deezer - '' ids when Deezer has
331 # no such album.
332 conn.execute("""
333 CREATE TABLE IF NOT EXISTS discover_album_map (
334 spotify_id TEXT PRIMARY KEY,
335 deezer_album_id TEXT NOT NULL DEFAULT '',
336 deezer_artist_id TEXT NOT NULL DEFAULT '',
337 cover_url TEXT NOT NULL DEFAULT '',
338 fetched_at TEXT NOT NULL
339 )
340 """)
342 @staticmethod
343 def _migrate(conn: Any) -> None:
344 """One-time upgrade of single-user tables to per-user ones."""
346 columns = {
347 row[1]
348 for row in conn.execute('PRAGMA table_info(discover_listens)')
349 }
350 if 'user_id' not in columns:
351 conn.execute("""
352 CREATE TABLE discover_listens_new (
353 artist_key TEXT NOT NULL,
354 user_id INTEGER NOT NULL DEFAULT 0,
355 name TEXT NOT NULL,
356 plays INTEGER NOT NULL DEFAULT 0,
357 last_played TEXT NOT NULL,
358 PRIMARY KEY (artist_key, user_id)
359 )
360 """)
361 conn.execute("""
362 INSERT INTO discover_listens_new
363 (artist_key, user_id, name, plays, last_played)
364 SELECT artist_key, 0, name, plays, last_played
365 FROM discover_listens
366 """)
367 conn.execute('DROP TABLE discover_listens')
368 conn.execute(
369 'ALTER TABLE discover_listens_new RENAME TO discover_listens'
370 )
371 columns = {
372 row[1]
373 for row in conn.execute('PRAGMA table_info(discover_blocked)')
374 }
375 if 'user_id' not in columns:
376 conn.execute("""
377 CREATE TABLE discover_blocked_new (
378 artist_key TEXT NOT NULL,
379 user_id INTEGER NOT NULL DEFAULT 0,
380 name TEXT NOT NULL,
381 blocked_at TEXT NOT NULL,
382 PRIMARY KEY (artist_key, user_id)
383 )
384 """)
385 conn.execute("""
386 INSERT INTO discover_blocked_new
387 (artist_key, user_id, name, blocked_at)
388 SELECT artist_key, 0, name, blocked_at
389 FROM discover_blocked
390 """)
391 conn.execute('DROP TABLE discover_blocked')
392 conn.execute(
393 'ALTER TABLE discover_blocked_new RENAME TO discover_blocked'
394 )
396 # Listens
398 def record_listen(
399 self,
400 name: str,
401 when: Optional[datetime] = None,
402 play_id: str = '',
403 user_id: int = 0,
404 ) -> Optional[dict[str, Any]]:
405 """Count one listen to *name* for *user_id*. ``None`` for a name
406 with no key.
408 *when* is when it was played - an app that was offline reports
409 its plays later; a late report never moves ``last_played`` back.
410 A *play_id* is counted once: the same play reported twice (a
411 retry after a lost answer) doesn't add up.
412 """
414 name = str(name or '').strip()
415 key = artist_key(name)
416 if not key:
417 return None
418 stamp = (when or _now()).isoformat()
419 uid = int(user_id or 0)
420 with self._connect() as conn:
421 fresh = True
422 if play_id:
423 cur = conn.execute(
424 'INSERT INTO discover_plays (play_id, recorded_at) '
425 'VALUES (?, ?) ON CONFLICT(play_id) DO NOTHING',
426 (play_id[:64], _now().isoformat()),
427 )
428 fresh = cur.rowcount > 0
429 if fresh:
430 conn.execute(
431 """INSERT INTO discover_listens
432 (artist_key, user_id, name, plays, last_played)
433 VALUES (?, ?, ?, 1, ?)
434 ON CONFLICT(artist_key, user_id) DO UPDATE SET
435 name = excluded.name,
436 plays = discover_listens.plays + 1,
437 last_played = CASE
438 WHEN excluded.last_played > discover_listens.last_played
439 THEN excluded.last_played
440 ELSE discover_listens.last_played
441 END""",
442 (key, uid, name, stamp),
443 )
444 row = conn.execute(
445 'SELECT name, plays, last_played FROM discover_listens '
446 'WHERE artist_key = ? AND user_id = ?',
447 (key, uid),
448 ).fetchone()
449 return dict(row)
451 @property
452 def _name_sort(self) -> str:
453 """Text sort matching SQLite's byte order on both backends."""
454 if self.db.dialect is Dialect.SQLITE:
455 return 'name'
456 return 'name COLLATE "C"'
458 def listens(self, user_id: Optional[int] = None) -> list[dict[str, Any]]:
459 order = f'ORDER BY plays DESC, {self._name_sort}'
460 with self._connect() as conn:
461 if user_id is None:
462 rows = conn.execute(
463 'SELECT name, plays, last_played FROM discover_listens '
464 + order
465 ).fetchall()
466 else:
467 rows = conn.execute(
468 'SELECT name, plays, last_played FROM discover_listens '
469 'WHERE user_id = ? ' + order,
470 (int(user_id),),
471 ).fetchall()
472 return [dict(row) for row in rows]
474 def clear_listens(self, user_id: Optional[int] = None) -> int:
475 with self._connect() as conn:
476 if user_id is None:
477 return conn.execute('DELETE FROM discover_listens').rowcount
478 cur = conn.execute(
479 'DELETE FROM discover_listens WHERE user_id = ?',
480 (int(user_id),),
481 )
482 return cur.rowcount
484 def reassign_legacy(self, user_id: int) -> int:
485 """Give pre-feature listens and blocks to *user_id* (once)."""
487 if not user_id:
488 return 0
489 with self._connect() as conn:
490 listens = conn.execute(
491 'UPDATE discover_listens SET user_id = ? WHERE user_id = 0',
492 (int(user_id),),
493 ).rowcount
494 blocked = conn.execute(
495 'UPDATE discover_blocked SET user_id = ? WHERE user_id = 0',
496 (int(user_id),),
497 ).rowcount
498 return listens + blocked
500 def remove_user(self, user_id: int) -> None:
501 """Forget a deleted account's listens and blocks."""
503 with self._connect() as conn:
504 conn.execute(
505 'DELETE FROM discover_listens WHERE user_id = ?',
506 (int(user_id),),
507 )
508 conn.execute(
509 'DELETE FROM discover_blocked WHERE user_id = ?',
510 (int(user_id),),
511 )
513 # Block list
515 def block(self, name: str, user_id: int = 0) -> Optional[dict[str, Any]]:
516 """Never suggest *name* again. ``None`` for a name with no key."""
518 name = str(name or '').strip()
519 key = artist_key(name)
520 if not key:
521 return None
522 stamp = _now().isoformat()
523 uid = int(user_id or 0)
524 with self._connect() as conn:
525 conn.execute(
526 """INSERT INTO discover_blocked (artist_key, user_id, name,
527 blocked_at)
528 VALUES (?, ?, ?, ?)
529 ON CONFLICT(artist_key, user_id) DO NOTHING""",
530 (key, uid, name, stamp),
531 )
532 row = conn.execute(
533 'SELECT name, blocked_at FROM discover_blocked '
534 'WHERE artist_key = ? AND user_id = ?',
535 (key, uid),
536 ).fetchone()
537 return dict(row)
539 def unblock(self, name: str, user_id: Optional[int] = None) -> bool:
540 with self._connect() as conn:
541 if user_id is None:
542 cur = conn.execute(
543 'DELETE FROM discover_blocked WHERE artist_key = ?',
544 (artist_key(name),),
545 )
546 else:
547 cur = conn.execute(
548 'DELETE FROM discover_blocked WHERE artist_key = ? '
549 'AND user_id = ?',
550 (artist_key(name), int(user_id)),
551 )
552 return cur.rowcount > 0
554 def blocked(self, user_id: Optional[int] = None) -> list[dict[str, Any]]:
555 order = f'ORDER BY blocked_at DESC, {self._name_sort}'
556 with self._connect() as conn:
557 if user_id is None:
558 rows = conn.execute(
559 'SELECT name, blocked_at FROM discover_blocked ' + order
560 ).fetchall()
561 else:
562 rows = conn.execute(
563 'SELECT name, blocked_at FROM discover_blocked '
564 'WHERE user_id = ? ' + order,
565 (int(user_id),),
566 ).fetchall()
567 return [dict(row) for row in rows]
569 def blocked_keys(self, user_id: Optional[int] = None) -> set[str]:
570 with self._connect() as conn:
571 if user_id is None:
572 rows = conn.execute(
573 'SELECT artist_key FROM discover_blocked'
574 ).fetchall()
575 else:
576 rows = conn.execute(
577 'SELECT artist_key FROM discover_blocked WHERE user_id '
578 '= ?',
579 (int(user_id),),
580 ).fetchall()
581 return {row['artist_key'] for row in rows}
583 # Related-artists cache
585 def cached_related(self, key: str) -> Optional[dict[str, Any]]:
586 """``{deezer_id, related, fetched_at}`` for a seed, or ``None``."""
588 with self._connect() as conn:
589 row = conn.execute(
590 'SELECT deezer_id, related_json, fetched_at '
591 'FROM discover_related WHERE artist_key = ?',
592 (key,),
593 ).fetchone()
594 if row is None:
595 return None
596 try:
597 related = json.loads(row['related_json'])
598 except ValueError:
599 related = []
600 return {
601 'deezer_id': row['deezer_id'],
602 'related': related if isinstance(related, list) else [],
603 'fetched_at': _parse_time(row['fetched_at']),
604 }
606 def save_related(
607 self,
608 key: str,
609 deezer_id: str,
610 related: Related,
611 when: Optional[datetime] = None,
612 ) -> None:
613 with self._connect() as conn:
614 conn.execute(
615 """INSERT INTO discover_related
616 (artist_key, deezer_id, related_json, fetched_at)
617 VALUES (?, ?, ?, ?)
618 ON CONFLICT(artist_key) DO UPDATE SET
619 deezer_id = excluded.deezer_id,
620 related_json = excluded.related_json,
621 fetched_at = excluded.fetched_at""",
622 (
623 key,
624 deezer_id or '',
625 json.dumps(related),
626 (when or _now()).isoformat(),
627 ),
628 )
630 # Spotify search cache
632 def cached_search(self, key: str) -> Optional[dict[str, Any]]:
633 """``{results, fetched_at}`` for an artist's search, or ``None``."""
635 with self._connect() as conn:
636 row = conn.execute(
637 'SELECT results_json, fetched_at FROM discover_search '
638 'WHERE artist_key = ?',
639 (key,),
640 ).fetchone()
641 if row is None:
642 return None
643 try:
644 results = json.loads(row['results_json'])
645 except ValueError:
646 results = {}
647 return {
648 'results': results if isinstance(results, dict) else {},
649 'fetched_at': _parse_time(row['fetched_at']),
650 }
652 def save_search(
653 self,
654 key: str,
655 results: dict[str, Any],
656 when: Optional[datetime] = None,
657 ) -> None:
658 with self._connect() as conn:
659 conn.execute(
660 """INSERT INTO discover_search
661 (artist_key, results_json, fetched_at)
662 VALUES (?, ?, ?)
663 ON CONFLICT(artist_key) DO UPDATE SET
664 results_json = excluded.results_json,
665 fetched_at = excluded.fetched_at""",
666 (key, json.dumps(results), (when or _now()).isoformat()),
667 )
669 # Deezer discography / editorial playlist cache
671 def cached_deezer(self, key: str, part: str) -> Optional[dict[str, Any]]:
672 """``{value, fetched_at}`` for an artist's Deezer *part*
673 (``'albums'`` or ``'playlist'``), or ``None`` if never fetched."""
675 column = _DEEZER_PARTS[part]
676 with self._connect() as conn:
677 row = conn.execute(
678 f'SELECT {column}_json AS value, {column}_at AS fetched_at '
679 'FROM discover_deezer WHERE artist_key = ?',
680 (key,),
681 ).fetchone()
682 if row is None or row['fetched_at'] is None:
683 return None
684 try:
685 value = json.loads(row['value'] or 'null')
686 except ValueError:
687 value = None
688 return {'value': value, 'fetched_at': _parse_time(row['fetched_at'])}
690 def save_deezer(
691 self,
692 key: str,
693 part: str,
694 value: Any,
695 when: Optional[datetime] = None,
696 ) -> None:
697 column = _DEEZER_PARTS[part]
698 with self._connect() as conn:
699 conn.execute(
700 f"""INSERT INTO discover_deezer
701 (artist_key, {column}_json, {column}_at)
702 VALUES (?, ?, ?)
703 ON CONFLICT(artist_key) DO UPDATE SET
704 {column}_json = excluded.{column}_json,
705 {column}_at = excluded.{column}_at""",
706 (key, json.dumps(value), (when or _now()).isoformat()),
707 )
709 # Spotify -> Deezer album matches
711 def cached_album_match(self, spotify_id: str) -> Optional[dict[str, Any]]:
712 """``{deezer_album_id, deezer_artist_id, cover_url, fetched_at}``
713 for a Spotify album, or ``None`` if never matched."""
715 with self._connect() as conn:
716 row = conn.execute(
717 'SELECT deezer_album_id, deezer_artist_id, cover_url, '
718 'fetched_at FROM discover_album_map WHERE spotify_id = ?',
719 (spotify_id,),
720 ).fetchone()
721 if row is None:
722 return None
723 return {
724 'deezer_album_id': row['deezer_album_id'],
725 'deezer_artist_id': row['deezer_artist_id'],
726 'cover_url': row['cover_url'],
727 'fetched_at': _parse_time(row['fetched_at']),
728 }
730 def save_album_match(
731 self,
732 spotify_id: str,
733 match: Optional[dict[str, Any]],
734 when: Optional[datetime] = None,
735 ) -> None:
736 """Remember *spotify_id*'s Deezer album (*match*, a
737 :func:`downtify.deezer.search_albums_by` row), or - ``None`` -
738 that Deezer has none."""
740 match = match or {}
741 with self._connect() as conn:
742 conn.execute(
743 """INSERT INTO discover_album_map
744 (spotify_id, deezer_album_id, deezer_artist_id,
745 cover_url, fetched_at)
746 VALUES (?, ?, ?, ?, ?)
747 ON CONFLICT(spotify_id) DO UPDATE SET
748 deezer_album_id = excluded.deezer_album_id,
749 deezer_artist_id = excluded.deezer_artist_id,
750 cover_url = excluded.cover_url,
751 fetched_at = excluded.fetched_at""",
752 (
753 spotify_id,
754 str(match.get('album_id') or ''),
755 str(match.get('artist_id') or ''),
756 str(match.get('cover_url') or ''),
757 (when or _now()).isoformat(),
758 ),
759 )
762# ── Albums and playlists (pure) ──────────────────────────────────────────
765def album_key(artist: str, title: str) -> str:
766 """The key an album is known by across services: its artist's
767 :func:`artist_key` and its :func:`~downtify.file_naming.title_key`, so
768 ``Dummy (Deluxe Edition)`` on Spotify is the ``Dummy`` on disk."""
770 return f'{artist_key(artist)}|{title_key(title)}'
773def _album_item(
774 album: dict[str, Any], artist: str, reason: str, because: list[str]
775) -> dict[str, Any]:
776 return {
777 'name': album['name'],
778 'artist': artist,
779 'year': album.get('year') or '',
780 'cover_url': album.get('cover_url') or '',
781 'spotify_id': album['id'],
782 'url': f'https://open.spotify.com/album/{album["id"]}',
783 'reason': reason,
784 'because': because,
785 }
788def _playlist_item(
789 playlist: dict[str, Any], artist: str, reason: str
790) -> dict[str, Any]:
791 return {
792 'name': playlist['name'],
793 'owner': playlist.get('owner') or '',
794 'cover_url': playlist.get('cover_url') or '',
795 'spotify_id': playlist['id'],
796 'url': f'https://open.spotify.com/playlist/{playlist["id"]}',
797 'reason': reason,
798 'artist': artist,
799 }
802def _albums_by(results: dict[str, Any], name: str) -> list[dict[str, Any]]:
803 """The albums in an artist's search *results* that are theirs (first
804 credited artist), full albums before singles/EPs, otherwise in
805 Spotify's (popularity) order."""
807 key = artist_key(name)
808 own = [
809 album
810 for album in results.get('albums') or []
811 if album.get('artists') and artist_key(album['artists'][0]) == key
812 ]
813 return sorted(own, key=lambda a: a.get('type') != 'ALBUM')
816def _editorial_playlist(
817 results: dict[str, Any], name: str, *, radio: bool
818) -> Optional[dict[str, Any]]:
819 """Spotify's own "<name> Radio" (*radio*) or "This Is <name>" playlist
820 in an artist's search *results*, or ``None``."""
822 key = artist_key(name)
823 for playlist in results.get('playlists') or []:
824 if artist_key(playlist.get('owner')) != SPOTIFY_OWNER:
825 continue
826 title = artist_key(playlist.get('name'))
827 if key not in title:
828 continue
829 # What's left once the artist's own name is out: "Radiohead" must
830 # not make "This Is Radiohead" a radio mix.
831 rest = title.replace(key, ' ')
832 if radio and 'radio' in rest.split():
833 return playlist
834 if not radio and 'this is' in rest:
835 return playlist
836 return None
839def pick_collections(
840 suggested: list[dict[str, Any]],
841 seeds: list[str],
842 searches: dict[str, dict[str, Any]],
843 owned_albums: set[str],
844 owned_playlists: set[str],
845) -> dict[str, Any]:
846 """Albums and playlists from each artist's Spotify *searches* (keyed
847 by :func:`artist_key`).
849 *suggested* are :func:`rank_candidates` results (best first), *seeds*
850 the user's heaviest library artists. Returns ``{albums, more_albums,
851 playlists, artist_urls}``:
853 * ``albums``: each suggested artist's top album not in
854 *owned_albums* (keys from :func:`album_key`), ``reason: 'similar'``.
855 * ``more_albums``: up to :data:`MORE_FROM_PER_SEED` albums per seed
856 not in *owned_albums*, ``reason: 'more_from'``.
857 * ``playlists``: the seeds' "<artist> Radio", then the suggested
858 artists' "This Is <artist>" (``reason: 'radio'`` / ``'this_is'``),
859 leaving out ids in *owned_playlists*.
860 * ``artist_urls``: ``{name: Spotify artist URL}`` for every suggested
861 artist the search found by exact name.
862 """
864 albums: list[dict[str, Any]] = []
865 more: list[dict[str, Any]] = []
866 radios: list[dict[str, Any]] = []
867 this_is: list[dict[str, Any]] = []
868 artist_urls: dict[str, str] = {}
869 seen_albums: set[str] = set(owned_albums)
870 seen_playlists: set[str] = set(owned_playlists)
872 def take_album(album, artist, reason, because, into) -> bool:
873 key = album_key(artist, album['name'])
874 if key in seen_albums:
875 return False
876 seen_albums.add(key)
877 into.append(_album_item(album, artist, reason, because))
878 return True
880 def take_playlist(playlist, artist, reason, into) -> None:
881 if playlist is None or playlist['id'] in seen_playlists:
882 return
883 seen_playlists.add(playlist['id'])
884 into.append(_playlist_item(playlist, artist, reason))
886 for seed in seeds:
887 results = searches.get(artist_key(seed)) or {}
888 taken = 0
889 for album in _albums_by(results, seed):
890 if taken >= MORE_FROM_PER_SEED:
891 break
892 if take_album(album, seed, 'more_from', [seed], more):
893 taken += 1
894 take_playlist(
895 _editorial_playlist(results, seed, radio=True),
896 seed,
897 'radio',
898 radios,
899 )
901 for artist in suggested:
902 name = artist['name']
903 results = searches.get(artist_key(name)) or {}
904 for found in results.get('artists') or []:
905 if artist_key(found.get('name')) == artist_key(name):
906 artist_urls[name] = (
907 f'https://open.spotify.com/artist/{found["id"]}'
908 )
909 break
910 for album in _albums_by(results, name):
911 if take_album(
912 album, name, 'similar', artist.get('because') or [], albums
913 ):
914 break
915 take_playlist(
916 _editorial_playlist(results, name, radio=False),
917 name,
918 'this_is',
919 this_is,
920 )
922 return {
923 'albums': albums,
924 'more_albums': more,
925 'playlists': radios + this_is,
926 'artist_urls': artist_urls,
927 }
930# ── Putting it together ──────────────────────────────────────────────────
933def _related_for_seed(
934 store: DiscoverStore,
935 key: str,
936 name: str,
937 *,
938 lookup_id: Callable[[str], Optional[str]],
939 fetch_related: Callable[[str, int], Related],
940 now: datetime,
941) -> tuple[Related, bool]:
942 """A seed's related artists, from the cache while fresh, else Deezer.
944 Returns ``(related, failed)``. When Deezer can't be asked (down, rate
945 limited), a stale cached answer is used if there is one, and nothing
946 is saved - "couldn't find out" is never remembered as "none". A seed
947 Deezer doesn't know at all *is* remembered (with no id and no related
948 artists), so it isn't searched for again until the cache runs out.
949 """
951 cached = store.cached_related(key)
952 if (
953 cached is not None
954 and cached['fetched_at'] is not None
955 and now - cached['fetched_at'] < RELATED_TTL
956 ):
957 return cached['related'], False
959 stale = cached['related'] if cached is not None else []
960 try:
961 deezer_id = (cached or {}).get('deezer_id') or lookup_id(name)
962 if not deezer_id:
963 store.save_related(key, '', [], when=now)
964 return [], False
965 related = fetch_related(deezer_id, RELATED_PER_SEED)
966 except ValueError:
967 return stale, True
968 store.save_related(key, deezer_id, related, when=now)
969 return related, False
972def recommendations(
973 store: DiscoverStore,
974 library: list[dict[str, Any]],
975 *,
976 limit: int = RESULT_LIMIT,
977 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id,
978 fetch_related: Callable[[str, int], Related] = deezer.related_artists,
979 now: Optional[datetime] = None,
980 user_id: int = 0,
981) -> dict[str, Any]:
982 """Suggested artists for a library.
984 *library* is ``{name, tracks, liked}`` per library artist (what the
985 Library page already groups - see ``frontend/src/lib/library.js``).
986 Returns ``{artists, seeds, partial}``: the ranked suggestions (see
987 :func:`rank_candidates`), the names of the seeds they came from, and
988 whether Deezer failed for any seed - the list is then built from what
989 could be fetched (plus stale cache), not an error.
990 """
992 now = now or _now()
993 library = [a for a in library if isinstance(a, dict)]
994 seeds = pick_seeds(seed_weights(library, store.listens(user_id), now))
995 if not seeds:
996 return {'artists': [], 'seeds': [], 'partial': False}
998 def fetch(seed: tuple[str, str, float]) -> tuple[str, Related, bool]:
999 key, name, _weight = seed
1000 try:
1001 related, failed = _related_for_seed(
1002 store,
1003 key,
1004 name,
1005 lookup_id=lookup_id,
1006 fetch_related=fetch_related,
1007 now=now,
1008 )
1009 except Exception:
1010 logger.opt(exception=True).warning(
1011 'Discover: related artists failed for {!r}', name
1012 )
1013 return key, [], True
1014 return key, related, failed
1016 with ThreadPoolExecutor(max_workers=_WORKERS) as pool:
1017 results = list(pool.map(fetch, seeds))
1019 related = {key: rows for key, rows, _failed in results}
1020 exclude = {artist_key(a.get('name')) for a in library}
1021 exclude |= store.blocked_keys(user_id)
1022 exclude.discard('')
1023 return {
1024 'artists': rank_candidates(seeds, related, exclude, limit=limit),
1025 'seeds': [name for _key, name, _weight in seeds],
1026 'partial': any(failed for _key, _rows, failed in results),
1027 }
1030def _search_for(
1031 store: DiscoverStore,
1032 name: str,
1033 *,
1034 search: Callable[[str], dict[str, Any]],
1035 now: datetime,
1036) -> tuple[dict[str, Any], bool]:
1037 """An artist's Spotify search, from the cache while fresh. Returns
1038 ``(results, failed)``; same failure rules as :func:`_related_for_seed`
1039 - a failed search falls back to a stale answer and is never saved."""
1041 key = artist_key(name)
1042 cached = store.cached_search(key)
1043 if (
1044 cached is not None
1045 and cached['fetched_at'] is not None
1046 and now - cached['fetched_at'] < RELATED_TTL
1047 ):
1048 return cached['results'], False
1049 try:
1050 results = search(name)
1051 except ValueError:
1052 return (cached['results'] if cached else {}), True
1053 store.save_search(key, results, when=now)
1054 return results, False
1057def collections(
1058 store: DiscoverStore,
1059 library: list[dict[str, Any]],
1060 owned_albums: Iterable[dict[str, Any]] = (),
1061 owned_playlists: Iterable[str] = (),
1062 *,
1063 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id,
1064 fetch_related: Callable[[str, int], Related] = deezer.related_artists,
1065 search: Callable[[str], dict[str, Any]] = spotify.search,
1066 now: Optional[datetime] = None,
1067) -> dict[str, Any]:
1068 """Albums and playlists for a library (see :func:`pick_collections`).
1070 *library* is what :func:`recommendations` takes (its answer comes from
1071 the same cache, so this costs no extra Deezer requests once the
1072 artists are shown); *owned_albums* ``{artist, title}`` per library
1073 album, *owned_playlists* the Spotify ids of downloaded playlists.
1074 ``partial`` is ``True`` when a search failed for some artist.
1075 """
1077 now = now or _now()
1078 library = [a for a in library if isinstance(a, dict)]
1079 ranked = recommendations(
1080 store,
1081 library,
1082 lookup_id=lookup_id,
1083 fetch_related=fetch_related,
1084 now=now,
1085 )
1086 suggested = ranked['artists'][:COLLECTION_SUGGESTED]
1087 seeds = ranked['seeds'][:COLLECTION_SEEDS]
1088 names = list(dict.fromkeys([*seeds, *(a['name'] for a in suggested)]))
1090 def fetch(name: str) -> tuple[str, dict[str, Any], bool]:
1091 try:
1092 results, failed = _search_for(store, name, search=search, now=now)
1093 except Exception:
1094 logger.opt(exception=True).warning(
1095 'Discover: Spotify search failed for {!r}', name
1096 )
1097 return name, {}, True
1098 return name, results, failed
1100 with ThreadPoolExecutor(max_workers=_WORKERS) as pool:
1101 results = list(pool.map(fetch, names))
1103 picked = pick_collections(
1104 suggested,
1105 seeds,
1106 {artist_key(name): found for name, found, _failed in results},
1107 {
1108 album_key(a.get('artist'), a.get('title'))
1109 for a in owned_albums
1110 if isinstance(a, dict)
1111 },
1112 {str(pid) for pid in owned_playlists if pid},
1113 )
1114 picked['partial'] = ranked['partial'] or any(
1115 failed for _name, _found, failed in results
1116 )
1117 return picked
1120# ── Deezer first, then Spotify matched to Deezer ─────────────────────────
1121#
1122# The web page's album and playlist shelves. Deezer's own picks come first
1123# (deezer_collections: quick, and every album opens in the Finder); what
1124# Spotify picks is added after (spotify_collections), each album matched
1125# back to Deezer by name when Deezer has it. Playlists aren't matched - a
1126# "100% <artist>" (Deezer's editors) and a "This Is <artist>" (Spotify's)
1127# are different lists - so each service keeps its own shelf.
1128#
1129# collections() above, Spotify only, is kept as it is: other clients may
1130# rely on it.
1133def _release_rank(album: dict[str, Any]) -> tuple[bool, int]:
1134 """Sort key for an artist's Deezer releases: full albums first (as
1135 :func:`_albums_by` does on Spotify), then the most fans."""
1137 kind = str(album.get('release_type') or '').lower()
1138 return kind != 'album', -int(album.get('fans') or 0)
1141def _deezer_album_item(
1142 album: dict[str, Any], artist: str, reason: str, because: list[str]
1143) -> dict[str, Any]:
1144 album_id = str(album['album_id'])
1145 return {
1146 'name': album['name'],
1147 'artist': artist,
1148 'year': album.get('year') or '',
1149 'cover_url': album.get('cover_url') or '',
1150 'source': 'deezer',
1151 'deezer_album_id': album_id,
1152 'deezer_artist_id': str(album.get('artist_id') or ''),
1153 'url': album.get('url') or f'https://www.deezer.com/album/{album_id}',
1154 'key': album_key(artist, album['name']),
1155 'reason': reason,
1156 'because': because,
1157 }
1160def _deezer_playlist_item(
1161 playlist: dict[str, Any], artist: str
1162) -> dict[str, Any]:
1163 return {
1164 'name': playlist['name'],
1165 'owner': playlist.get('owner') or '',
1166 'cover_url': playlist.get('cover_url') or '',
1167 'source': 'deezer',
1168 'deezer_playlist_id': str(playlist['playlist_id']),
1169 'url': playlist['url'],
1170 'reason': 'essentials',
1171 'artist': artist,
1172 }
1175def pick_deezer_collections(
1176 suggested: list[dict[str, Any]],
1177 seeds: list[str],
1178 discographies: dict[str, list[dict[str, Any]]],
1179 playlists: dict[str, Optional[dict[str, Any]]],
1180 owned_albums: set[str],
1181 owned_playlists: set[str],
1182) -> dict[str, Any]:
1183 """The same shelves as :func:`pick_collections`, from Deezer.
1185 *discographies* and *playlists* are keyed by :func:`artist_key`: each
1186 artist's :func:`downtify.deezer.artist_discography` and
1187 :func:`downtify.deezer.editorial_playlist`. *owned_albums* holds
1188 :func:`album_key` values, *owned_playlists* the
1189 :func:`~downtify.file_naming.file_name_key` of every library playlist
1190 name - a downloaded Deezer playlist keeps its title as its name.
1191 Returns ``{albums, more_albums, playlists}``:
1193 * ``albums``: each suggested artist's best album not owned - a full
1194 album before a single or EP, the most fans first - ``reason:
1195 'similar'``.
1196 * ``more_albums``: up to :data:`MORE_FROM_PER_SEED` per seed, same
1197 order, ``reason: 'more_from'``.
1198 * ``playlists``: the suggested artists' "100%" playlists,
1199 ``reason: 'essentials'``.
1200 """
1202 albums: list[dict[str, Any]] = []
1203 more: list[dict[str, Any]] = []
1204 lists: list[dict[str, Any]] = []
1205 seen: set[str] = set(owned_albums)
1207 def take(album, artist, reason, because, into) -> bool:
1208 key = album_key(artist, album['name'])
1209 if key in seen:
1210 return False
1211 seen.add(key)
1212 into.append(_deezer_album_item(album, artist, reason, because))
1213 return True
1215 for seed in seeds:
1216 taken = 0
1217 releases = discographies.get(artist_key(seed)) or []
1218 for album in sorted(releases, key=_release_rank):
1219 if taken >= MORE_FROM_PER_SEED:
1220 break
1221 if take(album, seed, 'more_from', [seed], more):
1222 taken += 1
1224 for artist in suggested:
1225 name = artist['name']
1226 because = artist.get('because') or []
1227 releases = discographies.get(artist_key(name)) or []
1228 for album in sorted(releases, key=_release_rank):
1229 if take(album, name, 'similar', because, albums):
1230 break
1231 playlist = playlists.get(artist_key(name))
1232 if playlist and file_name_key(playlist['name']) not in owned_playlists:
1233 lists.append(_deezer_playlist_item(playlist, name))
1235 return {'albums': albums, 'more_albums': more, 'playlists': lists}
1238def _deezer_part(
1239 store: DiscoverStore,
1240 key: str,
1241 part: str,
1242 fetch: Callable[[], Any],
1243 *,
1244 now: datetime,
1245) -> tuple[Any, bool]:
1246 """An artist's Deezer *part* from the cache while fresh, else
1247 *fetch*. ``(value, failed)``, with the same rules as
1248 :func:`_related_for_seed`: a failure falls back to a stale answer and
1249 is never saved; "Deezer has none" (``None``) is saved like any answer.
1250 """
1252 cached = store.cached_deezer(key, part)
1253 if (
1254 cached is not None
1255 and cached['fetched_at'] is not None
1256 and now - cached['fetched_at'] < RELATED_TTL
1257 ):
1258 return cached['value'], False
1259 try:
1260 value = fetch()
1261 except ValueError:
1262 return (cached['value'] if cached else None), True
1263 store.save_deezer(key, part, value, when=now)
1264 return value, False
1267def deezer_collections(
1268 store: DiscoverStore,
1269 library: list[dict[str, Any]],
1270 owned_albums: Iterable[dict[str, Any]] = (),
1271 owned_playlists: Iterable[str] = (),
1272 *,
1273 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id,
1274 fetch_related: Callable[[str, int], Related] = deezer.related_artists,
1275 fetch_albums: Callable[
1276 [str], list[dict[str, Any]]
1277 ] = deezer.artist_discography,
1278 fetch_playlist: Callable[
1279 [str], Optional[dict[str, Any]]
1280 ] = deezer.editorial_playlist,
1281 now: Optional[datetime] = None,
1282) -> dict[str, Any]:
1283 """Albums and playlists for a library from Deezer alone (see
1284 :func:`pick_deezer_collections`) - the page's first, quick answer.
1286 *library* and *owned_albums* are what :func:`collections` takes;
1287 *owned_playlists* are library playlist names. Built on the same ranked
1288 artists (cached), so it only asks Deezer for discographies and
1289 playlists: one each per artist, cached for :data:`RELATED_TTL`.
1290 ``partial`` is ``True`` when some of those couldn't be fetched.
1291 """
1293 now = now or _now()
1294 library = [a for a in library if isinstance(a, dict)]
1295 ranked = recommendations(
1296 store,
1297 library,
1298 lookup_id=lookup_id,
1299 fetch_related=fetch_related,
1300 now=now,
1301 )
1302 suggested = ranked['artists'][:COLLECTION_SUGGESTED]
1303 seeds = ranked['seeds'][:COLLECTION_SEEDS]
1305 # A suggestion carries its own Deezer id; a seed's was saved with its
1306 # related artists.
1307 names: dict[str, str] = {}
1308 ids: dict[str, str] = {}
1309 for seed in seeds:
1310 key = artist_key(seed)
1311 names[key] = seed
1312 ids[key] = (store.cached_related(key) or {}).get('deezer_id') or ''
1313 for artist in suggested:
1314 key = artist_key(artist['name'])
1315 names.setdefault(key, artist['name'])
1316 ids.setdefault(key, str(artist.get('deezer_id') or ''))
1318 jobs = [(key, 'albums') for key, deezer_id in ids.items() if deezer_id]
1319 jobs += [(artist_key(a['name']), 'playlist') for a in suggested]
1321 def fetcher(key: str, part: str) -> Callable[[], Any]:
1322 if part == 'albums':
1323 return lambda: fetch_albums(ids[key])
1324 return lambda: fetch_playlist(names[key])
1326 def run(job: tuple[str, str]) -> tuple[str, str, Any, bool]:
1327 key, part = job
1328 try:
1329 value, failed = _deezer_part(
1330 store, key, part, fetcher(key, part), now=now
1331 )
1332 except Exception:
1333 logger.opt(exception=True).warning(
1334 'Discover: Deezer {} failed for {!r}', part, names[key]
1335 )
1336 return key, part, None, True
1337 return key, part, value, failed
1339 with ThreadPoolExecutor(max_workers=_THROTTLED_WORKERS) as pool:
1340 results = list(pool.map(run, jobs))
1342 discographies = {
1343 key: value or [] for key, part, value, _ in results if part == 'albums'
1344 }
1345 playlists = {
1346 key: value for key, part, value, _ in results if part == 'playlist'
1347 }
1348 picked = pick_deezer_collections(
1349 suggested,
1350 seeds,
1351 discographies,
1352 playlists,
1353 {
1354 album_key(a.get('artist'), a.get('title'))
1355 for a in owned_albums
1356 if isinstance(a, dict)
1357 },
1358 {file_name_key(str(name)) for name in owned_playlists if name},
1359 )
1360 picked['partial'] = ranked['partial'] or any(
1361 failed for _key, _part, _value, failed in results
1362 )
1363 return picked
1366def _match_album(
1367 store: DiscoverStore,
1368 item: dict[str, Any],
1369 *,
1370 find: Callable[[str, str], list[dict[str, Any]]],
1371 now: datetime,
1372) -> tuple[Optional[dict[str, Any]], bool]:
1373 """A Spotify album *item*'s Deezer album, or ``None`` when Deezer has
1374 none: ``(match, failed)``.
1376 A row counts only when it's the same album by :func:`album_key` (same
1377 artist, same title once edition suffixes are dropped) - never "the
1378 closest". Matches are cached by Spotify id, a found one for
1379 :data:`MATCH_TTL_FOUND`, a missing one for :data:`MATCH_TTL_MISSING`;
1380 a failure falls back to a stale match and is never saved.
1381 """
1383 spotify_id = str(item['spotify_id'])
1384 cached = store.cached_album_match(spotify_id)
1385 if cached is not None and cached['fetched_at'] is not None:
1386 found = bool(cached['deezer_album_id'])
1387 ttl = MATCH_TTL_FOUND if found else MATCH_TTL_MISSING
1388 if now - cached['fetched_at'] < ttl:
1389 return (cached if found else None), False
1390 try:
1391 rows = find(item['artist'], item['name'])
1392 except ValueError:
1393 stale = cached if cached and cached['deezer_album_id'] else None
1394 return stale, True
1395 wanted = album_key(item['artist'], item['name'])
1396 row = next(
1397 (
1398 row
1399 for row in rows
1400 if album_key(row.get('artist'), row.get('name')) == wanted
1401 ),
1402 None,
1403 )
1404 store.save_album_match(spotify_id, row, when=now)
1405 if row is None:
1406 return None, False
1407 return {
1408 'deezer_album_id': str(row['album_id']),
1409 'deezer_artist_id': str(row.get('artist_id') or ''),
1410 'cover_url': row.get('cover_url') or '',
1411 }, False
1414def _matched_album_item(
1415 item: dict[str, Any], match: Optional[dict[str, Any]]
1416) -> dict[str, Any]:
1417 """A Spotify album *item* as Deezer's (opens in the Finder), or as
1418 Spotify's when Deezer has no match (opens on its Spotify page)."""
1420 key = album_key(item['artist'], item['name'])
1421 if match is None:
1422 return {**item, 'source': 'spotify', 'key': key}
1423 album_id = match['deezer_album_id']
1424 return {
1425 'name': item['name'],
1426 'artist': item['artist'],
1427 'year': item.get('year') or '',
1428 'cover_url': match.get('cover_url') or item.get('cover_url') or '',
1429 'source': 'deezer',
1430 'deezer_album_id': album_id,
1431 'deezer_artist_id': match.get('deezer_artist_id') or '',
1432 'url': f'https://www.deezer.com/album/{album_id}',
1433 'key': key,
1434 'reason': item['reason'],
1435 'because': item['because'],
1436 }
1439def spotify_collections(
1440 store: DiscoverStore,
1441 library: list[dict[str, Any]],
1442 owned_albums: Iterable[dict[str, Any]] = (),
1443 owned_playlists: Iterable[str] = (),
1444 shown: Iterable[str] = (),
1445 *,
1446 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id,
1447 fetch_related: Callable[[str, int], Related] = deezer.related_artists,
1448 search: Callable[[str], dict[str, Any]] = spotify.search,
1449 find_album: Callable[
1450 [str, str], list[dict[str, Any]]
1451 ] = deezer.search_albums_by,
1452 now: Optional[datetime] = None,
1453) -> dict[str, Any]:
1454 """What Spotify adds to :func:`deezer_collections` - the page's
1455 second, slower answer.
1457 Spotify's own picks (:func:`collections`, its searches cached), less
1458 the albums already *shown* (their :func:`album_key` values) - so an
1459 album both services pick costs no match - each matched to Deezer (see
1460 :func:`_match_album`). *owned_playlists* are Spotify playlist ids, as
1461 for :func:`collections`. Returns ``{albums, more_albums, playlists,
1462 partial}``: albums with ``source`` ``'deezer'`` (matched) or
1463 ``'spotify'`` (no match), and only the "This Is <artist>" playlists -
1464 Spotify's "Radio" mixes aren't offered here.
1465 """
1467 now = now or _now()
1468 picked = collections(
1469 store,
1470 library,
1471 owned_albums,
1472 owned_playlists,
1473 lookup_id=lookup_id,
1474 fetch_related=fetch_related,
1475 search=search,
1476 now=now,
1477 )
1478 skip = {str(key) for key in shown if key}
1479 jobs = [
1480 (shelf, item)
1481 for shelf in ('albums', 'more_albums')
1482 for item in picked[shelf]
1483 if album_key(item['artist'], item['name']) not in skip
1484 ]
1486 def run(job: tuple[str, dict[str, Any]]) -> tuple[str, dict, bool]:
1487 shelf, item = job
1488 try:
1489 match, failed = _match_album(store, item, find=find_album, now=now)
1490 except Exception:
1491 logger.opt(exception=True).warning(
1492 'Discover: matching {!r} to Deezer failed', item['name']
1493 )
1494 match, failed = None, True
1495 return shelf, _matched_album_item(item, match), failed
1497 with ThreadPoolExecutor(max_workers=_THROTTLED_WORKERS) as pool:
1498 results = list(pool.map(run, jobs))
1500 return {
1501 'albums': [item for shelf, item, _ in results if shelf == 'albums'],
1502 'more_albums': [
1503 item for shelf, item, _ in results if shelf == 'more_albums'
1504 ],
1505 'playlists': [
1506 {**playlist, 'source': 'spotify'}
1507 for playlist in picked['playlists']
1508 if playlist['reason'] == 'this_is'
1509 ],
1510 'partial': picked['partial']
1511 or any(failed for _shelf, _item, failed in results),
1512 }