Coverage for downtify/discover.py: 95%

507 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Discover: artists the user doesn't have yet, suggested from the ones 

2they do. 

3 

4Where the suggestions come from 

5------------------------------- 

6 

7Each *seed* is an artist already in the library. How much a seed counts 

8is its :func:`seed_weights`: how many of its tracks are in the library, 

9how many of those are liked, and how often (and how recently) the user 

10actually listened to it - the player reports a listen once a track has 

11played long enough (see :meth:`DiscoverStore.record_listen`). The 

12heaviest seeds are then looked up on Deezer, whose public, keyless 

13``/artist/{id}/related`` answers "fans of this artist also like" (see 

14:func:`downtify.deezer.related_artists`), and every related artist gets 

15points from each seed that suggests it (:func:`rank_candidates`). An 

16artist several of the user's favourites point to beats one only a single 

17seed mentions. 

18 

19Last.fm, which other tools use for this, needs an API key; Spotify's own 

20"Fans also like" needs a Spotify id per seed, and the embed pages this 

21project relies on (no Web API credentials) only hand that out per track. 

22Deezer needs neither. 

23 

24What is never suggested: an artist already in the library, and an artist 

25on the user's block list (:meth:`DiscoverStore.block`) - "Not interested" 

26on a suggestion, or a name typed in by hand. 

27 

28Albums and playlists 

29-------------------- 

30 

31:func:`collections` builds on the artists above, with the web player's 

32own search (anonymous, like everything else here - see 

33:func:`downtify.spotify.search`), one search per artist name: 

34 

35* **Albums for you** - the top album of each of the best suggested 

36 artists. 

37* **More from your artists** - the most popular albums of the user's own 

38 heaviest artists that aren't in the library yet. 

39* **Playlists** - Spotify's own "<artist> Radio" mixes for the user's 

40 heaviest artists (similar music around an artist they already love), 

41 then its "This Is <artist>" for the best suggested ones. 

42 

43Every result is a Spotify album or playlist, so opening one lands on the 

44same page a pasted link does, with the tracks' preview clips. Albums 

45already in the library and playlists already downloaded are left out. 

46 

47What is stored 

48-------------- 

49 

50Four tables in ``downtify_library.db``: the listen counts, the block 

51list, a per-seed cache of Deezer's answer (the seed's Deezer id and its 

52related artists) and a per-artist cache of the Spotify search, both 

53trusted for :data:`RELATED_TTL` so opening the page again doesn't cost a 

54round of requests - Deezer allows about 50 per 5 seconds, and a cold page 

55is two per seed. 

56""" 

57 

58from __future__ import annotations 

59 

60import json 

61import math 

62from collections.abc import Callable, Iterable 

63from concurrent.futures import ThreadPoolExecutor 

64from datetime import datetime, timedelta, timezone 

65from pathlib import Path 

66from typing import Any, Optional 

67 

68from loguru import logger 

69 

70from . import deezer, spotify 

71from .db.config import Dialect 

72from .db.core import Database, as_database 

73from .file_naming import file_name_key, title_key 

74 

75#: How long a seed's related artists are trusted before asking Deezer again. 

76RELATED_TTL = timedelta(days=7) 

77#: How many of the heaviest library artists are looked up. 

78MAX_SEEDS = 15 

79#: How many related artists are asked for per seed. 

80RELATED_PER_SEED = 20 

81#: How many suggestions are returned, at most. 

82RESULT_LIMIT = 48 

83#: Listening to an artist counts half as much after this many days. 

84LISTEN_HALF_LIFE_DAYS = 45 

85#: Deezer requests in flight at once (it rate-limits at ~50 per 5 s). 

86_WORKERS = 4 

87#: For Deezer lookups that go through its throttle (discographies, 

88#: playlists, album matches): more at once is safe, the throttle keeps the 

89#: overall rate under Deezer's limit. 

90_THROTTLED_WORKERS = 8 

91#: How many of the best suggested artists get an album / playlist. 

92COLLECTION_SUGGESTED = 12 

93#: How many of the heaviest library artists get albums / a radio mix. 

94COLLECTION_SEEDS = 6 

95#: Albums not in the library, at most, per library artist. 

96MORE_FROM_PER_SEED = 2 

97#: Spotify's own account, the owner of its editorial playlists. 

98SPOTIFY_OWNER = 'spotify' 

99#: How long a Spotify album's Deezer match is trusted: an album found on 

100#: Deezer stays the same album, one missing may be added any week. 

101MATCH_TTL_FOUND = timedelta(days=30) 

102MATCH_TTL_MISSING = RELATED_TTL 

103#: The per-artist Deezer cache's parts, by name -> column prefix (a fixed 

104#: list: the name ends up in the SQL). 

105_DEEZER_PARTS = {'albums': 'albums', 'playlist': 'playlist'} 

106 

107#: A seed's related artists, best match first (see ``deezer.related_artists``). 

108Related = list[dict[str, Any]] 

109 

110 

111def _now() -> datetime: 

112 return datetime.now(timezone.utc) 

113 

114 

115def artist_key(name: str) -> str: 

116 """The key two names of the same artist share: case, spaces and the 

117 characters a file name can't hold are ignored, so ``AC/DC`` in a tag 

118 and ``ACDC`` on Deezer are the same artist (see 

119 :func:`downtify.file_naming.file_name_key`).""" 

120 

121 return file_name_key(str(name or '')) 

122 

123 

124def _parse_time(value: Any) -> Optional[datetime]: 

125 try: 

126 parsed = datetime.fromisoformat(str(value)) 

127 except (TypeError, ValueError): 

128 return None 

129 if parsed.tzinfo is None: 

130 parsed = parsed.replace(tzinfo=timezone.utc) 

131 return parsed 

132 

133 

134# ── Ranking (pure) ─────────────────────────────────────────────────────── 

135 

136 

137def seed_weights( 

138 library: Iterable[dict[str, Any]], 

139 listens: Iterable[dict[str, Any]], 

140 now: Optional[datetime] = None, 

141) -> dict[str, dict[str, Any]]: 

142 """How much each library artist says about the user's taste. 

143 

144 *library* is ``{name, tracks, liked}`` per artist in the library; 

145 *listens* the rows of :meth:`DiscoverStore.listens`. The weight grows 

146 with each signal but flattens out (``log1p``), so a 400-track 

147 discography doesn't drown everything else, and listening counts most, 

148 liking next, owning least. A listen fades with the time since the 

149 artist was last played (:data:`LISTEN_HALF_LIFE_DAYS`). 

150 

151 Only artists in *library* can be seeds - a listen to an artist whose 

152 tracks have since been deleted doesn't count. Returns ``{key: {name, 

153 weight}}``, keyed by :func:`artist_key`; artists with no weight at all 

154 are left out. 

155 """ 

156 

157 now = now or _now() 

158 plays: dict[str, float] = {} 

159 for row in listens: 

160 key = artist_key(row.get('name')) 

161 if not key: 

162 continue 

163 last = _parse_time(row.get('last_played')) 

164 age_days = ( 

165 max(0.0, (now - last).total_seconds() / 86400) if last else 0 

166 ) 

167 fade = 0.5 ** (age_days / LISTEN_HALF_LIFE_DAYS) 

168 plays[key] = ( 

169 plays.get(key, 0.0) + max(0, int(row.get('plays') or 0)) * fade 

170 ) 

171 

172 weights: dict[str, dict[str, Any]] = {} 

173 for artist in library: 

174 name = str(artist.get('name') or '').strip() 

175 key = artist_key(name) 

176 if not key: 

177 continue 

178 tracks = max(0, int(artist.get('tracks') or 0)) 

179 liked = max(0, int(artist.get('liked') or 0)) 

180 weight = ( 

181 math.log1p(tracks) 

182 + 1.5 * math.log1p(liked) 

183 + 2.0 * math.log1p(plays.get(key, 0.0)) 

184 ) 

185 if weight <= 0: 

186 continue 

187 entry = weights.setdefault(key, {'name': name, 'weight': 0.0}) 

188 entry['weight'] += weight 

189 return weights 

190 

191 

192def pick_seeds( 

193 weights: dict[str, dict[str, Any]], limit: int = MAX_SEEDS 

194) -> list[tuple[str, str, float]]: 

195 """The *limit* heaviest seeds as ``(key, name, weight)``, heaviest 

196 first (ties by name, so the pick is stable).""" 

197 

198 ranked = sorted( 

199 weights.items(), 

200 key=lambda item: (-item[1]['weight'], item[1]['name'].casefold()), 

201 ) 

202 return [(key, v['name'], v['weight']) for key, v in ranked[:limit]] 

203 

204 

205def rank_candidates( 

206 seeds: list[tuple[str, str, float]], 

207 related: dict[str, Related], 

208 exclude: set[str], 

209 limit: int = RESULT_LIMIT, 

210) -> list[dict[str, Any]]: 

211 """Every related artist, scored by the seeds that suggest it. 

212 

213 A seed gives a related artist its own weight, less the further down 

214 Deezer's list it sits; an artist several seeds suggest adds up all of 

215 theirs. Keys in *exclude* (the library, the block list) are never 

216 suggested, and neither is a seed itself. 

217 

218 Each result is ``{name, deezer_id, picture_url, fans, score, 

219 because}`` - ``because`` the names of up to three seeds that suggested 

220 it, the biggest contributor first. Best first; ties go to the artist 

221 with more fans. 

222 """ 

223 

224 seed_keys = {key for key, _name, _weight in seeds} 

225 found: dict[str, dict[str, Any]] = {} 

226 for seed_key, seed_name, weight in seeds: 

227 for position, artist in enumerate(related.get(seed_key) or []): 

228 key = artist_key(artist.get('name')) 

229 if not key or key in exclude or key in seed_keys: 

230 continue 

231 points = weight / (1 + 0.15 * position) 

232 entry = found.get(key) 

233 if entry is None: 

234 entry = found[key] = { 

235 'name': artist['name'], 

236 'deezer_id': str(artist.get('deezer_id') or ''), 

237 'picture_url': str(artist.get('picture_url') or ''), 

238 'fans': int(artist.get('fans') or 0), 

239 'score': 0.0, 

240 '_by': {}, 

241 } 

242 entry['score'] += points 

243 entry['_by'][seed_name] = entry['_by'].get(seed_name, 0) + points 

244 

245 ranked = sorted( 

246 found.values(), 

247 key=lambda e: (-e['score'], -e['fans'], e['name'].casefold()), 

248 )[:limit] 

249 for entry in ranked: 

250 by = entry.pop('_by') 

251 entry['because'] = sorted(by, key=lambda n: -by[n])[:3] 

252 entry['score'] = round(entry['score'], 3) 

253 return ranked 

254 

255 

256# ── Storage ────────────────────────────────────────────────────────────── 

257 

258 

259class DiscoverStore: 

260 """Listen counts, the block list, and the related-artists cache.""" 

261 

262 def __init__(self, db_path: str | Path | Database) -> None: 

263 self.db = as_database(db_path, 'library') 

264 if isinstance(db_path, Database): 

265 self._path = str(self.db.path) if self.db.path else None 

266 else: 

267 self._path = str(db_path) 

268 self._init_db() 

269 

270 def _connect(self): 

271 return self.db.connect() 

272 

273 def _init_db(self) -> None: 

274 # PostgreSQL gets its schema from the migrations. 

275 if self.db.dialect is not Dialect.SQLITE: 

276 return 

277 with self._connect() as conn: 

278 conn.execute(""" 

279 CREATE TABLE IF NOT EXISTS discover_listens ( 

280 artist_key TEXT NOT NULL, 

281 user_id INTEGER NOT NULL DEFAULT 0, 

282 name TEXT NOT NULL, 

283 plays INTEGER NOT NULL DEFAULT 0, 

284 last_played TEXT NOT NULL, 

285 PRIMARY KEY (artist_key, user_id) 

286 ) 

287 """) 

288 conn.execute(""" 

289 CREATE TABLE IF NOT EXISTS discover_plays ( 

290 play_id TEXT PRIMARY KEY, 

291 recorded_at TEXT NOT NULL 

292 ) 

293 """) 

294 conn.execute(""" 

295 CREATE TABLE IF NOT EXISTS discover_blocked ( 

296 artist_key TEXT NOT NULL, 

297 user_id INTEGER NOT NULL DEFAULT 0, 

298 name TEXT NOT NULL, 

299 blocked_at TEXT NOT NULL, 

300 PRIMARY KEY (artist_key, user_id) 

301 ) 

302 """) 

303 self._migrate(conn) 

304 conn.execute(""" 

305 CREATE TABLE IF NOT EXISTS discover_search ( 

306 artist_key TEXT PRIMARY KEY, 

307 results_json TEXT NOT NULL DEFAULT '{}', 

308 fetched_at TEXT NOT NULL 

309 ) 

310 """) 

311 conn.execute(""" 

312 CREATE TABLE IF NOT EXISTS discover_related ( 

313 artist_key TEXT PRIMARY KEY, 

314 deezer_id TEXT NOT NULL DEFAULT '', 

315 related_json TEXT NOT NULL DEFAULT '[]', 

316 fetched_at TEXT NOT NULL 

317 ) 

318 """) 

319 # An artist's Deezer discography and editorial "100%" 

320 # playlist, each fetched (and trusted) on its own. 

321 conn.execute(""" 

322 CREATE TABLE IF NOT EXISTS discover_deezer ( 

323 artist_key TEXT PRIMARY KEY, 

324 albums_json TEXT, 

325 albums_at TEXT, 

326 playlist_json TEXT, 

327 playlist_at TEXT 

328 ) 

329 """) 

330 # A Spotify album's match on Deezer - '' ids when Deezer has 

331 # no such album. 

332 conn.execute(""" 

333 CREATE TABLE IF NOT EXISTS discover_album_map ( 

334 spotify_id TEXT PRIMARY KEY, 

335 deezer_album_id TEXT NOT NULL DEFAULT '', 

336 deezer_artist_id TEXT NOT NULL DEFAULT '', 

337 cover_url TEXT NOT NULL DEFAULT '', 

338 fetched_at TEXT NOT NULL 

339 ) 

340 """) 

341 

342 @staticmethod 

343 def _migrate(conn: Any) -> None: 

344 """One-time upgrade of single-user tables to per-user ones.""" 

345 

346 columns = { 

347 row[1] 

348 for row in conn.execute('PRAGMA table_info(discover_listens)') 

349 } 

350 if 'user_id' not in columns: 

351 conn.execute(""" 

352 CREATE TABLE discover_listens_new ( 

353 artist_key TEXT NOT NULL, 

354 user_id INTEGER NOT NULL DEFAULT 0, 

355 name TEXT NOT NULL, 

356 plays INTEGER NOT NULL DEFAULT 0, 

357 last_played TEXT NOT NULL, 

358 PRIMARY KEY (artist_key, user_id) 

359 ) 

360 """) 

361 conn.execute(""" 

362 INSERT INTO discover_listens_new 

363 (artist_key, user_id, name, plays, last_played) 

364 SELECT artist_key, 0, name, plays, last_played 

365 FROM discover_listens 

366 """) 

367 conn.execute('DROP TABLE discover_listens') 

368 conn.execute( 

369 'ALTER TABLE discover_listens_new RENAME TO discover_listens' 

370 ) 

371 columns = { 

372 row[1] 

373 for row in conn.execute('PRAGMA table_info(discover_blocked)') 

374 } 

375 if 'user_id' not in columns: 

376 conn.execute(""" 

377 CREATE TABLE discover_blocked_new ( 

378 artist_key TEXT NOT NULL, 

379 user_id INTEGER NOT NULL DEFAULT 0, 

380 name TEXT NOT NULL, 

381 blocked_at TEXT NOT NULL, 

382 PRIMARY KEY (artist_key, user_id) 

383 ) 

384 """) 

385 conn.execute(""" 

386 INSERT INTO discover_blocked_new 

387 (artist_key, user_id, name, blocked_at) 

388 SELECT artist_key, 0, name, blocked_at 

389 FROM discover_blocked 

390 """) 

391 conn.execute('DROP TABLE discover_blocked') 

392 conn.execute( 

393 'ALTER TABLE discover_blocked_new RENAME TO discover_blocked' 

394 ) 

395 

396 # Listens 

397 

398 def record_listen( 

399 self, 

400 name: str, 

401 when: Optional[datetime] = None, 

402 play_id: str = '', 

403 user_id: int = 0, 

404 ) -> Optional[dict[str, Any]]: 

405 """Count one listen to *name* for *user_id*. ``None`` for a name 

406 with no key. 

407 

408 *when* is when it was played - an app that was offline reports 

409 its plays later; a late report never moves ``last_played`` back. 

410 A *play_id* is counted once: the same play reported twice (a 

411 retry after a lost answer) doesn't add up. 

412 """ 

413 

414 name = str(name or '').strip() 

415 key = artist_key(name) 

416 if not key: 

417 return None 

418 stamp = (when or _now()).isoformat() 

419 uid = int(user_id or 0) 

420 with self._connect() as conn: 

421 fresh = True 

422 if play_id: 

423 cur = conn.execute( 

424 'INSERT INTO discover_plays (play_id, recorded_at) ' 

425 'VALUES (?, ?) ON CONFLICT(play_id) DO NOTHING', 

426 (play_id[:64], _now().isoformat()), 

427 ) 

428 fresh = cur.rowcount > 0 

429 if fresh: 

430 conn.execute( 

431 """INSERT INTO discover_listens 

432 (artist_key, user_id, name, plays, last_played) 

433 VALUES (?, ?, ?, 1, ?) 

434 ON CONFLICT(artist_key, user_id) DO UPDATE SET 

435 name = excluded.name, 

436 plays = discover_listens.plays + 1, 

437 last_played = CASE 

438 WHEN excluded.last_played > discover_listens.last_played 

439 THEN excluded.last_played 

440 ELSE discover_listens.last_played 

441 END""", 

442 (key, uid, name, stamp), 

443 ) 

444 row = conn.execute( 

445 'SELECT name, plays, last_played FROM discover_listens ' 

446 'WHERE artist_key = ? AND user_id = ?', 

447 (key, uid), 

448 ).fetchone() 

449 return dict(row) 

450 

451 @property 

452 def _name_sort(self) -> str: 

453 """Text sort matching SQLite's byte order on both backends.""" 

454 if self.db.dialect is Dialect.SQLITE: 

455 return 'name' 

456 return 'name COLLATE "C"' 

457 

458 def listens(self, user_id: Optional[int] = None) -> list[dict[str, Any]]: 

459 order = f'ORDER BY plays DESC, {self._name_sort}' 

460 with self._connect() as conn: 

461 if user_id is None: 

462 rows = conn.execute( 

463 'SELECT name, plays, last_played FROM discover_listens ' 

464 + order 

465 ).fetchall() 

466 else: 

467 rows = conn.execute( 

468 'SELECT name, plays, last_played FROM discover_listens ' 

469 'WHERE user_id = ? ' + order, 

470 (int(user_id),), 

471 ).fetchall() 

472 return [dict(row) for row in rows] 

473 

474 def clear_listens(self, user_id: Optional[int] = None) -> int: 

475 with self._connect() as conn: 

476 if user_id is None: 

477 return conn.execute('DELETE FROM discover_listens').rowcount 

478 cur = conn.execute( 

479 'DELETE FROM discover_listens WHERE user_id = ?', 

480 (int(user_id),), 

481 ) 

482 return cur.rowcount 

483 

484 def reassign_legacy(self, user_id: int) -> int: 

485 """Give pre-feature listens and blocks to *user_id* (once).""" 

486 

487 if not user_id: 

488 return 0 

489 with self._connect() as conn: 

490 listens = conn.execute( 

491 'UPDATE discover_listens SET user_id = ? WHERE user_id = 0', 

492 (int(user_id),), 

493 ).rowcount 

494 blocked = conn.execute( 

495 'UPDATE discover_blocked SET user_id = ? WHERE user_id = 0', 

496 (int(user_id),), 

497 ).rowcount 

498 return listens + blocked 

499 

500 def remove_user(self, user_id: int) -> None: 

501 """Forget a deleted account's listens and blocks.""" 

502 

503 with self._connect() as conn: 

504 conn.execute( 

505 'DELETE FROM discover_listens WHERE user_id = ?', 

506 (int(user_id),), 

507 ) 

508 conn.execute( 

509 'DELETE FROM discover_blocked WHERE user_id = ?', 

510 (int(user_id),), 

511 ) 

512 

513 # Block list 

514 

515 def block(self, name: str, user_id: int = 0) -> Optional[dict[str, Any]]: 

516 """Never suggest *name* again. ``None`` for a name with no key.""" 

517 

518 name = str(name or '').strip() 

519 key = artist_key(name) 

520 if not key: 

521 return None 

522 stamp = _now().isoformat() 

523 uid = int(user_id or 0) 

524 with self._connect() as conn: 

525 conn.execute( 

526 """INSERT INTO discover_blocked (artist_key, user_id, name, 

527 blocked_at) 

528 VALUES (?, ?, ?, ?) 

529 ON CONFLICT(artist_key, user_id) DO NOTHING""", 

530 (key, uid, name, stamp), 

531 ) 

532 row = conn.execute( 

533 'SELECT name, blocked_at FROM discover_blocked ' 

534 'WHERE artist_key = ? AND user_id = ?', 

535 (key, uid), 

536 ).fetchone() 

537 return dict(row) 

538 

539 def unblock(self, name: str, user_id: Optional[int] = None) -> bool: 

540 with self._connect() as conn: 

541 if user_id is None: 

542 cur = conn.execute( 

543 'DELETE FROM discover_blocked WHERE artist_key = ?', 

544 (artist_key(name),), 

545 ) 

546 else: 

547 cur = conn.execute( 

548 'DELETE FROM discover_blocked WHERE artist_key = ? ' 

549 'AND user_id = ?', 

550 (artist_key(name), int(user_id)), 

551 ) 

552 return cur.rowcount > 0 

553 

554 def blocked(self, user_id: Optional[int] = None) -> list[dict[str, Any]]: 

555 order = f'ORDER BY blocked_at DESC, {self._name_sort}' 

556 with self._connect() as conn: 

557 if user_id is None: 

558 rows = conn.execute( 

559 'SELECT name, blocked_at FROM discover_blocked ' + order 

560 ).fetchall() 

561 else: 

562 rows = conn.execute( 

563 'SELECT name, blocked_at FROM discover_blocked ' 

564 'WHERE user_id = ? ' + order, 

565 (int(user_id),), 

566 ).fetchall() 

567 return [dict(row) for row in rows] 

568 

569 def blocked_keys(self, user_id: Optional[int] = None) -> set[str]: 

570 with self._connect() as conn: 

571 if user_id is None: 

572 rows = conn.execute( 

573 'SELECT artist_key FROM discover_blocked' 

574 ).fetchall() 

575 else: 

576 rows = conn.execute( 

577 'SELECT artist_key FROM discover_blocked WHERE user_id ' 

578 '= ?', 

579 (int(user_id),), 

580 ).fetchall() 

581 return {row['artist_key'] for row in rows} 

582 

583 # Related-artists cache 

584 

585 def cached_related(self, key: str) -> Optional[dict[str, Any]]: 

586 """``{deezer_id, related, fetched_at}`` for a seed, or ``None``.""" 

587 

588 with self._connect() as conn: 

589 row = conn.execute( 

590 'SELECT deezer_id, related_json, fetched_at ' 

591 'FROM discover_related WHERE artist_key = ?', 

592 (key,), 

593 ).fetchone() 

594 if row is None: 

595 return None 

596 try: 

597 related = json.loads(row['related_json']) 

598 except ValueError: 

599 related = [] 

600 return { 

601 'deezer_id': row['deezer_id'], 

602 'related': related if isinstance(related, list) else [], 

603 'fetched_at': _parse_time(row['fetched_at']), 

604 } 

605 

606 def save_related( 

607 self, 

608 key: str, 

609 deezer_id: str, 

610 related: Related, 

611 when: Optional[datetime] = None, 

612 ) -> None: 

613 with self._connect() as conn: 

614 conn.execute( 

615 """INSERT INTO discover_related 

616 (artist_key, deezer_id, related_json, fetched_at) 

617 VALUES (?, ?, ?, ?) 

618 ON CONFLICT(artist_key) DO UPDATE SET 

619 deezer_id = excluded.deezer_id, 

620 related_json = excluded.related_json, 

621 fetched_at = excluded.fetched_at""", 

622 ( 

623 key, 

624 deezer_id or '', 

625 json.dumps(related), 

626 (when or _now()).isoformat(), 

627 ), 

628 ) 

629 

630 # Spotify search cache 

631 

632 def cached_search(self, key: str) -> Optional[dict[str, Any]]: 

633 """``{results, fetched_at}`` for an artist's search, or ``None``.""" 

634 

635 with self._connect() as conn: 

636 row = conn.execute( 

637 'SELECT results_json, fetched_at FROM discover_search ' 

638 'WHERE artist_key = ?', 

639 (key,), 

640 ).fetchone() 

641 if row is None: 

642 return None 

643 try: 

644 results = json.loads(row['results_json']) 

645 except ValueError: 

646 results = {} 

647 return { 

648 'results': results if isinstance(results, dict) else {}, 

649 'fetched_at': _parse_time(row['fetched_at']), 

650 } 

651 

652 def save_search( 

653 self, 

654 key: str, 

655 results: dict[str, Any], 

656 when: Optional[datetime] = None, 

657 ) -> None: 

658 with self._connect() as conn: 

659 conn.execute( 

660 """INSERT INTO discover_search 

661 (artist_key, results_json, fetched_at) 

662 VALUES (?, ?, ?) 

663 ON CONFLICT(artist_key) DO UPDATE SET 

664 results_json = excluded.results_json, 

665 fetched_at = excluded.fetched_at""", 

666 (key, json.dumps(results), (when or _now()).isoformat()), 

667 ) 

668 

669 # Deezer discography / editorial playlist cache 

670 

671 def cached_deezer(self, key: str, part: str) -> Optional[dict[str, Any]]: 

672 """``{value, fetched_at}`` for an artist's Deezer *part* 

673 (``'albums'`` or ``'playlist'``), or ``None`` if never fetched.""" 

674 

675 column = _DEEZER_PARTS[part] 

676 with self._connect() as conn: 

677 row = conn.execute( 

678 f'SELECT {column}_json AS value, {column}_at AS fetched_at ' 

679 'FROM discover_deezer WHERE artist_key = ?', 

680 (key,), 

681 ).fetchone() 

682 if row is None or row['fetched_at'] is None: 

683 return None 

684 try: 

685 value = json.loads(row['value'] or 'null') 

686 except ValueError: 

687 value = None 

688 return {'value': value, 'fetched_at': _parse_time(row['fetched_at'])} 

689 

690 def save_deezer( 

691 self, 

692 key: str, 

693 part: str, 

694 value: Any, 

695 when: Optional[datetime] = None, 

696 ) -> None: 

697 column = _DEEZER_PARTS[part] 

698 with self._connect() as conn: 

699 conn.execute( 

700 f"""INSERT INTO discover_deezer 

701 (artist_key, {column}_json, {column}_at) 

702 VALUES (?, ?, ?) 

703 ON CONFLICT(artist_key) DO UPDATE SET 

704 {column}_json = excluded.{column}_json, 

705 {column}_at = excluded.{column}_at""", 

706 (key, json.dumps(value), (when or _now()).isoformat()), 

707 ) 

708 

709 # Spotify -> Deezer album matches 

710 

711 def cached_album_match(self, spotify_id: str) -> Optional[dict[str, Any]]: 

712 """``{deezer_album_id, deezer_artist_id, cover_url, fetched_at}`` 

713 for a Spotify album, or ``None`` if never matched.""" 

714 

715 with self._connect() as conn: 

716 row = conn.execute( 

717 'SELECT deezer_album_id, deezer_artist_id, cover_url, ' 

718 'fetched_at FROM discover_album_map WHERE spotify_id = ?', 

719 (spotify_id,), 

720 ).fetchone() 

721 if row is None: 

722 return None 

723 return { 

724 'deezer_album_id': row['deezer_album_id'], 

725 'deezer_artist_id': row['deezer_artist_id'], 

726 'cover_url': row['cover_url'], 

727 'fetched_at': _parse_time(row['fetched_at']), 

728 } 

729 

730 def save_album_match( 

731 self, 

732 spotify_id: str, 

733 match: Optional[dict[str, Any]], 

734 when: Optional[datetime] = None, 

735 ) -> None: 

736 """Remember *spotify_id*'s Deezer album (*match*, a 

737 :func:`downtify.deezer.search_albums_by` row), or - ``None`` - 

738 that Deezer has none.""" 

739 

740 match = match or {} 

741 with self._connect() as conn: 

742 conn.execute( 

743 """INSERT INTO discover_album_map 

744 (spotify_id, deezer_album_id, deezer_artist_id, 

745 cover_url, fetched_at) 

746 VALUES (?, ?, ?, ?, ?) 

747 ON CONFLICT(spotify_id) DO UPDATE SET 

748 deezer_album_id = excluded.deezer_album_id, 

749 deezer_artist_id = excluded.deezer_artist_id, 

750 cover_url = excluded.cover_url, 

751 fetched_at = excluded.fetched_at""", 

752 ( 

753 spotify_id, 

754 str(match.get('album_id') or ''), 

755 str(match.get('artist_id') or ''), 

756 str(match.get('cover_url') or ''), 

757 (when or _now()).isoformat(), 

758 ), 

759 ) 

760 

761 

762# ── Albums and playlists (pure) ────────────────────────────────────────── 

763 

764 

765def album_key(artist: str, title: str) -> str: 

766 """The key an album is known by across services: its artist's 

767 :func:`artist_key` and its :func:`~downtify.file_naming.title_key`, so 

768 ``Dummy (Deluxe Edition)`` on Spotify is the ``Dummy`` on disk.""" 

769 

770 return f'{artist_key(artist)}|{title_key(title)}' 

771 

772 

773def _album_item( 

774 album: dict[str, Any], artist: str, reason: str, because: list[str] 

775) -> dict[str, Any]: 

776 return { 

777 'name': album['name'], 

778 'artist': artist, 

779 'year': album.get('year') or '', 

780 'cover_url': album.get('cover_url') or '', 

781 'spotify_id': album['id'], 

782 'url': f'https://open.spotify.com/album/{album["id"]}', 

783 'reason': reason, 

784 'because': because, 

785 } 

786 

787 

788def _playlist_item( 

789 playlist: dict[str, Any], artist: str, reason: str 

790) -> dict[str, Any]: 

791 return { 

792 'name': playlist['name'], 

793 'owner': playlist.get('owner') or '', 

794 'cover_url': playlist.get('cover_url') or '', 

795 'spotify_id': playlist['id'], 

796 'url': f'https://open.spotify.com/playlist/{playlist["id"]}', 

797 'reason': reason, 

798 'artist': artist, 

799 } 

800 

801 

802def _albums_by(results: dict[str, Any], name: str) -> list[dict[str, Any]]: 

803 """The albums in an artist's search *results* that are theirs (first 

804 credited artist), full albums before singles/EPs, otherwise in 

805 Spotify's (popularity) order.""" 

806 

807 key = artist_key(name) 

808 own = [ 

809 album 

810 for album in results.get('albums') or [] 

811 if album.get('artists') and artist_key(album['artists'][0]) == key 

812 ] 

813 return sorted(own, key=lambda a: a.get('type') != 'ALBUM') 

814 

815 

816def _editorial_playlist( 

817 results: dict[str, Any], name: str, *, radio: bool 

818) -> Optional[dict[str, Any]]: 

819 """Spotify's own "<name> Radio" (*radio*) or "This Is <name>" playlist 

820 in an artist's search *results*, or ``None``.""" 

821 

822 key = artist_key(name) 

823 for playlist in results.get('playlists') or []: 

824 if artist_key(playlist.get('owner')) != SPOTIFY_OWNER: 

825 continue 

826 title = artist_key(playlist.get('name')) 

827 if key not in title: 

828 continue 

829 # What's left once the artist's own name is out: "Radiohead" must 

830 # not make "This Is Radiohead" a radio mix. 

831 rest = title.replace(key, ' ') 

832 if radio and 'radio' in rest.split(): 

833 return playlist 

834 if not radio and 'this is' in rest: 

835 return playlist 

836 return None 

837 

838 

839def pick_collections( 

840 suggested: list[dict[str, Any]], 

841 seeds: list[str], 

842 searches: dict[str, dict[str, Any]], 

843 owned_albums: set[str], 

844 owned_playlists: set[str], 

845) -> dict[str, Any]: 

846 """Albums and playlists from each artist's Spotify *searches* (keyed 

847 by :func:`artist_key`). 

848 

849 *suggested* are :func:`rank_candidates` results (best first), *seeds* 

850 the user's heaviest library artists. Returns ``{albums, more_albums, 

851 playlists, artist_urls}``: 

852 

853 * ``albums``: each suggested artist's top album not in 

854 *owned_albums* (keys from :func:`album_key`), ``reason: 'similar'``. 

855 * ``more_albums``: up to :data:`MORE_FROM_PER_SEED` albums per seed 

856 not in *owned_albums*, ``reason: 'more_from'``. 

857 * ``playlists``: the seeds' "<artist> Radio", then the suggested 

858 artists' "This Is <artist>" (``reason: 'radio'`` / ``'this_is'``), 

859 leaving out ids in *owned_playlists*. 

860 * ``artist_urls``: ``{name: Spotify artist URL}`` for every suggested 

861 artist the search found by exact name. 

862 """ 

863 

864 albums: list[dict[str, Any]] = [] 

865 more: list[dict[str, Any]] = [] 

866 radios: list[dict[str, Any]] = [] 

867 this_is: list[dict[str, Any]] = [] 

868 artist_urls: dict[str, str] = {} 

869 seen_albums: set[str] = set(owned_albums) 

870 seen_playlists: set[str] = set(owned_playlists) 

871 

872 def take_album(album, artist, reason, because, into) -> bool: 

873 key = album_key(artist, album['name']) 

874 if key in seen_albums: 

875 return False 

876 seen_albums.add(key) 

877 into.append(_album_item(album, artist, reason, because)) 

878 return True 

879 

880 def take_playlist(playlist, artist, reason, into) -> None: 

881 if playlist is None or playlist['id'] in seen_playlists: 

882 return 

883 seen_playlists.add(playlist['id']) 

884 into.append(_playlist_item(playlist, artist, reason)) 

885 

886 for seed in seeds: 

887 results = searches.get(artist_key(seed)) or {} 

888 taken = 0 

889 for album in _albums_by(results, seed): 

890 if taken >= MORE_FROM_PER_SEED: 

891 break 

892 if take_album(album, seed, 'more_from', [seed], more): 

893 taken += 1 

894 take_playlist( 

895 _editorial_playlist(results, seed, radio=True), 

896 seed, 

897 'radio', 

898 radios, 

899 ) 

900 

901 for artist in suggested: 

902 name = artist['name'] 

903 results = searches.get(artist_key(name)) or {} 

904 for found in results.get('artists') or []: 

905 if artist_key(found.get('name')) == artist_key(name): 

906 artist_urls[name] = ( 

907 f'https://open.spotify.com/artist/{found["id"]}' 

908 ) 

909 break 

910 for album in _albums_by(results, name): 

911 if take_album( 

912 album, name, 'similar', artist.get('because') or [], albums 

913 ): 

914 break 

915 take_playlist( 

916 _editorial_playlist(results, name, radio=False), 

917 name, 

918 'this_is', 

919 this_is, 

920 ) 

921 

922 return { 

923 'albums': albums, 

924 'more_albums': more, 

925 'playlists': radios + this_is, 

926 'artist_urls': artist_urls, 

927 } 

928 

929 

930# ── Putting it together ────────────────────────────────────────────────── 

931 

932 

933def _related_for_seed( 

934 store: DiscoverStore, 

935 key: str, 

936 name: str, 

937 *, 

938 lookup_id: Callable[[str], Optional[str]], 

939 fetch_related: Callable[[str, int], Related], 

940 now: datetime, 

941) -> tuple[Related, bool]: 

942 """A seed's related artists, from the cache while fresh, else Deezer. 

943 

944 Returns ``(related, failed)``. When Deezer can't be asked (down, rate 

945 limited), a stale cached answer is used if there is one, and nothing 

946 is saved - "couldn't find out" is never remembered as "none". A seed 

947 Deezer doesn't know at all *is* remembered (with no id and no related 

948 artists), so it isn't searched for again until the cache runs out. 

949 """ 

950 

951 cached = store.cached_related(key) 

952 if ( 

953 cached is not None 

954 and cached['fetched_at'] is not None 

955 and now - cached['fetched_at'] < RELATED_TTL 

956 ): 

957 return cached['related'], False 

958 

959 stale = cached['related'] if cached is not None else [] 

960 try: 

961 deezer_id = (cached or {}).get('deezer_id') or lookup_id(name) 

962 if not deezer_id: 

963 store.save_related(key, '', [], when=now) 

964 return [], False 

965 related = fetch_related(deezer_id, RELATED_PER_SEED) 

966 except ValueError: 

967 return stale, True 

968 store.save_related(key, deezer_id, related, when=now) 

969 return related, False 

970 

971 

972def recommendations( 

973 store: DiscoverStore, 

974 library: list[dict[str, Any]], 

975 *, 

976 limit: int = RESULT_LIMIT, 

977 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id, 

978 fetch_related: Callable[[str, int], Related] = deezer.related_artists, 

979 now: Optional[datetime] = None, 

980 user_id: int = 0, 

981) -> dict[str, Any]: 

982 """Suggested artists for a library. 

983 

984 *library* is ``{name, tracks, liked}`` per library artist (what the 

985 Library page already groups - see ``frontend/src/lib/library.js``). 

986 Returns ``{artists, seeds, partial}``: the ranked suggestions (see 

987 :func:`rank_candidates`), the names of the seeds they came from, and 

988 whether Deezer failed for any seed - the list is then built from what 

989 could be fetched (plus stale cache), not an error. 

990 """ 

991 

992 now = now or _now() 

993 library = [a for a in library if isinstance(a, dict)] 

994 seeds = pick_seeds(seed_weights(library, store.listens(user_id), now)) 

995 if not seeds: 

996 return {'artists': [], 'seeds': [], 'partial': False} 

997 

998 def fetch(seed: tuple[str, str, float]) -> tuple[str, Related, bool]: 

999 key, name, _weight = seed 

1000 try: 

1001 related, failed = _related_for_seed( 

1002 store, 

1003 key, 

1004 name, 

1005 lookup_id=lookup_id, 

1006 fetch_related=fetch_related, 

1007 now=now, 

1008 ) 

1009 except Exception: 

1010 logger.opt(exception=True).warning( 

1011 'Discover: related artists failed for {!r}', name 

1012 ) 

1013 return key, [], True 

1014 return key, related, failed 

1015 

1016 with ThreadPoolExecutor(max_workers=_WORKERS) as pool: 

1017 results = list(pool.map(fetch, seeds)) 

1018 

1019 related = {key: rows for key, rows, _failed in results} 

1020 exclude = {artist_key(a.get('name')) for a in library} 

1021 exclude |= store.blocked_keys(user_id) 

1022 exclude.discard('') 

1023 return { 

1024 'artists': rank_candidates(seeds, related, exclude, limit=limit), 

1025 'seeds': [name for _key, name, _weight in seeds], 

1026 'partial': any(failed for _key, _rows, failed in results), 

1027 } 

1028 

1029 

1030def _search_for( 

1031 store: DiscoverStore, 

1032 name: str, 

1033 *, 

1034 search: Callable[[str], dict[str, Any]], 

1035 now: datetime, 

1036) -> tuple[dict[str, Any], bool]: 

1037 """An artist's Spotify search, from the cache while fresh. Returns 

1038 ``(results, failed)``; same failure rules as :func:`_related_for_seed` 

1039 - a failed search falls back to a stale answer and is never saved.""" 

1040 

1041 key = artist_key(name) 

1042 cached = store.cached_search(key) 

1043 if ( 

1044 cached is not None 

1045 and cached['fetched_at'] is not None 

1046 and now - cached['fetched_at'] < RELATED_TTL 

1047 ): 

1048 return cached['results'], False 

1049 try: 

1050 results = search(name) 

1051 except ValueError: 

1052 return (cached['results'] if cached else {}), True 

1053 store.save_search(key, results, when=now) 

1054 return results, False 

1055 

1056 

1057def collections( 

1058 store: DiscoverStore, 

1059 library: list[dict[str, Any]], 

1060 owned_albums: Iterable[dict[str, Any]] = (), 

1061 owned_playlists: Iterable[str] = (), 

1062 *, 

1063 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id, 

1064 fetch_related: Callable[[str, int], Related] = deezer.related_artists, 

1065 search: Callable[[str], dict[str, Any]] = spotify.search, 

1066 now: Optional[datetime] = None, 

1067) -> dict[str, Any]: 

1068 """Albums and playlists for a library (see :func:`pick_collections`). 

1069 

1070 *library* is what :func:`recommendations` takes (its answer comes from 

1071 the same cache, so this costs no extra Deezer requests once the 

1072 artists are shown); *owned_albums* ``{artist, title}`` per library 

1073 album, *owned_playlists* the Spotify ids of downloaded playlists. 

1074 ``partial`` is ``True`` when a search failed for some artist. 

1075 """ 

1076 

1077 now = now or _now() 

1078 library = [a for a in library if isinstance(a, dict)] 

1079 ranked = recommendations( 

1080 store, 

1081 library, 

1082 lookup_id=lookup_id, 

1083 fetch_related=fetch_related, 

1084 now=now, 

1085 ) 

1086 suggested = ranked['artists'][:COLLECTION_SUGGESTED] 

1087 seeds = ranked['seeds'][:COLLECTION_SEEDS] 

1088 names = list(dict.fromkeys([*seeds, *(a['name'] for a in suggested)])) 

1089 

1090 def fetch(name: str) -> tuple[str, dict[str, Any], bool]: 

1091 try: 

1092 results, failed = _search_for(store, name, search=search, now=now) 

1093 except Exception: 

1094 logger.opt(exception=True).warning( 

1095 'Discover: Spotify search failed for {!r}', name 

1096 ) 

1097 return name, {}, True 

1098 return name, results, failed 

1099 

1100 with ThreadPoolExecutor(max_workers=_WORKERS) as pool: 

1101 results = list(pool.map(fetch, names)) 

1102 

1103 picked = pick_collections( 

1104 suggested, 

1105 seeds, 

1106 {artist_key(name): found for name, found, _failed in results}, 

1107 { 

1108 album_key(a.get('artist'), a.get('title')) 

1109 for a in owned_albums 

1110 if isinstance(a, dict) 

1111 }, 

1112 {str(pid) for pid in owned_playlists if pid}, 

1113 ) 

1114 picked['partial'] = ranked['partial'] or any( 

1115 failed for _name, _found, failed in results 

1116 ) 

1117 return picked 

1118 

1119 

1120# ── Deezer first, then Spotify matched to Deezer ───────────────────────── 

1121# 

1122# The web page's album and playlist shelves. Deezer's own picks come first 

1123# (deezer_collections: quick, and every album opens in the Finder); what 

1124# Spotify picks is added after (spotify_collections), each album matched 

1125# back to Deezer by name when Deezer has it. Playlists aren't matched - a 

1126# "100% <artist>" (Deezer's editors) and a "This Is <artist>" (Spotify's) 

1127# are different lists - so each service keeps its own shelf. 

1128# 

1129# collections() above, Spotify only, is kept as it is: other clients may 

1130# rely on it. 

1131 

1132 

1133def _release_rank(album: dict[str, Any]) -> tuple[bool, int]: 

1134 """Sort key for an artist's Deezer releases: full albums first (as 

1135 :func:`_albums_by` does on Spotify), then the most fans.""" 

1136 

1137 kind = str(album.get('release_type') or '').lower() 

1138 return kind != 'album', -int(album.get('fans') or 0) 

1139 

1140 

1141def _deezer_album_item( 

1142 album: dict[str, Any], artist: str, reason: str, because: list[str] 

1143) -> dict[str, Any]: 

1144 album_id = str(album['album_id']) 

1145 return { 

1146 'name': album['name'], 

1147 'artist': artist, 

1148 'year': album.get('year') or '', 

1149 'cover_url': album.get('cover_url') or '', 

1150 'source': 'deezer', 

1151 'deezer_album_id': album_id, 

1152 'deezer_artist_id': str(album.get('artist_id') or ''), 

1153 'url': album.get('url') or f'https://www.deezer.com/album/{album_id}', 

1154 'key': album_key(artist, album['name']), 

1155 'reason': reason, 

1156 'because': because, 

1157 } 

1158 

1159 

1160def _deezer_playlist_item( 

1161 playlist: dict[str, Any], artist: str 

1162) -> dict[str, Any]: 

1163 return { 

1164 'name': playlist['name'], 

1165 'owner': playlist.get('owner') or '', 

1166 'cover_url': playlist.get('cover_url') or '', 

1167 'source': 'deezer', 

1168 'deezer_playlist_id': str(playlist['playlist_id']), 

1169 'url': playlist['url'], 

1170 'reason': 'essentials', 

1171 'artist': artist, 

1172 } 

1173 

1174 

1175def pick_deezer_collections( 

1176 suggested: list[dict[str, Any]], 

1177 seeds: list[str], 

1178 discographies: dict[str, list[dict[str, Any]]], 

1179 playlists: dict[str, Optional[dict[str, Any]]], 

1180 owned_albums: set[str], 

1181 owned_playlists: set[str], 

1182) -> dict[str, Any]: 

1183 """The same shelves as :func:`pick_collections`, from Deezer. 

1184 

1185 *discographies* and *playlists* are keyed by :func:`artist_key`: each 

1186 artist's :func:`downtify.deezer.artist_discography` and 

1187 :func:`downtify.deezer.editorial_playlist`. *owned_albums* holds 

1188 :func:`album_key` values, *owned_playlists* the 

1189 :func:`~downtify.file_naming.file_name_key` of every library playlist 

1190 name - a downloaded Deezer playlist keeps its title as its name. 

1191 Returns ``{albums, more_albums, playlists}``: 

1192 

1193 * ``albums``: each suggested artist's best album not owned - a full 

1194 album before a single or EP, the most fans first - ``reason: 

1195 'similar'``. 

1196 * ``more_albums``: up to :data:`MORE_FROM_PER_SEED` per seed, same 

1197 order, ``reason: 'more_from'``. 

1198 * ``playlists``: the suggested artists' "100%" playlists, 

1199 ``reason: 'essentials'``. 

1200 """ 

1201 

1202 albums: list[dict[str, Any]] = [] 

1203 more: list[dict[str, Any]] = [] 

1204 lists: list[dict[str, Any]] = [] 

1205 seen: set[str] = set(owned_albums) 

1206 

1207 def take(album, artist, reason, because, into) -> bool: 

1208 key = album_key(artist, album['name']) 

1209 if key in seen: 

1210 return False 

1211 seen.add(key) 

1212 into.append(_deezer_album_item(album, artist, reason, because)) 

1213 return True 

1214 

1215 for seed in seeds: 

1216 taken = 0 

1217 releases = discographies.get(artist_key(seed)) or [] 

1218 for album in sorted(releases, key=_release_rank): 

1219 if taken >= MORE_FROM_PER_SEED: 

1220 break 

1221 if take(album, seed, 'more_from', [seed], more): 

1222 taken += 1 

1223 

1224 for artist in suggested: 

1225 name = artist['name'] 

1226 because = artist.get('because') or [] 

1227 releases = discographies.get(artist_key(name)) or [] 

1228 for album in sorted(releases, key=_release_rank): 

1229 if take(album, name, 'similar', because, albums): 

1230 break 

1231 playlist = playlists.get(artist_key(name)) 

1232 if playlist and file_name_key(playlist['name']) not in owned_playlists: 

1233 lists.append(_deezer_playlist_item(playlist, name)) 

1234 

1235 return {'albums': albums, 'more_albums': more, 'playlists': lists} 

1236 

1237 

1238def _deezer_part( 

1239 store: DiscoverStore, 

1240 key: str, 

1241 part: str, 

1242 fetch: Callable[[], Any], 

1243 *, 

1244 now: datetime, 

1245) -> tuple[Any, bool]: 

1246 """An artist's Deezer *part* from the cache while fresh, else 

1247 *fetch*. ``(value, failed)``, with the same rules as 

1248 :func:`_related_for_seed`: a failure falls back to a stale answer and 

1249 is never saved; "Deezer has none" (``None``) is saved like any answer. 

1250 """ 

1251 

1252 cached = store.cached_deezer(key, part) 

1253 if ( 

1254 cached is not None 

1255 and cached['fetched_at'] is not None 

1256 and now - cached['fetched_at'] < RELATED_TTL 

1257 ): 

1258 return cached['value'], False 

1259 try: 

1260 value = fetch() 

1261 except ValueError: 

1262 return (cached['value'] if cached else None), True 

1263 store.save_deezer(key, part, value, when=now) 

1264 return value, False 

1265 

1266 

1267def deezer_collections( 

1268 store: DiscoverStore, 

1269 library: list[dict[str, Any]], 

1270 owned_albums: Iterable[dict[str, Any]] = (), 

1271 owned_playlists: Iterable[str] = (), 

1272 *, 

1273 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id, 

1274 fetch_related: Callable[[str, int], Related] = deezer.related_artists, 

1275 fetch_albums: Callable[ 

1276 [str], list[dict[str, Any]] 

1277 ] = deezer.artist_discography, 

1278 fetch_playlist: Callable[ 

1279 [str], Optional[dict[str, Any]] 

1280 ] = deezer.editorial_playlist, 

1281 now: Optional[datetime] = None, 

1282) -> dict[str, Any]: 

1283 """Albums and playlists for a library from Deezer alone (see 

1284 :func:`pick_deezer_collections`) - the page's first, quick answer. 

1285 

1286 *library* and *owned_albums* are what :func:`collections` takes; 

1287 *owned_playlists* are library playlist names. Built on the same ranked 

1288 artists (cached), so it only asks Deezer for discographies and 

1289 playlists: one each per artist, cached for :data:`RELATED_TTL`. 

1290 ``partial`` is ``True`` when some of those couldn't be fetched. 

1291 """ 

1292 

1293 now = now or _now() 

1294 library = [a for a in library if isinstance(a, dict)] 

1295 ranked = recommendations( 

1296 store, 

1297 library, 

1298 lookup_id=lookup_id, 

1299 fetch_related=fetch_related, 

1300 now=now, 

1301 ) 

1302 suggested = ranked['artists'][:COLLECTION_SUGGESTED] 

1303 seeds = ranked['seeds'][:COLLECTION_SEEDS] 

1304 

1305 # A suggestion carries its own Deezer id; a seed's was saved with its 

1306 # related artists. 

1307 names: dict[str, str] = {} 

1308 ids: dict[str, str] = {} 

1309 for seed in seeds: 

1310 key = artist_key(seed) 

1311 names[key] = seed 

1312 ids[key] = (store.cached_related(key) or {}).get('deezer_id') or '' 

1313 for artist in suggested: 

1314 key = artist_key(artist['name']) 

1315 names.setdefault(key, artist['name']) 

1316 ids.setdefault(key, str(artist.get('deezer_id') or '')) 

1317 

1318 jobs = [(key, 'albums') for key, deezer_id in ids.items() if deezer_id] 

1319 jobs += [(artist_key(a['name']), 'playlist') for a in suggested] 

1320 

1321 def fetcher(key: str, part: str) -> Callable[[], Any]: 

1322 if part == 'albums': 

1323 return lambda: fetch_albums(ids[key]) 

1324 return lambda: fetch_playlist(names[key]) 

1325 

1326 def run(job: tuple[str, str]) -> tuple[str, str, Any, bool]: 

1327 key, part = job 

1328 try: 

1329 value, failed = _deezer_part( 

1330 store, key, part, fetcher(key, part), now=now 

1331 ) 

1332 except Exception: 

1333 logger.opt(exception=True).warning( 

1334 'Discover: Deezer {} failed for {!r}', part, names[key] 

1335 ) 

1336 return key, part, None, True 

1337 return key, part, value, failed 

1338 

1339 with ThreadPoolExecutor(max_workers=_THROTTLED_WORKERS) as pool: 

1340 results = list(pool.map(run, jobs)) 

1341 

1342 discographies = { 

1343 key: value or [] for key, part, value, _ in results if part == 'albums' 

1344 } 

1345 playlists = { 

1346 key: value for key, part, value, _ in results if part == 'playlist' 

1347 } 

1348 picked = pick_deezer_collections( 

1349 suggested, 

1350 seeds, 

1351 discographies, 

1352 playlists, 

1353 { 

1354 album_key(a.get('artist'), a.get('title')) 

1355 for a in owned_albums 

1356 if isinstance(a, dict) 

1357 }, 

1358 {file_name_key(str(name)) for name in owned_playlists if name}, 

1359 ) 

1360 picked['partial'] = ranked['partial'] or any( 

1361 failed for _key, _part, _value, failed in results 

1362 ) 

1363 return picked 

1364 

1365 

1366def _match_album( 

1367 store: DiscoverStore, 

1368 item: dict[str, Any], 

1369 *, 

1370 find: Callable[[str, str], list[dict[str, Any]]], 

1371 now: datetime, 

1372) -> tuple[Optional[dict[str, Any]], bool]: 

1373 """A Spotify album *item*'s Deezer album, or ``None`` when Deezer has 

1374 none: ``(match, failed)``. 

1375 

1376 A row counts only when it's the same album by :func:`album_key` (same 

1377 artist, same title once edition suffixes are dropped) - never "the 

1378 closest". Matches are cached by Spotify id, a found one for 

1379 :data:`MATCH_TTL_FOUND`, a missing one for :data:`MATCH_TTL_MISSING`; 

1380 a failure falls back to a stale match and is never saved. 

1381 """ 

1382 

1383 spotify_id = str(item['spotify_id']) 

1384 cached = store.cached_album_match(spotify_id) 

1385 if cached is not None and cached['fetched_at'] is not None: 

1386 found = bool(cached['deezer_album_id']) 

1387 ttl = MATCH_TTL_FOUND if found else MATCH_TTL_MISSING 

1388 if now - cached['fetched_at'] < ttl: 

1389 return (cached if found else None), False 

1390 try: 

1391 rows = find(item['artist'], item['name']) 

1392 except ValueError: 

1393 stale = cached if cached and cached['deezer_album_id'] else None 

1394 return stale, True 

1395 wanted = album_key(item['artist'], item['name']) 

1396 row = next( 

1397 ( 

1398 row 

1399 for row in rows 

1400 if album_key(row.get('artist'), row.get('name')) == wanted 

1401 ), 

1402 None, 

1403 ) 

1404 store.save_album_match(spotify_id, row, when=now) 

1405 if row is None: 

1406 return None, False 

1407 return { 

1408 'deezer_album_id': str(row['album_id']), 

1409 'deezer_artist_id': str(row.get('artist_id') or ''), 

1410 'cover_url': row.get('cover_url') or '', 

1411 }, False 

1412 

1413 

1414def _matched_album_item( 

1415 item: dict[str, Any], match: Optional[dict[str, Any]] 

1416) -> dict[str, Any]: 

1417 """A Spotify album *item* as Deezer's (opens in the Finder), or as 

1418 Spotify's when Deezer has no match (opens on its Spotify page).""" 

1419 

1420 key = album_key(item['artist'], item['name']) 

1421 if match is None: 

1422 return {**item, 'source': 'spotify', 'key': key} 

1423 album_id = match['deezer_album_id'] 

1424 return { 

1425 'name': item['name'], 

1426 'artist': item['artist'], 

1427 'year': item.get('year') or '', 

1428 'cover_url': match.get('cover_url') or item.get('cover_url') or '', 

1429 'source': 'deezer', 

1430 'deezer_album_id': album_id, 

1431 'deezer_artist_id': match.get('deezer_artist_id') or '', 

1432 'url': f'https://www.deezer.com/album/{album_id}', 

1433 'key': key, 

1434 'reason': item['reason'], 

1435 'because': item['because'], 

1436 } 

1437 

1438 

1439def spotify_collections( 

1440 store: DiscoverStore, 

1441 library: list[dict[str, Any]], 

1442 owned_albums: Iterable[dict[str, Any]] = (), 

1443 owned_playlists: Iterable[str] = (), 

1444 shown: Iterable[str] = (), 

1445 *, 

1446 lookup_id: Callable[[str], Optional[str]] = deezer.lookup_artist_id, 

1447 fetch_related: Callable[[str, int], Related] = deezer.related_artists, 

1448 search: Callable[[str], dict[str, Any]] = spotify.search, 

1449 find_album: Callable[ 

1450 [str, str], list[dict[str, Any]] 

1451 ] = deezer.search_albums_by, 

1452 now: Optional[datetime] = None, 

1453) -> dict[str, Any]: 

1454 """What Spotify adds to :func:`deezer_collections` - the page's 

1455 second, slower answer. 

1456 

1457 Spotify's own picks (:func:`collections`, its searches cached), less 

1458 the albums already *shown* (their :func:`album_key` values) - so an 

1459 album both services pick costs no match - each matched to Deezer (see 

1460 :func:`_match_album`). *owned_playlists* are Spotify playlist ids, as 

1461 for :func:`collections`. Returns ``{albums, more_albums, playlists, 

1462 partial}``: albums with ``source`` ``'deezer'`` (matched) or 

1463 ``'spotify'`` (no match), and only the "This Is <artist>" playlists - 

1464 Spotify's "Radio" mixes aren't offered here. 

1465 """ 

1466 

1467 now = now or _now() 

1468 picked = collections( 

1469 store, 

1470 library, 

1471 owned_albums, 

1472 owned_playlists, 

1473 lookup_id=lookup_id, 

1474 fetch_related=fetch_related, 

1475 search=search, 

1476 now=now, 

1477 ) 

1478 skip = {str(key) for key in shown if key} 

1479 jobs = [ 

1480 (shelf, item) 

1481 for shelf in ('albums', 'more_albums') 

1482 for item in picked[shelf] 

1483 if album_key(item['artist'], item['name']) not in skip 

1484 ] 

1485 

1486 def run(job: tuple[str, dict[str, Any]]) -> tuple[str, dict, bool]: 

1487 shelf, item = job 

1488 try: 

1489 match, failed = _match_album(store, item, find=find_album, now=now) 

1490 except Exception: 

1491 logger.opt(exception=True).warning( 

1492 'Discover: matching {!r} to Deezer failed', item['name'] 

1493 ) 

1494 match, failed = None, True 

1495 return shelf, _matched_album_item(item, match), failed 

1496 

1497 with ThreadPoolExecutor(max_workers=_THROTTLED_WORKERS) as pool: 

1498 results = list(pool.map(run, jobs)) 

1499 

1500 return { 

1501 'albums': [item for shelf, item, _ in results if shelf == 'albums'], 

1502 'more_albums': [ 

1503 item for shelf, item, _ in results if shelf == 'more_albums' 

1504 ], 

1505 'playlists': [ 

1506 {**playlist, 'source': 'spotify'} 

1507 for playlist in picked['playlists'] 

1508 if playlist['reason'] == 'this_is' 

1509 ], 

1510 'partial': picked['partial'] 

1511 or any(failed for _shelf, _item, failed in results), 

1512 }