Coverage for downtify/spotify.py: 85%
897 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Resolve Spotify URLs without using the official Spotify Web API.
3Reads the public ``open.spotify.com/embed`` pages, which expose the same
4data the Spotify embedded player consumes. No client credentials, no
5authentication and no premium account are required.
6"""
8from __future__ import annotations
10import json
11import re
12import threading
13import time
14from concurrent.futures import ThreadPoolExecutor
15from typing import Any, Optional
16from urllib.parse import urlsplit
18import httpx
19from loguru import logger
21from .album_artist import album_artist_fields
22from .file_naming import file_name_key
23from .telemetry import json_log_blob, redact_sensitive_mapping
25SPOTIFY_URL_RE = re.compile(
26 r'(?:https?://)?(?:open\.)?spotify\.com/'
27 r'(?:intl-[a-z]{2}/)?'
28 r'(?P<type>track|album|playlist|artist|episode|show)/'
29 r'(?P<id>[A-Za-z0-9]+)'
30)
32_USER_AGENT = (
33 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
34 '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
35)
37# Spotify often serves a tiny HTML shell (no ``music:release_date`` meta) to
38# full Chrome-like user agents while still sending the static OG-rich document
39# for a minimal ``Mozilla/5.0`` probe — reuse that behaviour here only.
40_ALBUM_OPEN_PAGE_UA = 'Mozilla/5.0'
42# The track embed carries no album at all (no ``album`` key, no album
43# uri), but the canonical open.spotify.com/track/{id} HTML names it in
44# its Open Graph music tags, with the track's position on it.
45_TRACK_OPEN_PAGE_ALBUM = re.compile(
46 r'<meta\s+name=["\']music:album["\']\s+content=["\']'
47 r'https://open\.spotify\.com/album/([A-Za-z0-9]{22})',
48 re.I,
49)
50_TRACK_OPEN_PAGE_ALBUM_TRACK = re.compile(
51 r'<meta\s+name=["\']music:album:track["\']\s+content=["\'](\d+)["\']',
52 re.I,
53)
55# Embed album payloads often set ``releaseDate`` to ``null``. The canonical
56# open.spotify.com/album/{id} HTML still publishes ``music:release_date``.
57_ALBUM_OPEN_PAGE_META_RELEASE = re.compile(
58 r'<meta\s+name=["\']music:release_date["\']\s+content=["\']([^"\']+)["\']',
59 re.I,
60)
61_ALBUM_OPEN_PAGE_DATE_PUBLISHED = re.compile(
62 r'"datePublished"\s*:\s*"([^"]+)"'
63)
66def parse_spotify_url(url: str) -> Optional[tuple[str, str]]:
67 """Return ``(type, id)`` for a Spotify URL/URI or ``None`` if not one."""
69 if not url:
70 return None
71 if url.startswith('spotify:'):
72 try:
73 _, kind, sid = url.split(':', 2)
74 except ValueError:
75 return None
76 return kind, sid
77 match = SPOTIFY_URL_RE.search(url)
78 if not match:
79 return None
80 return match.group('type'), match.group('id')
83_EMBED_TRANSIENT_HTTP = frozenset({429, 502, 503, 504})
84_EMBED_FETCH_ATTEMPTS = 3
87def _fetch_embed_json(kind: str, spotify_id: str) -> dict[str, Any]:
88 url = f'https://open.spotify.com/embed/{kind}/{spotify_id}'
89 response: httpx.Response | None = None
90 for attempt in range(_EMBED_FETCH_ATTEMPTS):
91 response = httpx.get(
92 url,
93 headers={
94 'User-Agent': _USER_AGENT,
95 'Accept-Language': 'en-US,en;q=0.9',
96 },
97 timeout=15,
98 )
99 try:
100 response.raise_for_status()
101 except httpx.HTTPStatusError as exc:
102 status = exc.response.status_code
103 if (
104 status in _EMBED_TRANSIENT_HTTP
105 and attempt < _EMBED_FETCH_ATTEMPTS - 1
106 ):
107 logger.debug(
108 'Spotify embed HTTP {} for {}/{}, retry {}/{}',
109 status,
110 kind,
111 spotify_id,
112 attempt + 1,
113 _EMBED_FETCH_ATTEMPTS,
114 )
115 time.sleep(1.0 * (attempt + 1))
116 continue
117 raise
118 break
119 if response is None:
120 raise RuntimeError('Spotify embed fetch did not run')
121 match = re.search(
122 r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>',
123 response.text,
124 re.DOTALL,
125 )
126 if not match:
127 raise ValueError('Spotify embed payload not found')
128 data = json.loads(match.group(1))
129 preview = json_log_blob(redact_sensitive_mapping(data))
130 logger.debug(
131 'Spotify embed NEXT_DATA ({}/{}, {} chars redacted preview): {}',
132 kind,
133 spotify_id,
134 len(preview),
135 preview,
136 )
137 _log_spotify_embed_entity_summary(kind, spotify_id, data)
138 return data
141def _log_spotify_embed_entity_summary(
142 kind: str,
143 spotify_id: str,
144 payload: dict[str, Any],
145) -> None:
146 """INFO line so release/track shape is visible without DEBUG."""
148 try:
149 ent = _entity_from(payload)
150 except ValueError as ex:
151 logger.warning(
152 'Spotify embed: no usable entity kind={} id={}: {}',
153 kind,
154 spotify_id,
155 ex,
156 )
157 return
158 track_list = ent.get('trackList') or []
159 alt_items = []
160 nested = ent.get('tracks')
161 if isinstance(nested, dict):
162 alt_items = nested.get('items') or []
163 rd_raw = ent.get('releaseDate')
164 if isinstance(rd_raw, dict):
165 rd_summary = 'dict isoString={!r}'.format(rd_raw.get('isoString'))
166 elif rd_raw is None:
167 rd_summary = 'null'
168 elif isinstance(rd_raw, str):
169 rd_summary = f'str:{rd_raw[:32]!r}'
170 else:
171 rd_summary = type(rd_raw).__name__
172 logger.info(
173 (
174 'Spotify embed OK: {} {} title={!r} type={!r} '
175 'trackList_len={} tracks_items_len={} releaseDate={}'
176 ),
177 kind,
178 spotify_id,
179 ent.get('name') or ent.get('title'),
180 ent.get('type'),
181 len(track_list),
182 len(alt_items),
183 rd_summary,
184 )
187def _entity_from(payload: dict[str, Any]) -> dict[str, Any]:
188 page_props = payload.get('props', {}).get('pageProps', {}) or {}
189 candidates: list[Any] = [
190 page_props.get('state', {}).get('data', {}).get('entity')
191 if isinstance(page_props.get('state'), dict)
192 else None,
193 page_props.get('entity'),
194 page_props.get('data', {}).get('entity')
195 if isinstance(page_props.get('data'), dict)
196 else None,
197 ]
198 for candidate in candidates:
199 if isinstance(candidate, dict):
200 return candidate
201 raise ValueError('Spotify entity not found in embed payload')
204def fetch_embed_entity(kind: str, spotify_id: str) -> dict[str, Any]:
205 """The raw embed entity for a Spotify URL of any ``kind``.
207 Tracks/albums/playlists are read through the richer helpers below
208 (:func:`playlist_info_and_tracks` and friends); this is for kinds
209 those don't model, namely ``show`` and ``episode`` — see
210 :mod:`downtify.podcasts`. The embed always resolves a show or
211 episode link to an ``entity`` of type ``"episode"`` (the show's most
212 recent one for a ``/show/`` link), whose ``subtitle`` is the show's
213 name — there is no separate "show" entity shape to parse.
214 """
216 return _entity_from(_fetch_embed_json(kind, spotify_id))
219def _embed_row_track(item: dict[str, Any]) -> Optional[dict[str, Any]]:
220 """Track dict for a playlist/album embed row.
222 Rows often nest the payload under ``track`` while the artist line lives in
223 a sibling ``subtitle`` field on the wrapper — copy it onto the track so
224 :func:`_artist_names` can see it.
225 """
227 inner = item.get('track')
228 if isinstance(inner, dict):
229 sub = item.get('subtitle')
230 if isinstance(sub, str) and sub.strip() and not inner.get('subtitle'):
231 return {**inner, 'subtitle': sub}
232 return inner
233 return item if isinstance(item, dict) else None
236#: Spotify publishes an album cover at 640, 300 and 64 px under one image
237#: id, the size picked by a fixed prefix - so any of them can be pointed
238#: at the 640 px one without another request.
239_SMALL_COVER_PREFIXES = ('ab67616d00001e02', 'ab67616d00004851')
240LARGEST_COVER_PREFIX = 'ab67616d0000b273'
241#: Spotify's image hosts: the older i.scdn.co, and the
242#: image-cdn-<region>.spotifycdn.com ones its embeds now return.
243_IMAGE_URL = re.compile(
244 r'^https?://(?:i\.scdn\.co|image-cdn-[a-z0-9]+\.spotifycdn\.com)/image/'
245)
248def largest_cover_url(url: str) -> str:
249 """*url* (a Spotify album cover) at the largest size Spotify publishes;
250 anything else comes back unchanged."""
252 if not url or not _IMAGE_URL.match(url):
253 return url
254 for prefix in _SMALL_COVER_PREFIXES:
255 if prefix in url:
256 return url.replace(prefix, LARGEST_COVER_PREFIX)
257 return url
260def _largest_image(sources: list[dict[str, Any]]) -> str:
261 """The largest of *sources* - an album cover always at its 640 px size
262 (:func:`largest_cover_url`), even when the list offers only smaller
263 ones: an embed sometimes lists just the 300 and 64 px variants."""
265 if not sources:
266 return ''
267 sized = [s for s in sources if isinstance(s, dict) and s.get('url')]
268 if not sized:
269 return ''
270 # Track/album/playlist covers use ``width``; an artist's
271 # ``visualIdentity.image`` entries use ``maxWidth`` instead - check
272 # both so an artist photo actually sorts by size instead of staying
273 # in whatever order the embed happened to list it.
274 sized.sort(
275 key=lambda s: int(s.get('width') or s.get('maxWidth') or 0),
276 reverse=True,
277 )
278 return largest_cover_url(sized[0]['url'])
281def _cover_url(entity: dict[str, Any]) -> str:
282 candidates: list[dict[str, Any]] = []
283 cover_art = entity.get('coverArt') or {}
284 if isinstance(cover_art, dict):
285 candidates += cover_art.get('sources') or []
286 visual = entity.get('visualIdentity') or {}
287 if isinstance(visual, dict):
288 candidates += visual.get('image') or []
289 album = entity.get('album') or {}
290 if isinstance(album, dict):
291 nested = album.get('coverArt') or {}
292 if isinstance(nested, dict):
293 candidates += nested.get('sources') or []
294 images = album.get('images')
295 if isinstance(images, list):
296 candidates += images
297 return _largest_image(candidates)
300def _artist_names(entity: dict[str, Any]) -> list[str]:
301 raw = entity.get('artists') or []
302 names: list[str] = []
303 if isinstance(raw, list):
304 for item in raw:
305 if isinstance(item, str):
306 if item.strip():
307 names.append(item.strip())
308 elif isinstance(item, dict):
309 name = item.get('name')
310 if name:
311 names.append(name)
312 if names:
313 return names
314 return [
315 d['name']
316 for d in _artists_from_subtitle(entity.get('subtitle'))
317 if d.get('name')
318 ]
321def _artists_unverified(entity: dict[str, Any]) -> bool:
322 """Whether :func:`_artist_names` had to read *entity*'s artists from an
323 ambiguous ``subtitle`` (see :func:`_ambiguous_subtitle`) rather than a
324 structured list - so they're re-read from the per-track embed before
325 the track is downloaded (:func:`enrich_track_from_spotify_if_sparse`).
326 """
328 raw = entity.get('artists')
329 if isinstance(raw, list) and any(
330 (isinstance(a, str) and a.strip())
331 or (isinstance(a, dict) and a.get('name'))
332 for a in raw
333 ):
334 return False
335 return _ambiguous_subtitle(entity.get('subtitle'))
338def _album_artist_from_subtitle(subtitle: Any) -> dict[str, Any]:
339 """``album_artist`` (and ``compilation``) fields from an album embed's
340 ``subtitle`` - the album's own primary artist (``'Various Artists'``
341 for a compilation), not its tracks' artists."""
343 names = _artists_from_subtitle(subtitle)
344 return album_artist_fields(names[0]['name'] if names else '')
347def _normalize_release_date_text(raw: Any) -> str:
348 """Normalize Spotify ``isoString`` (or similar) to ``YYYY-MM-DD`` or year."""
350 if not isinstance(raw, str):
351 return ''
352 text = raw.strip()
353 if not text:
354 return ''
355 if 'T' in text:
356 text = text.split('T', 1)[0].strip()
357 if (
358 len(text) >= 10
359 and text[4:5] == '-'
360 and text[7:8] == '-'
361 and text[:4].isdigit()
362 and text[5:7].isdigit()
363 and text[8:10].isdigit()
364 ):
365 return text[:10]
366 # Month precision strings like ``2024-06`` → tag-friendly first-of-month.
367 if (
368 len(text) >= 7
369 and text[4:5] == '-'
370 and text[:4].isdigit()
371 and text[5:7].isdigit()
372 ):
373 return f'{text[:4]}-{text[5:7]}-01'
374 if len(text) >= 4 and text[:4].isdigit():
375 return text[:4]
376 return text
379def _calendar_component(raw: Any) -> Optional[int]:
380 """Coerce embed/GraphQL month or day ints (reject bool)."""
382 if raw is None or isinstance(raw, bool):
383 return None
384 if isinstance(raw, int):
385 return raw
386 if isinstance(raw, float) and raw.is_integer():
387 return int(raw)
388 if isinstance(raw, str) and raw.strip().isdigit():
389 return int(raw.strip())
390 return None
393def _coerce_four_digit_year(raw: Any) -> Optional[int]:
394 if isinstance(raw, bool):
395 return None
396 if isinstance(raw, int) and 1000 <= raw <= 9999:
397 return raw
398 if isinstance(raw, float) and raw.is_integer():
399 y = int(raw)
400 return y if 1000 <= y <= 9999 else None
401 if isinstance(raw, str):
402 ys = raw.strip()
403 if ys.isdigit() and len(ys) == 4:
404 y = int(ys)
405 return y if 1000 <= y <= 9999 else None
406 return None
409def _release_date_raw_from_field(release: Any, *, _depth: int = 0) -> str:
410 """Extract a tagging-friendly date string from Spotify embed/GraphQL fields.
412 Older payloads expose ``releaseDate.isoString``. Many catalogue rows omit
413 ``isoString`` and only send ``year`` (optionally ``month``/``day`` and
414 ``precision``); without handling those we end up tagging files with no
415 release date even though Spotify showed a year on the canvas.
416 """
418 if _depth > 6:
419 return ''
420 if isinstance(release, dict):
421 cand = release.get('isoString')
422 if isinstance(cand, str) and cand.strip():
423 out = _normalize_release_date_text(cand.strip())
424 if out:
425 return out
427 nested = release.get('date')
428 nested_out = (
429 _release_date_raw_from_field(nested, _depth=_depth + 1)
430 if nested is not None
431 else ''
432 )
433 if nested_out:
434 return nested_out
436 y_part = _coerce_four_digit_year(release.get('year'))
437 ys = str(y_part) if y_part is not None else ''
438 mi = _calendar_component(release.get('month'))
439 di = _calendar_component(release.get('day'))
440 pv = release.get('precision')
441 precision = pv.casefold() if isinstance(pv, str) else ''
443 if precision == 'year' and ys:
444 return ys
446 if ys and mi is not None and 1 <= mi <= 12:
447 mm = f'{mi:02d}'
448 if di is not None and 1 <= di <= 31:
449 return f'{ys}-{mm}-{di:02d}'
450 if precision in {'month', 'day'}:
451 return f'{ys}-{mm}-01'
453 if ys:
454 return ys
456 return ''
457 if isinstance(release, str):
458 return _normalize_release_date_text(release)
459 return ''
462def _album_release_date_from_open_page(album_id: str) -> str:
463 """Parse release date from the public album HTML when embed omits it."""
465 if not album_id or not re.fullmatch(r'[A-Za-z0-9]+', album_id):
466 return ''
467 try:
468 resp = httpx.get(
469 f'https://open.spotify.com/album/{album_id}',
470 headers={
471 'User-Agent': _ALBUM_OPEN_PAGE_UA,
472 'Accept-Language': 'en-US,en;q=0.9',
473 },
474 timeout=15,
475 )
476 resp.raise_for_status()
477 except Exception:
478 logger.opt(exception=True).debug(
479 'Spotify open album page fetch failed for release_date id={}',
480 album_id,
481 )
482 return ''
484 html = resp.text
485 m = _ALBUM_OPEN_PAGE_META_RELEASE.search(html)
486 if not m:
487 m = _ALBUM_OPEN_PAGE_DATE_PUBLISHED.search(html)
488 if not m:
489 return ''
490 normalized = _normalize_release_date_text(m.group(1).strip())
491 if normalized:
492 logger.debug(
493 'Spotify album {} release_date from open page: {!r}',
494 album_id,
495 normalized,
496 )
497 return normalized
500def _release_date_str(entity: dict[str, Any]) -> str:
501 """Prefer full calendar date when the embed exposes ``isoString``."""
503 rd = _release_date_raw_from_field(entity.get('releaseDate'))
504 if rd:
505 return rd
506 album = entity.get('album') or {}
507 if isinstance(album, dict):
508 for key in ('releaseDate', 'date'):
509 rd = _release_date_raw_from_field(album.get(key))
510 if rd:
511 return rd
512 return ''
515def _year_from_release_date(rd: str) -> str:
516 if len(rd) >= 4 and rd[:4].isdigit():
517 return rd[:4]
518 return ''
521def _track_dict(
522 entity: dict[str, Any],
523 *,
524 track_id: str,
525 fallback_album: str = '',
526 fallback_cover: str = '',
527 fallback_release_date: str = '',
528) -> dict[str, Any]:
529 duration_ms = entity.get('duration') or entity.get('duration_ms') or 0
530 album = entity.get('album') or {}
531 album_name = album.get('name', '') if isinstance(album, dict) else ''
532 cover = _cover_url(entity) or fallback_cover
533 names = _artist_names(entity)
534 release_date = _release_date_str(entity)
535 if not release_date:
536 release_date = (fallback_release_date or '').strip()
537 year = _year_from_release_date(release_date)
538 if not year and not release_date:
539 logger.info(
540 'Spotify resolved row has no year/release_date: '
541 'track_id={!r} title={!r} album={!r} raw_releaseDate={!r}',
542 track_id,
543 entity.get('name') or entity.get('title'),
544 album_name or fallback_album,
545 entity.get('releaseDate'),
546 )
547 row: dict[str, Any] = {
548 'song_id': track_id,
549 'name': entity.get('name') or entity.get('title') or '',
550 'artists': names,
551 'artist': ', '.join(names),
552 'album_name': album_name or fallback_album,
553 'cover_url': cover,
554 'duration': int(int(duration_ms) / 1000) if duration_ms else 0,
555 'url': f'https://open.spotify.com/track/{track_id}'
556 if track_id
557 else '',
558 'explicit': bool(entity.get('isExplicit') or entity.get('explicit')),
559 'release_date': release_date,
560 'year': year,
561 'source': 'spotify',
562 }
563 if _artists_unverified(entity):
564 row['artists_unverified'] = True
565 raw_tn = entity.get('trackNumber')
566 if raw_tn is None:
567 raw_tn = entity.get('track_number')
568 if raw_tn is not None:
569 try:
570 tn = int(raw_tn)
571 except (TypeError, ValueError):
572 tn = 0
573 if tn > 0:
574 row['track_number'] = tn
575 if isinstance(album, dict):
576 raw_total = album.get('trackCount')
577 if raw_total is None and isinstance(album.get('tracks'), dict):
578 raw_total = album['tracks'].get('total')
579 if raw_total is not None:
580 try:
581 tt = int(raw_total)
582 except (TypeError, ValueError):
583 tt = 0
584 if tt > 0:
585 row['album_track_total'] = tt
586 return row
589def _merge_full_track_metadata(
590 song: dict[str, Any], full: dict[str, Any]
591) -> dict[str, Any]:
592 """Apply per-track embed fields onto a sparse playlist/browse row."""
594 merged = dict(song)
595 for key in (
596 'cover_url',
597 'year',
598 'release_date',
599 'album_name',
600 'artists',
601 'artist',
602 'track_number',
603 'album_track_total',
604 ):
605 value = full.get(key)
606 if value is None or not value or value == []:
607 continue
608 if key in {'track_number', 'album_track_total'}:
609 if merged.get(key) is not None:
610 continue
611 try:
612 iv = int(value)
613 except (TypeError, ValueError):
614 continue
615 if iv <= 0:
616 continue
617 merged[key] = iv
618 continue
619 if key == 'artists':
620 # The row's own artists win, unless they were only read from
621 # an ambiguous subtitle: the per-track embed's list is exact.
622 if merged.get('artists') and not merged.get('artists_unverified'):
623 continue
624 merged['artists'] = value
625 merged['artist'] = full.get('artist') or ', '.join(value)
626 if not full.get('artists_unverified'):
627 merged.pop('artists_unverified', None)
628 continue
629 merged[key] = value
630 if not str(merged.get('album_artist') or '').strip() and full.get(
631 'album_artist'
632 ):
633 merged['album_artist'] = full['album_artist']
634 if full.get('compilation'):
635 merged['compilation'] = True
636 return merged
639def enrich_track_from_spotify_if_sparse(
640 song: dict[str, Any], *, with_album: bool = True
641) -> dict[str, Any]:
642 """Fill missing Spotify tagging fields from the per-track embed.
644 Playlist browse rows often omit ``year``, ``release_date``, and
645 ``track_number``; monitored playlists re-fetch each track the same way.
646 A row with no album (a single track from search) also gets its album
647 looked up (see :func:`track_from_id`'s ``with_album``), so the file
648 isn't left without an album tag - unless *with_album* is off, for a
649 caller that fills the album in some cheaper way.
650 """
652 if song.get('source') != 'spotify':
653 return song
654 track_id = song.get('song_id')
655 if not isinstance(track_id, str) or not re.fullmatch(
656 r'[A-Za-z0-9]{22}', track_id
657 ):
658 return song
659 has_date = bool(
660 str(song.get('release_date') or '').strip()
661 or str(song.get('year') or '').strip()
662 )
663 has_track = song.get('track_number') is not None
664 need_album = with_album and not str(song.get('album_name') or '').strip()
665 unverified = bool(song.get('artists_unverified'))
666 if has_date and has_track and not need_album and not unverified:
667 return song
668 try:
669 full = (
670 track_from_id(track_id, with_album=True)
671 if need_album
672 else track_from_id(track_id)
673 )
674 except Exception:
675 logger.opt(exception=True).debug(
676 'Per-track Spotify enrich failed for {}', track_id
677 )
678 return song
679 return _merge_full_track_metadata(song, full)
682def track_from_id(
683 track_id: str, *, with_album: bool = False
684) -> dict[str, Any]:
685 """A Spotify track as a song dict, from its embed page.
687 The track embed has no album (see :data:`_TRACK_OPEN_PAGE_ALBUM`), so
688 ``album_name`` comes back empty. With *with_album* it is looked up
689 too (:func:`_fill_album_from_open_page`, two more requests) - what a
690 single-track download needs, since nothing else supplies the album
691 tag for it. Callers that already know the album (a playlist or album
692 row being topped up) leave it off.
693 """
695 payload = _fetch_embed_json('track', track_id)
696 entity = _entity_from(payload)
697 song = _track_dict(entity, track_id=track_id)
698 if with_album and not song.get('album_name'):
699 _fill_album_from_open_page(song, track_id)
700 return song
703def _album_ref_from_open_track_page(
704 track_id: str,
705) -> tuple[str, Optional[int]]:
706 """``(album_id, track_number)`` from the public track page's
707 ``music:album`` / ``music:album:track`` meta tags, or ``('', None)``
708 when the page can't be fetched or doesn't name an album."""
710 if not re.fullmatch(r'[A-Za-z0-9]{22}', track_id or ''):
711 return '', None
712 try:
713 resp = httpx.get(
714 f'https://open.spotify.com/track/{track_id}',
715 headers={
716 'User-Agent': _ALBUM_OPEN_PAGE_UA,
717 'Accept-Language': 'en-US,en;q=0.9',
718 },
719 timeout=15,
720 )
721 resp.raise_for_status()
722 except Exception:
723 logger.opt(exception=True).debug(
724 'Spotify open track page fetch failed for album id={}', track_id
725 )
726 return '', None
727 html = resp.text
728 album = _TRACK_OPEN_PAGE_ALBUM.search(html)
729 if not album:
730 return '', None
731 number = _TRACK_OPEN_PAGE_ALBUM_TRACK.search(html)
732 track_number = int(number.group(1)) if number else None
733 return album.group(1), (track_number if track_number else None)
736def _fill_album_from_open_page(song: dict[str, Any], track_id: str) -> None:
737 """Fill *song*'s ``album_name`` - and, when it has none yet, its
738 ``track_number`` / ``album_track_total`` - from the album the public
739 track page names. Leaves *song* as it is on any failure: a missing
740 album tag is better than a failed download."""
742 album_id, track_number = _album_ref_from_open_track_page(track_id)
743 if not album_id:
744 return
745 try:
746 entity = _entity_from(_fetch_embed_json('album', album_id))
747 except Exception:
748 logger.opt(exception=True).debug(
749 'Spotify album embed lookup failed for {}', album_id
750 )
751 return
752 name = str(entity.get('name') or entity.get('title') or '').strip()
753 if not name:
754 return
755 song['album_name'] = name
756 if not song.get('album_artist'):
757 song.update(_album_artist_from_subtitle(entity.get('subtitle')))
758 if track_number and not song.get('track_number'):
759 song['track_number'] = track_number
760 track_items = (
761 entity.get('trackList')
762 or (entity.get('tracks') or {}).get('items')
763 or []
764 )
765 if track_items and not song.get('album_track_total'):
766 song['album_track_total'] = len(track_items)
769def album_tracks_from_id(album_id: str) -> list[dict[str, Any]]:
770 payload = _fetch_embed_json('album', album_id)
771 entity = _entity_from(payload)
772 album_name = entity.get('name') or ''
773 cover = _cover_url(entity)
774 track_items = (
775 entity.get('trackList')
776 or (entity.get('tracks') or {}).get('items')
777 or []
778 )
779 album_track_total = len(track_items)
780 album_release_date = _release_date_str(entity)
781 if not album_release_date:
782 album_release_date = _album_release_date_from_open_page(album_id)
783 # The album's own artist, the same on every track - never worked out
784 # from the tracks' artists (a guest on one track doesn't make the
785 # album a compilation).
786 album_fields = _album_artist_from_subtitle(entity.get('subtitle'))
787 songs: list[dict[str, Any]] = []
788 for tracklist_slot, item in enumerate(track_items, start=1):
789 if not isinstance(item, dict):
790 continue
791 track = _embed_row_track(item)
792 if not isinstance(track, dict):
793 continue
794 track_id = track.get('id') or _id_from_uri(track.get('uri', ''))
795 if not track_id:
796 continue
797 track = dict(track)
798 if not track.get('artists') and not track.get('subtitle'):
799 track['artists'] = (
800 entity.get('artists')
801 or _artists_from_subtitle(entity.get('subtitle'))
802 or []
803 )
804 row = _track_dict(
805 track,
806 track_id=track_id,
807 fallback_album=album_name,
808 fallback_cover=cover,
809 fallback_release_date=album_release_date,
810 )
811 row['track_number'] = tracklist_slot
812 row['album_track_total'] = album_track_total
813 row['preview_url'] = _track_preview_url(track)
814 row.update(album_fields)
815 songs.append(row)
816 return songs
819def _parse_playlist_tracks(entity: dict[str, Any]) -> list[dict[str, Any]]:
820 fallback_cover = _cover_url(entity)
821 track_items = entity.get('trackList') or []
822 songs: list[dict[str, Any]] = []
823 for item in track_items:
824 if not isinstance(item, dict):
825 continue
826 track = _embed_row_track(item)
827 if not isinstance(track, dict):
828 continue
829 track_id = track.get('id') or _id_from_uri(track.get('uri', ''))
830 if not track_id:
831 continue
832 song = _track_dict(
833 dict(track),
834 track_id=track_id,
835 fallback_cover=fallback_cover,
836 )
837 song['preview_url'] = _track_preview_url(track)
838 songs.append(song)
839 return songs
842def _with_embed_previews(
843 songs: list[dict[str, Any]], entity: dict[str, Any]
844) -> list[dict[str, Any]]:
845 """*songs* (from the paginated GraphQL path, which carries no preview
846 clips) with each one's ``preview_url`` taken from the embed page's own
847 track list where it has the song - the embed only lists the first
848 ~50-100 tracks, so the rest keep none (the page can still look a clip
849 up on demand, see ``GET /api/preview``)."""
851 previews = {
852 song['song_id']: song['preview_url']
853 for song in _parse_playlist_tracks(entity)
854 if song.get('preview_url')
855 }
856 for song in songs:
857 if not song.get('preview_url'):
858 song['preview_url'] = previews.get(song.get('song_id'), '')
859 return songs
862def _artists_from_subtitle(subtitle: Any) -> list[dict[str, str]]:
863 """The artists in an embed row's joined ``subtitle``.
865 Spotify joins them with a comma and a NO-BREAK space
866 (``"Earth, Wind & Fire,\\xa0The Emotions"``) or a full-width comma,
867 while a comma inside one name is followed by an ordinary space - so
868 only those separators split. A subtitle with no such separator is one
869 artist, commas and all; whether it truly is gets checked against the
870 per-track embed before download (see :func:`_ambiguous_subtitle`).
871 """
873 if not isinstance(subtitle, str) or not subtitle:
874 return []
875 return [
876 {'name': ' '.join(name.replace('\xa0', ' ').split())}
877 for name in re.split(r'\s*(?:,\xa0|,)\s*', subtitle)
878 if name.strip()
879 ]
882def _ambiguous_subtitle(subtitle: Any) -> bool:
883 """Whether a name read from *subtitle* still holds a comma - one
884 artist ("Earth, Wind & Fire") or, in a format Spotify hasn't been
885 seen to use, several joined with a plain ``", "``."""
887 return any(',' in d['name'] for d in _artists_from_subtitle(subtitle))
890def _token_from_embed_payload(payload: dict[str, Any]) -> Optional[str]:
891 """Extract the Spotify access token baked into the NEXT_DATA server state."""
892 try:
893 return payload['props']['pageProps']['state']['settings']['session'][
894 'accessToken'
895 ]
896 except (KeyError, TypeError):
897 return None
900_PARTNER_API = 'https://api-partner.spotify.com/pathfinder/v1/query'
901# sha256 of the fetchPlaylist GraphQL document in the Spotify web player.
902# Update when the player bundle rolls and the API returns PersistedQueryNotFound.
903# Location in the bundle: new tz.l("fetchPlaylist","query","<hash>",null)
904_GRAPHQL_HASH = (
905 'a65e12194ed5fc443a1cdebed5fabe33ca5b07b987185d63c72483867ad13cb4'
906)
909def _track_dict_from_graphql_item(
910 item: dict[str, Any],
911) -> Optional[dict[str, Any]]:
912 iv2 = item.get('itemV2') or {}
913 if iv2.get('__typename') != 'TrackResponseWrapper':
914 return None
915 track = iv2.get('data') or {}
916 if not isinstance(track, dict):
917 return None
918 track_id = _id_from_uri(track.get('uri') or '')
919 if not track_id:
920 return None
921 artists = [
922 a['profile']['name']
923 for a in (track.get('artists') or {}).get('items', [])
924 if isinstance(a, dict)
925 and isinstance(a.get('profile'), dict)
926 and a['profile'].get('name')
927 ]
928 album = track.get('albumOfTrack') or {}
929 album_name = album.get('name', '') if isinstance(album, dict) else ''
930 cover_sources = (album.get('coverArt') or {}).get('sources') or []
931 cover_url = _largest_image(cover_sources)
932 duration_ms = (track.get('trackDuration') or {}).get(
933 'totalMilliseconds'
934 ) or 0
935 label = (track.get('contentRating') or {}).get('label') or ''
936 gql_release = ''
937 if isinstance(album, dict):
938 for key in ('date', 'releaseDate'):
939 gql_release = _release_date_raw_from_field(album.get(key))
940 if gql_release:
941 break
942 if not gql_release:
943 logger.info(
944 'Spotify GraphQL track lacks album release date: '
945 'track_id={!r} title={!r} album={!r} '
946 'albumOfTrack_keys={}',
947 track_id,
948 track.get('name'),
949 album_name,
950 sorted(album.keys()) if isinstance(album, dict) else (),
951 )
952 return {
953 'song_id': track_id,
954 'name': track.get('name') or '',
955 'artists': artists,
956 'artist': ', '.join(artists),
957 'album_name': album_name,
958 'cover_url': cover_url,
959 'duration': int(duration_ms / 1000) if duration_ms else 0,
960 'url': f'https://open.spotify.com/track/{track_id}',
961 'explicit': label.upper() == 'EXPLICIT',
962 'release_date': gql_release,
963 'year': _year_from_release_date(gql_release),
964 'source': 'spotify',
965 **_graphql_album_artist(album),
966 }
969def _graphql_album_artist(album: Any) -> dict[str, Any]:
970 """``album_artist`` (and ``compilation``) fields from a GraphQL
971 ``albumOfTrack``: its first credited artist, matched against Spotify's
972 "Various Artists" by id as well as by name."""
974 if not isinstance(album, dict):
975 return {}
976 for item in (album.get('artists') or {}).get('items') or []:
977 if not isinstance(item, dict):
978 continue
979 name = (item.get('profile') or {}).get('name') or ''
980 if name:
981 return album_artist_fields(
982 name, artist_id=_id_from_uri(item.get('uri') or '')
983 )
984 return {}
987def _graphql_fetch_page(
988 playlist_id: str, token: str, offset: int, limit: int = 100
989) -> dict[str, Any]:
990 resp = httpx.get(
991 _PARTNER_API,
992 params={
993 'operationName': 'fetchPlaylist',
994 'variables': json.dumps({
995 'uri': f'spotify:playlist:{playlist_id}',
996 'offset': offset,
997 'limit': limit,
998 'enableWatchFeedEntrypoint': False,
999 'includeEpisodeContentRatingsV2': False,
1000 }),
1001 'extensions': json.dumps({
1002 'persistedQuery': {
1003 'version': 1,
1004 'sha256Hash': _GRAPHQL_HASH,
1005 }
1006 }),
1007 },
1008 headers={
1009 'Authorization': f'Bearer {token}',
1010 'User-Agent': _USER_AGENT,
1011 'app-platform': 'WebPlayer',
1012 },
1013 timeout=15,
1014 )
1015 resp.raise_for_status()
1016 data = resp.json()
1017 logger.debug(
1018 'Spotify GraphQL fetchPlaylist id={} offset={}: {}',
1019 playlist_id,
1020 offset,
1021 json_log_blob(redact_sensitive_mapping(data))[:12000],
1022 )
1023 if 'errors' in data:
1024 raise ValueError(f'GraphQL errors: {data["errors"]}')
1025 return data['data']['playlistV2']
1028_GRAPHQL_PAGE_LIMIT = 100
1029# Pages after the first are fetched this many at a time — enough to make
1030# a long playlist resolve several times faster, few enough to stay polite
1031# to Spotify's API.
1032_GRAPHQL_PAGE_CONCURRENCY = 4
1035def _graphql_page_items(pv2: dict[str, Any]) -> tuple[list[Any], int]:
1036 content = pv2.get('content') or {}
1037 return content.get('items') or [], content.get('totalCount') or 0
1040def _graphql_sequential_items(
1041 playlist_id: str, token: str, offset: int
1042) -> list[Any]:
1043 """Every item from ``offset`` on, one page after another, advancing by
1044 however many items each page actually returned."""
1046 items: list[Any] = []
1047 while True:
1048 page, total = _graphql_page_items(
1049 _graphql_fetch_page(
1050 playlist_id, token, offset, _GRAPHQL_PAGE_LIMIT
1051 )
1052 )
1053 items.extend(page)
1054 offset += len(page)
1055 if not page or offset >= total:
1056 return items
1059def _graphql_all_tracks(
1060 playlist_id: str, token: str
1061) -> tuple[Optional[str], list[dict[str, Any]]]:
1062 """Return ``(playlist_name_or_None, all_tracks)`` via partner GraphQL.
1064 The first page reports the playlist's size, so the remaining pages are
1065 requested concurrently (:data:`_GRAPHQL_PAGE_CONCURRENCY` at a time)
1066 at fixed offsets instead of one after another. That assumes every page
1067 but the last comes back full; if one doesn't, the fixed offsets would
1068 skip items, so the rest is walked sequentially from the first short
1069 page — the old behavior — instead.
1070 """
1072 first = _graphql_fetch_page(playlist_id, token, 0, _GRAPHQL_PAGE_LIMIT)
1073 playlist_name = first.get('name') or None
1074 items, total = _graphql_page_items(first)
1075 page_size = len(items)
1076 if page_size and page_size < total:
1077 offsets = list(range(page_size, total, page_size))
1078 with ThreadPoolExecutor(
1079 max_workers=_GRAPHQL_PAGE_CONCURRENCY,
1080 thread_name_prefix='downtify-spotify-page',
1081 ) as pool:
1082 pages = list(
1083 pool.map(
1084 lambda offset: _graphql_page_items(
1085 _graphql_fetch_page(
1086 playlist_id, token, offset, _GRAPHQL_PAGE_LIMIT
1087 )
1088 )[0],
1089 offsets,
1090 )
1091 )
1092 for offset, page in zip(offsets, pages):
1093 if len(page) < page_size and offset != offsets[-1]:
1094 items.extend(
1095 _graphql_sequential_items(playlist_id, token, offset)
1096 )
1097 break
1098 items.extend(page)
1099 songs = [
1100 td for item in items if (td := _track_dict_from_graphql_item(item))
1101 ]
1102 return playlist_name, songs
1105def playlist_tracks_from_id(playlist_id: str) -> list[dict[str, Any]]:
1106 payload = _fetch_embed_json('playlist', playlist_id)
1107 entity = _entity_from(payload)
1108 token = _token_from_embed_payload(payload)
1109 if token:
1110 try:
1111 _, tracks = _graphql_all_tracks(playlist_id, token)
1112 return _with_embed_previews(tracks, entity)
1113 except Exception:
1114 logger.opt(exception=True).warning(
1115 'GraphQL pagination failed for {}; using embed data (limited)',
1116 playlist_id,
1117 )
1118 return _parse_playlist_tracks(entity)
1121def playlist_info_and_tracks(
1122 playlist_id: str,
1123) -> tuple[str, list[dict[str, Any]]]:
1124 """Return ``(playlist_name, tracks)`` fetching all tracks via partner GraphQL."""
1125 payload = _fetch_embed_json('playlist', playlist_id)
1126 entity = _entity_from(payload)
1127 embed_name = entity.get('name') or entity.get('title') or playlist_id
1128 token = _token_from_embed_payload(payload)
1129 if token:
1130 try:
1131 graphql_name, tracks = _graphql_all_tracks(playlist_id, token)
1132 return graphql_name or embed_name, _with_embed_previews(
1133 tracks, entity
1134 )
1135 except Exception:
1136 logger.opt(exception=True).warning(
1137 'GraphQL pagination failed for {}; using embed data (limited)',
1138 playlist_id,
1139 )
1140 return embed_name, _parse_playlist_tracks(entity)
1143def playlist_cover_url_from_id(playlist_id: str) -> str:
1144 """Largest cover art Spotify's public playlist embed page offers.
1146 Playlist tracks carry no cover of their own — every track's
1147 ``cover_url`` already falls back to this same image (see
1148 ``_parse_playlist_tracks``) — so this reads it once, directly off
1149 the playlist entity, for saving alongside the playlist's M3U file.
1150 """
1151 payload = _fetch_embed_json('playlist', playlist_id)
1152 entity = _entity_from(payload)
1153 return _cover_url(entity)
1156def _id_from_uri(uri: str) -> str:
1157 if not uri:
1158 return ''
1159 parts = uri.split(':')
1160 return parts[-1] if parts else ''
1163def artist_name_from_id(artist_id: str) -> str:
1164 """Return an artist's display name from their embed page.
1166 The artist embed only carries the name, image and a top-tracks
1167 preview — Spotify does not expose a discography there, so callers
1168 that need releases resolve the artist against another provider by
1169 name. Raises ``ValueError`` when the name can't be read.
1170 """
1172 payload = _fetch_embed_json('artist', artist_id)
1173 entity = _entity_from(payload)
1174 name = (entity.get('name') or entity.get('title') or '').strip()
1175 if not name:
1176 raise ValueError(f'Could not read artist name for {artist_id}')
1177 return name
1180def _artist_entity(
1181 artist_id: str,
1182) -> tuple[dict[str, Any], str, str, Optional[str]]:
1183 """``(entity, name, cover_url, token)`` from an artist embed page.
1185 ``token`` is the anonymous session token baked into the same payload.
1186 Raises ``ValueError`` when the artist name can't be read.
1187 """
1189 payload = _fetch_embed_json('artist', artist_id)
1190 entity = _entity_from(payload)
1191 name = (entity.get('name') or entity.get('title') or '').strip()
1192 if not name:
1193 raise ValueError(f'Could not read artist name for {artist_id}')
1194 return entity, name, _cover_url(entity), _token_from_embed_payload(payload)
1197# sha256 of the queryArtistOverview GraphQL document in the Spotify web
1198# player. Update when the player bundle rolls and the API returns
1199# PersistedQueryNotFound. Location in the bundle:
1200# new tz.l("queryArtistOverview","query","<hash>",null)
1201_ARTIST_OVERVIEW_HASH = (
1202 '9f8134ef565e78621f1e1793555bd6633c5ac144ae0f89604ed3ae3f80b3c8e6'
1203)
1206# sha256 of the artist discography GraphQL document (queryArtistDiscography
1207# Albums / Singles / Compilations / All / Overview all share it). It lives
1208# in a lazily loaded chunk of the web player, not the main bundle:
1209# web-player/xpui-routes-artist.<hash>.js, as
1210# new tz.l("queryArtistDiscographyAll","query","<hash>",null)
1211_ARTIST_DISCOGRAPHY_HASH = (
1212 '5e07d323febb57b4a56a42abbf781490e58764aa45feb6e3dc0591564fc56599'
1213)
1214# Unlike the overview, this query pages: ``limit`` is required (the
1215# request fails without it) and larger values are accepted.
1216_DISCOGRAPHY_PAGE_LIMIT = 100
1217_DISCOGRAPHY_MAX_PAGES = 20
1220def _partner_query(
1221 operation: str,
1222 variables: dict[str, Any],
1223 sha256_hash: str,
1224 token: str,
1225) -> dict[str, Any]:
1226 """The ``data`` of one persisted query against the player's GraphQL."""
1228 resp = httpx.get(
1229 _PARTNER_API,
1230 params={
1231 'operationName': operation,
1232 'variables': json.dumps(variables),
1233 'extensions': json.dumps({
1234 'persistedQuery': {'version': 1, 'sha256Hash': sha256_hash}
1235 }),
1236 },
1237 headers={
1238 'Authorization': f'Bearer {token}',
1239 'User-Agent': _USER_AGENT,
1240 'app-platform': 'WebPlayer',
1241 },
1242 timeout=15,
1243 )
1244 resp.raise_for_status()
1245 data = resp.json()
1246 if 'errors' in data:
1247 raise ValueError(f'GraphQL errors: {data["errors"]}')
1248 return data['data']
1251def _artist_discography(artist_id: str, token: str) -> dict[str, Any]:
1252 """The ``discography`` section of the player's artist overview.
1254 It holds ``topTracks`` (each with its album's uri only) and the
1255 artist's releases (albums, singles, compilations) with names, cover
1256 and date — though only the first page of each list: all the albums,
1257 but for a long catalogue just the ten latest singles.
1258 """
1260 data = _partner_query(
1261 'queryArtistOverview',
1262 {
1263 'uri': f'spotify:artist:{artist_id}',
1264 'locale': '',
1265 'includePrerelease': True,
1266 },
1267 _ARTIST_OVERVIEW_HASH,
1268 token,
1269 )
1270 return data['artistUnion']['discography']
1273def _artist_full_discography(artist_id: str, token: str) -> dict[str, Any]:
1274 """Every release of an artist, paging through ``queryArtistDiscographyAll``.
1276 Returns ``{'all': {'items': [...]}}``: one item per release group
1277 (a release and its alternate editions), in the same shape
1278 :func:`_artist_releases` reads from the overview.
1279 """
1281 items: list[Any] = []
1282 for _ in range(_DISCOGRAPHY_MAX_PAGES):
1283 data = _partner_query(
1284 'queryArtistDiscographyAll',
1285 {
1286 'uri': f'spotify:artist:{artist_id}',
1287 'locale': '',
1288 'offset': len(items),
1289 'limit': _DISCOGRAPHY_PAGE_LIMIT,
1290 },
1291 _ARTIST_DISCOGRAPHY_HASH,
1292 token,
1293 )
1294 page = data['artistUnion']['discography']['all']
1295 batch = page.get('items') or []
1296 items.extend(batch)
1297 if not batch or len(items) >= (page.get('totalCount') or 0):
1298 break
1299 return {'all': {'items': items}}
1302_RELEASE_TYPES = {
1303 'ALBUM': 'Album',
1304 'SINGLE': 'Single',
1305 'EP': 'EP',
1306 'COMPILATION': 'Compilation',
1307}
1310def _release_nodes(node: Any) -> list[dict[str, Any]]:
1311 """Every named ``spotify:album:`` node under *node*, in document order."""
1313 found: list[dict[str, Any]] = []
1314 if isinstance(node, dict):
1315 uri = node.get('uri')
1316 name = node.get('name')
1317 if (
1318 isinstance(uri, str)
1319 and uri.startswith('spotify:album:')
1320 and isinstance(name, str)
1321 and name.strip()
1322 ):
1323 found.append(node)
1324 for value in node.values():
1325 found.extend(_release_nodes(value))
1326 elif isinstance(node, list):
1327 for value in node:
1328 found.extend(_release_nodes(value))
1329 return found
1332def _release_sort_key(release: dict[str, Any]) -> tuple[int, int, int]:
1333 date = release.get('date')
1334 date = date if isinstance(date, dict) else {}
1336 def part(key: str) -> int:
1337 value = _calendar_component(date.get(key))
1338 return value or 0
1340 return part('year'), part('month'), part('day')
1343def _artist_releases(
1344 discography: dict[str, Any], artist_name: str
1345) -> list[dict[str, Any]]:
1346 """Release summaries (albums, singles, compilations), newest first.
1348 Same shape as ``providers.artist_albums_from_channel_id`` so the
1349 frontend lists them the same way. Reads either the full discography
1350 or the overview's shorter one; a release can appear in more than one
1351 section of the latter, so this de-duplicates.
1352 """
1354 nodes = _release_nodes({
1355 k: v for k, v in discography.items() if k != 'topTracks'
1356 })
1357 unique: dict[str, dict[str, Any]] = {}
1358 for node in nodes:
1359 unique.setdefault(node['uri'], node)
1360 releases: list[dict[str, Any]] = []
1361 for node in sorted(unique.values(), key=_release_sort_key, reverse=True):
1362 album_id = _id_from_uri(node['uri'])
1363 cover = node.get('coverArt') or {}
1364 kind = str(node.get('type') or '').upper()
1365 year, _month, _day = _release_sort_key(node)
1366 releases.append({
1367 'album_id': album_id,
1368 'name': node['name'].strip(),
1369 'artists': [artist_name] if artist_name else [],
1370 'artist': artist_name,
1371 'cover_url': _largest_image(
1372 cover.get('sources') or [] if isinstance(cover, dict) else []
1373 ),
1374 'year': str(year) if year else '',
1375 'explicit': False,
1376 'url': f'https://open.spotify.com/album/{album_id}',
1377 'source': 'spotify',
1378 'release_type': _RELEASE_TYPES.get(kind, kind.title() or 'Album'),
1379 })
1380 return releases
1383def artist_page_from_id(
1384 artist_id: str,
1385) -> tuple[str, str, list[dict[str, Any]]]:
1386 """``(name, cover_url, releases)`` for a Spotify artist.
1388 The name and photo come from the artist embed. The embed has no
1389 discography, so the releases come from the player's GraphQL: the
1390 full, paged discography, or — if that query no longer resolves (its
1391 hash rolled) — the overview's shorter list. If both fail the
1392 releases are just empty.
1393 """
1395 _, name, cover_url, token = _artist_entity(artist_id)
1396 releases: list[dict[str, Any]] = []
1397 if token:
1398 for source, fetch in (
1399 ('full discography', _artist_full_discography),
1400 ('overview', _artist_discography),
1401 ):
1402 try:
1403 releases = _artist_releases(fetch(artist_id, token), name)
1404 break
1405 except Exception:
1406 logger.opt(exception=True).warning(
1407 'Spotify releases unavailable for artist {} ({})',
1408 artist_id,
1409 source,
1410 )
1411 return name, cover_url, releases
1414def _album_names_by_uri(node: Any) -> dict[str, str]:
1415 """Every ``spotify:album:`` uri that comes with a name under *node*."""
1417 found: dict[str, str] = {}
1418 if isinstance(node, dict):
1419 uri = node.get('uri')
1420 name = node.get('name')
1421 if (
1422 isinstance(uri, str)
1423 and uri.startswith('spotify:album:')
1424 and isinstance(name, str)
1425 and name.strip()
1426 ):
1427 found[uri] = name.strip()
1428 for value in node.values():
1429 found.update(_album_names_by_uri(value))
1430 elif isinstance(node, list):
1431 for value in node:
1432 found.update(_album_names_by_uri(value))
1433 return found
1436def _album_name_from_embed(album_uri: str) -> str:
1437 try:
1438 payload = _fetch_embed_json('album', _id_from_uri(album_uri))
1439 entity = _entity_from(payload)
1440 except Exception:
1441 logger.opt(exception=True).debug(
1442 'Spotify album embed lookup failed for {}', album_uri
1443 )
1444 return ''
1445 return str(entity.get('name') or entity.get('title') or '').strip()
1448# The embed player's 30 s clips live on this host, as
1449# ``https://p.scdn.co/mp3-preview/<hash>``. Public, no auth, CORS open.
1450_PREVIEW_HOST = 'p.scdn.co'
1453def _track_preview_url(row: dict[str, Any]) -> str:
1454 """The 30 s preview clip of an embed row (``audioPreview.url``), or
1455 ``""`` when the row has none or it isn't an https link on
1456 :data:`_PREVIEW_HOST` - the page plays this URL as is, so nothing else
1457 is passed on."""
1459 preview = row.get('audioPreview')
1460 url = preview.get('url') if isinstance(preview, dict) else ''
1461 if not isinstance(url, str):
1462 return ''
1463 parts = urlsplit(url)
1464 if parts.scheme == 'https' and parts.hostname == _PREVIEW_HOST:
1465 return url
1466 return ''
1469def _play_count(raw: Any) -> int:
1470 """A ``playcount`` (the player sends it as a string), or 0 if unusable."""
1472 if isinstance(raw, bool):
1473 return 0
1474 try:
1475 return max(int(raw), 0)
1476 except (TypeError, ValueError):
1477 return 0
1480def _top_track_overview(
1481 artist_id: str, token: str
1482) -> dict[str, dict[str, Any]]:
1483 """``{track id: {'album_name', 'play_count'}}`` for an artist's top tracks.
1485 The artist embed's shelf rows carry neither an album nor a play count,
1486 and neither does the per-track embed. The web player's
1487 ``queryArtistOverview`` query (the same anonymous-token GraphQL endpoint
1488 playlists use) lists each top track's play count and its album by uri
1489 and, in the same response, the artist's releases by name — one request
1490 covers almost every track. An album missing from that list is looked up
1491 in its own embed instead.
1493 Best effort: any failure (the persisted-query hash rolled, the
1494 network) returns ``{}``, leaving those fields blank rather than failing
1495 the whole artist. A track only carries the keys that are known.
1496 """
1498 try:
1499 discography = _artist_discography(artist_id, token)
1500 names = _album_names_by_uri(discography)
1501 top = (discography.get('topTracks') or {}).get('items') or []
1502 tracks: dict[str, tuple[str, int]] = {}
1503 for item in top:
1504 track = item.get('track') if isinstance(item, dict) else None
1505 if not isinstance(track, dict):
1506 continue
1507 track_id = track.get('id') or _id_from_uri(track.get('uri') or '')
1508 album = track.get('albumOfTrack') or {}
1509 album_uri = album.get('uri') if isinstance(album, dict) else ''
1510 if track_id:
1511 tracks[track_id] = (
1512 album_uri or '',
1513 _play_count(track.get('playcount')),
1514 )
1515 except Exception:
1516 logger.opt(exception=True).warning(
1517 'Spotify top track details unavailable for artist {}', artist_id
1518 )
1519 return {}
1521 result: dict[str, dict[str, Any]] = {}
1522 for track_id, (album_uri, plays) in tracks.items():
1523 info: dict[str, Any] = {}
1524 if album_uri:
1525 if album_uri not in names:
1526 names[album_uri] = _album_name_from_embed(album_uri)
1527 if names[album_uri]:
1528 info['album_name'] = names[album_uri]
1529 if plays:
1530 info['play_count'] = plays
1531 if info:
1532 result[track_id] = info
1533 return result
1536# Top songs are each enriched with their own embed fetch (~0.6 s apiece):
1537# a handful at a time keeps a full shelf under a few seconds without
1538# hammering Spotify.
1539_TOP_SONGS_CONCURRENCY = 5
1542def artist_top_songs_from_id(
1543 artist_id: str,
1544 limit: Optional[int] = None,
1545) -> tuple[str, str, list[dict[str, Any]]]:
1546 """``(name, cover_url, songs)`` from an artist embed's top-tracks shelf.
1548 This is Spotify's own "Popular"/top-tracks preview shown at the top of
1549 the artist's home page (up to ~10 tracks, the same shelf a "This is
1550 <Artist>" playlist draws its first entries from) — not a discography,
1551 which :func:`artist_name_from_id`'s docstring already explains isn't
1552 exposed here. Named ``artist_top_songs_*`` (not ``*_top_tracks_*``) to
1553 match the equivalent YouTube Music primitive,
1554 :func:`providers.artist_top_songs_from_channel_id`, even though Spotify
1555 itself calls this shelf "Top Tracks". Raises ``ValueError`` when the
1556 artist name can't be read; returns an empty song list (not an error)
1557 when the shelf itself can't be parsed.
1559 The shelf lives under ``entity['trackList']`` — the same field name
1560 (and flat row shape, no ``track`` wrapper) already used by
1561 :func:`_parse_playlist_tracks` for playlists/albums; confirmed against
1562 a live embed fetch, since :func:`artist_name_from_id`'s own docstring
1563 only promised the preview was *somewhere* in the payload. Unlike a
1564 playlist row, a shelf row carries no per-track album art at all (no
1565 ``album``/``coverArt``), so each track is enriched via
1566 :func:`enrich_track_from_spotify_if_sparse` — the same per-track
1567 re-fetch a monitored playlist already does — so every song gets its
1568 own album cover instead of falling back to the artist's photo. The
1569 album *name* and the play count aren't in either embed; see
1570 :func:`_top_track_overview`. Each row also carries ``preview_url``: the
1571 30 s clip the embed player plays (``""`` when the row has none), see
1572 :func:`_track_preview_url`.
1574 *limit* keeps only the first that many songs, and is applied *before*
1575 the per-track enrichment - which is where the time goes - so asking
1576 for 5 of the shelf's 10 costs about half. The enrichments (and the
1577 overview request) run concurrently, in the shelf's own order.
1578 """
1580 entity, name, cover_url, token = _artist_entity(artist_id)
1582 items = entity.get('trackList') or []
1583 songs: list[dict[str, Any]] = []
1584 for item in items:
1585 if not isinstance(item, dict):
1586 continue
1587 track = _embed_row_track(item)
1588 if not isinstance(track, dict):
1589 continue
1590 track_id = track.get('id') or _id_from_uri(track.get('uri', ''))
1591 if not track_id:
1592 continue
1593 song = _track_dict(
1594 dict(track), track_id=track_id, fallback_cover=cover_url
1595 )
1596 song['preview_url'] = _track_preview_url(track)
1597 songs.append(song)
1598 if limit is not None and len(songs) >= limit:
1599 break
1600 if not songs:
1601 logger.warning(
1602 'No top-songs parsed from Spotify artist embed for {}', artist_id
1603 )
1604 return name, cover_url, songs
1605 with ThreadPoolExecutor(
1606 max_workers=min(_TOP_SONGS_CONCURRENCY, len(songs)) + 1,
1607 thread_name_prefix='downtify-spotify-top',
1608 ) as pool:
1609 overview_future = (
1610 pool.submit(_top_track_overview, artist_id, token)
1611 if token
1612 else None
1613 )
1614 # map() keeps the shelf's order whichever enrichment finishes first.
1615 # The overview below brings every top song's album name, so no
1616 # per-song album lookup here.
1617 songs = list(
1618 pool.map(
1619 lambda s: enrich_track_from_spotify_if_sparse(
1620 s, with_album=False
1621 ),
1622 songs,
1623 )
1624 )
1625 overview = overview_future.result() if overview_future else None
1626 if overview is not None:
1627 for song in songs:
1628 info = overview.get(song['song_id']) or {}
1629 if not song.get('album_name') and info.get('album_name'):
1630 song['album_name'] = info['album_name']
1631 if info.get('play_count'):
1632 song['play_count'] = info['play_count']
1633 return name, cover_url, songs
1636def artist_image_url_from_id(artist_id: str) -> str:
1637 """Largest square photo an artist's embed page offers, or ``""``.
1639 Reads the same ``visualIdentity.image`` field :func:`_cover_url`
1640 already checks for tracks/albums/playlists — see
1641 :func:`artist_name_from_id` for why an artist embed carries nothing
1642 richer than a name and this one photo. The wide banner shown on an
1643 artist's real Spotify page is a separate image the embed doesn't
1644 expose at all - see :func:`artist_banner_url_from_id`.
1645 """
1647 payload = _fetch_embed_json('artist', artist_id)
1648 entity = _entity_from(payload)
1649 return _cover_url(entity)
1652# sha256 of the queryArtistOverview GraphQL document in the Spotify web
1653# player - same persisted-query mechanism as _GRAPHQL_HASH above, just a
1654# different operation. Update when the player bundle rolls and the API
1655# returns PersistedQueryNotFound.
1656# Location in the bundle: new tz.l("queryArtistOverview","query","<hash>",null)
1657_ARTIST_OVERVIEW_HASH = (
1658 '9f8134ef565e78621f1e1793555bd6633c5ac144ae0f89604ed3ae3f80b3c8e6'
1659)
1662def _artist_overview(artist_id: str) -> dict[str, Any]:
1663 """Raw ``artistUnion`` from the persisted-GraphQL pathfinder API's
1664 ``queryArtistOverview`` operation, or ``{}`` on any failure -
1665 :func:`artist_banner_url_from_id` (``headerImage``) and
1666 :func:`related_artist_names_from_id` (``relatedContent.
1667 relatedArtists``) both read different fields off the same response,
1668 so a caller wanting both only pays for the request once by calling
1669 this directly instead.
1670 """
1672 payload = _fetch_embed_json('artist', artist_id)
1673 token = _token_from_embed_payload(payload)
1674 if not token:
1675 return {}
1676 try:
1677 resp = httpx.get(
1678 _PARTNER_API,
1679 params={
1680 'operationName': 'queryArtistOverview',
1681 'variables': json.dumps({
1682 'uri': f'spotify:artist:{artist_id}',
1683 'locale': '',
1684 'preReleaseV2': False,
1685 }),
1686 'extensions': json.dumps({
1687 'persistedQuery': {
1688 'version': 1,
1689 'sha256Hash': _ARTIST_OVERVIEW_HASH,
1690 }
1691 }),
1692 },
1693 headers={
1694 'Authorization': f'Bearer {token}',
1695 'User-Agent': _USER_AGENT,
1696 'app-platform': 'WebPlayer',
1697 },
1698 timeout=15,
1699 )
1700 resp.raise_for_status()
1701 data = resp.json()
1702 except Exception:
1703 logger.opt(exception=True).debug(
1704 'Spotify artist overview fetch failed for {}', artist_id
1705 )
1706 return {}
1707 artist = (data.get('data') or {}).get('artistUnion') or {}
1708 return artist if artist.get('__typename') == 'Artist' else {}
1711def artist_banner_url_from_id(artist_id: str) -> str:
1712 """Largest wide header/banner image an artist has set, or ``""``.
1714 Not exposed by the embed page at all (see
1715 :func:`artist_image_url_from_id`) - reuses the embed's anonymous
1716 access token against the same persisted-GraphQL pathfinder API
1717 :func:`_graphql_fetch_page` already calls for playlist pagination,
1718 just a different operation (``queryArtistOverview``, see
1719 :func:`_artist_overview`) that happens to surface the artist page's
1720 real header image alongside data we don't need here. Many artists
1721 simply don't have one set - this returns ``""`` rather than falling
1722 back to the square photo, so callers don't silently save the wrong
1723 shape as a "banner".
1724 """
1726 header = (_artist_overview(artist_id).get('headerImage') or {}).get(
1727 'data'
1728 ) or {}
1729 if header.get('__typename') != 'ImageV2':
1730 return ''
1731 return _largest_image(header.get('sources') or [])
1734def related_artist_names_from_id(artist_id: str) -> list[str]:
1735 """ "Fans also like"-equivalent names for a Spotify artist id, or
1736 ``[]`` - from the same ``queryArtistOverview`` call
1737 :func:`artist_banner_url_from_id` uses (see :func:`_artist_overview`),
1738 just ``relatedContent.relatedArtists`` instead of ``headerImage``.
1739 Spotify's own version of this also carries an id and photo per
1740 related artist, thrown away here since callers only want a name list
1741 to merge with Deezer's own related-artist names (see
1742 :func:`downtify.artist_profile.fetch_bio`).
1743 """
1745 items = (
1746 (_artist_overview(artist_id).get('relatedContent') or {}).get(
1747 'relatedArtists'
1748 )
1749 or {}
1750 ).get('items') or []
1751 names = []
1752 for item in items:
1753 if not isinstance(item, dict):
1754 continue
1755 name = str((item.get('profile') or {}).get('name') or '').strip()
1756 if name:
1757 names.append(name)
1758 return names
1761# sha256 of the searchSuggestions GraphQL document (the web player's
1762# search-as-you-type box) - same persisted-query mechanism as above.
1763# Update when the player bundle rolls and the API returns
1764# PersistedQueryNotFound.
1765# Location in the bundle: new tz.l("searchSuggestions","query","<hash>",null)
1766_SEARCH_SUGGESTIONS_HASH = (
1767 'b50ebd72524415b132ddaca04158fd7aca529da28be322c9924643c0633df5bd'
1768)
1769# Any public embed page carries an anonymous access token; this artist's
1770# is only fetched to get one.
1771_TOKEN_SEED_ARTIST_ID = '6XyY86QOPPrYVGvF9ch6wz'
1772_TOKEN_FALLBACK_TTL = 300.0
1773_token_lock = threading.Lock()
1774_token_cache: Optional[tuple[str, float]] = None
1777def _anonymous_token() -> Optional[str]:
1778 """An anonymous web-player access token, cached until it expires."""
1780 global _token_cache # noqa: PLW0603
1781 with _token_lock:
1782 cached = _token_cache
1783 if cached is not None and time.time() < cached[1]:
1784 return cached[0]
1785 try:
1786 payload = _fetch_embed_json('artist', _TOKEN_SEED_ARTIST_ID)
1787 except Exception:
1788 logger.opt(exception=True).debug(
1789 'Spotify anonymous token fetch failed'
1790 )
1791 return None
1792 token = _token_from_embed_payload(payload)
1793 if not token:
1794 return None
1795 try:
1796 expires = (
1797 int(
1798 payload['props']['pageProps']['state']['settings']['session'][
1799 'accessTokenExpirationTimestampMs'
1800 ]
1801 )
1802 / 1000
1803 - 60
1804 )
1805 except (KeyError, TypeError, ValueError):
1806 expires = time.time() + _TOKEN_FALLBACK_TTL
1807 with _token_lock:
1808 _token_cache = (token, expires)
1809 return token
1812def _search_suggestion_items(text: str) -> list[Any]:
1813 """The top results of the web player's search-as-you-type box
1814 (``searchSuggestions``) for *text*: artists, albums, playlists, tracks
1815 and more, each ``{item: {__typename, data}}``, in Spotify's order.
1817 It works with the same anonymous token the embed pages hand out, and
1818 needs a large ``numberOfTopResults`` (20): the default 5 often leaves
1819 what's wanted out. Raises :class:`ValueError` when Spotify didn't
1820 answer, so ``[]`` always means "nothing found".
1821 """
1823 token = _anonymous_token()
1824 if not token:
1825 raise ValueError('No anonymous Spotify token')
1826 try:
1827 resp = httpx.get(
1828 _PARTNER_API,
1829 params={
1830 'operationName': 'searchSuggestions',
1831 'variables': json.dumps({
1832 'query': text,
1833 'limit': 20,
1834 'numberOfTopResults': 20,
1835 'offset': 0,
1836 'includeAuthors': False,
1837 }),
1838 'extensions': json.dumps({
1839 'persistedQuery': {
1840 'version': 1,
1841 'sha256Hash': _SEARCH_SUGGESTIONS_HASH,
1842 }
1843 }),
1844 },
1845 headers={
1846 'Authorization': f'Bearer {token}',
1847 'User-Agent': _USER_AGENT,
1848 'app-platform': 'WebPlayer',
1849 },
1850 timeout=15,
1851 )
1852 resp.raise_for_status()
1853 data = resp.json()
1854 except Exception as exc:
1855 logger.opt(exception=True).debug(
1856 'Spotify search failed for {!r}', text
1857 )
1858 raise ValueError('Spotify search failed') from exc
1859 return (
1860 ((data.get('data') or {}).get('searchV2') or {}).get('topResultsV2')
1861 or {}
1862 ).get('itemsV2') or []
1865def search_artist_by_name(name: str) -> Optional[dict[str, str]]:
1866 """``{id, name}`` of the Spotify artist whose name matches *name*
1867 exactly - ignoring case and the characters a file name can't hold, so
1868 ``ACDC`` finds ``AC/DC`` (see :func:`downtify.file_naming.file_name_key`)
1869 - or ``None``.
1871 Spotify has no public search API, but the web player's own
1872 search-as-you-type box also returns the top entities, artists included
1873 (see :func:`_search_suggestion_items`). The response mixes in other
1874 artists, so only an exact name match counts - same strictness as every
1875 platform's own ``resolve_artist_id`` - never the first artist row. A
1876 name that's ambiguous on Spotify resolves to whichever exact match
1877 ranks first, which is why an id resolved from one of the artist's own
1878 downloaded tracks (see :func:`primary_artist_id_from_track_id`) is
1879 always preferred.
1880 """
1882 text = name.strip()
1883 wanted = file_name_key(text)
1884 if not wanted:
1885 return None
1886 try:
1887 items = _search_suggestion_items(text)
1888 except ValueError:
1889 return None
1890 for artist in search_results(items)['artists']:
1891 if file_name_key(artist['name']) == wanted:
1892 return {'id': artist['id'], 'name': artist['name']}
1893 return None
1896def search_results(items: list[Any]) -> dict[str, list[dict[str, Any]]]:
1897 """Search-suggestion *items* (see :func:`_search_suggestion_items`)
1898 sorted into ``{artists, albums, playlists}``, in Spotify's order.
1900 * artist: ``{id, name, image_url}``
1901 * album: ``{id, name, artists, year, cover_url, type}`` - ``type`` is
1902 Spotify's (``ALBUM``, ``SINGLE``, ``EP``, ``COMPILATION``)
1903 * playlist: ``{id, name, owner, cover_url}``
1905 Rows without an id or a name are left out.
1906 """
1908 out: dict[str, list[dict[str, Any]]] = {
1909 'artists': [],
1910 'albums': [],
1911 'playlists': [],
1912 }
1913 for entry in items:
1914 item = entry.get('item') if isinstance(entry, dict) else None
1915 if not isinstance(item, dict):
1916 continue
1917 data = item.get('data') or {}
1918 if not isinstance(data, dict):
1919 continue
1920 kind = item.get('__typename')
1921 item_id = _id_from_uri(str(data.get('uri') or ''))
1922 if kind == 'ArtistResponseWrapper':
1923 name = str((data.get('profile') or {}).get('name') or '').strip()
1924 sources = (
1925 (data.get('visuals') or {}).get('avatarImage') or {}
1926 ).get('sources') or []
1927 if item_id and name:
1928 out['artists'].append({
1929 'id': item_id,
1930 'name': name,
1931 'image_url': _largest_image(sources),
1932 })
1933 elif kind == 'AlbumResponseWrapper':
1934 name = str(data.get('name') or '').strip()
1935 artists = [
1936 str((a.get('profile') or {}).get('name') or '').strip()
1937 for a in (data.get('artists') or {}).get('items') or []
1938 if isinstance(a, dict)
1939 ]
1940 year = (data.get('date') or {}).get('year')
1941 if item_id and name:
1942 out['albums'].append({
1943 'id': item_id,
1944 'name': name,
1945 'artists': [a for a in artists if a],
1946 'year': str(year) if year else '',
1947 'cover_url': _largest_image(
1948 (data.get('coverArt') or {}).get('sources') or []
1949 ),
1950 'type': str(data.get('type') or ''),
1951 })
1952 elif kind == 'PlaylistResponseWrapper':
1953 name = str(data.get('name') or '').strip()
1954 owner = (data.get('ownerV2') or {}).get('data') or {}
1955 images = (data.get('images') or {}).get('items') or []
1956 cover = (
1957 _largest_image(images[0].get('sources') or [])
1958 if images and isinstance(images[0], dict)
1959 else ''
1960 )
1961 if item_id and name:
1962 out['playlists'].append({
1963 'id': item_id,
1964 'name': name,
1965 'owner': str(owner.get('name') or '').strip(),
1966 'cover_url': cover,
1967 })
1968 return out
1971def search(text: str) -> dict[str, list[dict[str, Any]]]:
1972 """:func:`search_results` for *text*. Raises :class:`ValueError` when
1973 Spotify didn't answer."""
1975 return search_results(_search_suggestion_items(text.strip()))
1978def primary_artist_id_from_track_id(track_id: str) -> Optional[str]:
1979 """Spotify id of *track_id*'s first credited artist, or ``None``.
1981 A track embed's raw ``artists`` entries carry ``{name, uri}`` each;
1982 :func:`_artist_names` throws the ``uri`` away since no other caller
1983 needs it. Used to find an artist's photo starting from a track
1984 already in the library (see :mod:`downtify.track_index`) - the most
1985 reliable route, since it can't pick a namesake the way
1986 :func:`search_artist_by_name` can.
1987 """
1989 payload = _fetch_embed_json('track', track_id)
1990 entity = _entity_from(payload)
1991 raw = entity.get('artists') or []
1992 if not isinstance(raw, list) or not raw:
1993 return None
1994 first = raw[0]
1995 if not isinstance(first, dict):
1996 return None
1997 return _id_from_uri(str(first.get('uri') or '')) or None
2000def credited_artist_id_from_track_id(
2001 track_id: str, name: str
2002) -> Optional[str]:
2003 """Spotify id of the artist *name* (compared as
2004 :func:`~downtify.file_naming.file_name_key`) among those credited on
2005 *track_id*, or ``None`` when the track doesn't credit them.
2007 Never another artist's id: a guest's page must not get the track's
2008 first artist (:func:`primary_artist_id_from_track_id`) - their own
2009 entry on the track carries their own ``uri``.
2010 """
2012 wanted = file_name_key(name)
2013 if not wanted:
2014 return None
2015 payload = _fetch_embed_json('track', track_id)
2016 for item in _entity_from(payload).get('artists') or []:
2017 if not isinstance(item, dict):
2018 continue
2019 if file_name_key(str(item.get('name') or '')) == wanted:
2020 return _id_from_uri(str(item.get('uri') or '')) or None
2021 return None
2024def resolve(url: str) -> Any:
2025 """Resolve any Spotify URL to a single song or a list of songs."""
2027 parsed = parse_spotify_url(url)
2028 if parsed is None:
2029 raise ValueError('Not a Spotify URL')
2030 kind, sid = parsed
2031 if kind == 'track':
2032 return track_from_id(sid, with_album=True)
2033 if kind == 'album':
2034 return album_tracks_from_id(sid)
2035 if kind == 'playlist':
2036 return playlist_tracks_from_id(sid)
2037 raise ValueError(f'Unsupported Spotify entity type: {kind}')