Coverage for downtify/download_policy.py: 97%
155 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Persistent, reversible exclusions and cumulative download attempt budgets."""
3from __future__ import annotations
5import hashlib
6import json
7import re
8import secrets
9import threading
10import time
11import unicodedata
12from typing import TYPE_CHECKING, Any, Callable
14from downtify.db.config import Dialect
16if TYPE_CHECKING:
17 from downtify.db.core import Connection
20def _is_postgres(conn: Connection) -> bool:
21 # Raw sqlite3 connections (used by tests) carry no ``dialect``.
22 return getattr(conn, 'dialect', Dialect.SQLITE) is Dialect.POSTGRES
25def _row_lock(conn: Connection) -> str:
26 """Lock the rows a read-modify-write is about to change (PostgreSQL)."""
27 return ' FOR UPDATE' if _is_postgres(conn) else ''
30def _begin_write(conn: Connection) -> None:
31 """Open an exclusive write section for find-or-create.
33 SQLite: ``BEGIN IMMEDIATE`` takes the database write lock. PostgreSQL
34 has no such statement (the connection already runs in a transaction),
35 and a row that does not exist yet cannot be locked with ``FOR UPDATE``,
36 so a transaction-scoped advisory lock serialises the find-or-create
37 across threads and processes. It is released at commit.
38 """
39 if _is_postgres(conn):
40 conn.execute(
41 "SELECT pg_advisory_xact_lock(hashtext('download_policy'))"
42 )
43 else:
44 conn.execute('BEGIN IMMEDIATE')
47class DownloadPolicyBlocked(Exception):
48 def __init__(self, reason: str):
49 super().__init__(reason)
50 self.reason = reason
53def _normalize(value: Any) -> str:
54 text = unicodedata.normalize('NFKC', str(value or '')).casefold()
55 # Collaborator suffixes do not change the selected version, but keep
56 # every other version qualifier (live, remix, radio edit, etc.).
57 text = re.sub(
58 r'\s*[\[(]\s*(?:feat\.?|ft\.?|featuring)\s+[^\])]+[\])]\s*$', '', text
59 )
60 return ' '.join(re.sub(r'[^\w]+', ' ', text).split())
63def recording(song: dict[str, Any]) -> dict[str, Any]:
64 raw_id = str(song.get('song_id') or song.get('spotify_id') or '').strip()
65 # Never persist a URL, query string or a token as a track identity.
66 track_id = raw_id if re.fullmatch(r'[A-Za-z0-9_:-]{1,100}', raw_id) else ''
67 artists = song.get('artists')
68 if isinstance(artists, list) and artists:
69 artist = artists[0]
70 else:
71 artist = str(song.get('artist') or '').split(';', 1)[0]
72 try:
73 duration = max(0, min(86400, int(float(song.get('duration') or 0))))
74 except (ValueError, TypeError, OverflowError):
75 duration = 0
76 return {
77 'track_id': track_id,
78 'title': str(song.get('name') or song.get('title') or '')[:500],
79 'artist': str(artist or '')[:300],
80 'album': str(song.get('album_name') or song.get('album') or '')[:500],
81 'duration': duration,
82 }
85def _fallback_key(row: dict[str, Any]) -> str:
86 title, artist, album = (
87 _normalize(row[k]) for k in ('title', 'artist', 'album')
88 )
89 if not title or not artist or not album or not row['duration']:
90 return ''
91 data = json.dumps(
92 [title, artist, album, row['duration']], ensure_ascii=True
93 )
94 return 'metadata:' + hashlib.sha256(data.encode()).hexdigest()
97class DownloadPolicy:
98 """Tracks only actual attempt time, never waiting between watch checks."""
100 def __init__(
101 self, connect: Callable, *, clock: Callable[[], float] = time.monotonic
102 ):
103 self._connect = connect
104 self.clock = clock
105 self._lock = threading.RLock()
106 self._active: set[str] = set()
107 with self._connect() as conn:
108 # PostgreSQL migrations own the schema; only SQLite creates it.
109 if not _is_postgres(conn):
110 self._init_sqlite(conn)
112 @staticmethod
113 def _init_sqlite(conn: Connection) -> None:
114 conn.execute("""CREATE TABLE IF NOT EXISTS download_policy (
115 id TEXT PRIMARY KEY, track_id TEXT NOT NULL DEFAULT '',
116 title TEXT NOT NULL, artist TEXT NOT NULL, album TEXT NOT NULL,
117 duration INTEGER NOT NULL, title_key TEXT NOT NULL,
118 artist_key TEXT NOT NULL, album_key TEXT NOT NULL,
119 fallback_key TEXT NOT NULL, elapsed REAL NOT NULL DEFAULT 0,
120 blocked INTEGER NOT NULL DEFAULT 0, reason TEXT NOT NULL DEFAULT '',
121 requested_by INTEGER NOT NULL DEFAULT 0, updated REAL NOT NULL
122 )""")
123 conn.execute(
124 'CREATE INDEX IF NOT EXISTS idx_policy_track ON download_policy(track_id)'
125 )
126 conn.execute(
127 'CREATE INDEX IF NOT EXISTS idx_policy_match ON download_policy(title_key,artist_key)'
128 )
130 @staticmethod
131 def _find(conn, item: dict[str, Any]):
132 if item['track_id']:
133 row = conn.execute(
134 'SELECT * FROM download_policy WHERE track_id=?'
135 + _row_lock(conn),
136 (item['track_id'],),
137 ).fetchone()
138 if row is not None:
139 return row
140 if not item['duration']:
141 return None
142 title, artist, album = (
143 _normalize(item[k]) for k in ('title', 'artist', 'album')
144 )
145 if not title or not artist:
146 return None
147 rows = conn.execute(
148 """SELECT * FROM download_policy WHERE title_key=? AND artist_key=?
149 AND duration>0 AND abs(duration-?)<=3"""
150 + _row_lock(conn),
151 (title, artist, item['duration']),
152 ).fetchall()
153 # Manually excluding a named version may span releases; automatic
154 # budget accounting also requires the same album to avoid merging
155 # ambiguous recordings. Never guess between multiple matches.
156 rows = [
157 row
158 for row in rows
159 if row['reason'] == 'manual'
160 or (album and row['album_key'] == album)
161 ]
162 return rows[0] if len(rows) == 1 else None
164 def _get_or_create(self, conn, item: dict[str, Any]):
165 row = self._find(conn, item)
166 if row is not None:
167 return row
168 fallback = _fallback_key(item)
169 if not item['track_id'] and not fallback:
170 raise ValueError('identity_uncertain')
171 ident = secrets.token_hex(16)
172 conn.execute(
173 """INSERT INTO download_policy
174 (id,track_id,title,artist,album,duration,title_key,artist_key,album_key,fallback_key,updated)
175 VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
176 (
177 ident,
178 item['track_id'],
179 item['title'],
180 item['artist'],
181 item['album'],
182 item['duration'],
183 _normalize(item['title']),
184 _normalize(item['artist']),
185 _normalize(item['album']),
186 fallback,
187 time.time(),
188 ),
189 )
190 return conn.execute(
191 'SELECT * FROM download_policy WHERE id=?', (ident,)
192 ).fetchone()
194 def blocked(self, song: dict[str, Any]) -> bool:
195 with self._connect() as conn:
196 row = self._find(conn, recording(song))
197 return bool(row and row['blocked'])
199 def suppress(
200 self, song: dict[str, Any], *, user_id: int
201 ) -> dict[str, Any]:
202 item = recording(song)
203 with self._lock, self._connect() as conn:
204 _begin_write(conn)
205 row = self._get_or_create(conn, item)
206 conn.execute(
207 "UPDATE download_policy SET blocked=1,reason='manual',requested_by=?,updated=? WHERE id=?",
208 (user_id, time.time(), row['id']),
209 )
210 return {'id': row['id'], 'reason': 'manual'}
212 def resume(self, ident: str) -> None:
213 with self._lock, self._connect() as conn:
214 if ident in self._active:
215 raise ValueError('attempt_still_stopping')
216 changed = conn.execute(
217 "UPDATE download_policy SET blocked=0,reason='',elapsed=0,updated=? WHERE id=?",
218 (time.time(), ident),
219 ).rowcount
220 if not changed:
221 raise ValueError('item_not_found')
223 def exclusions(self) -> list[dict[str, Any]]:
224 with self._connect() as conn:
225 rows = conn.execute("""SELECT id,title,artist,album,duration,elapsed,reason,updated
226 FROM download_policy WHERE blocked=1 ORDER BY updated DESC LIMIT 1000""").fetchall()
227 return [dict(row) for row in rows]
229 def begin(self, song: dict[str, Any], limit_seconds: int):
230 item = recording(song)
231 with self._lock, self._connect() as conn:
232 _begin_write(conn)
233 row = self._get_or_create(conn, item)
234 ident = row['id']
235 if row['blocked']:
236 raise DownloadPolicyBlocked('download_excluded')
237 if ident in self._active:
238 raise DownloadPolicyBlocked('download_already_active')
239 self._active.add(ident)
240 return Attempt(
241 self, ident, float(row['elapsed']), max(0, limit_seconds)
242 )
244 def _checkpoint(self, ident: str, elapsed: float, limit: int) -> bool:
245 with self._lock, self._connect() as conn:
246 row = conn.execute(
247 'SELECT blocked,reason,elapsed FROM download_policy WHERE id=?'
248 + _row_lock(conn),
249 (ident,),
250 ).fetchone()
251 if row is None:
252 return True
253 total = max(float(row['elapsed']), elapsed)
254 blocked = bool(row['blocked']) or bool(limit and total >= limit)
255 reason = row['reason'] or ('timeout' if blocked else '')
256 conn.execute(
257 'UPDATE download_policy SET elapsed=?,blocked=?,reason=?,updated=? WHERE id=?',
258 (total, int(blocked), reason, time.time(), ident),
259 )
260 return blocked
262 def succeeded(self, song: dict[str, Any]) -> None:
263 with self._lock, self._connect() as conn:
264 row = self._find(conn, recording(song))
265 if row is not None and not row['blocked']:
266 conn.execute(
267 'UPDATE download_policy SET elapsed=0,updated=? WHERE id=?',
268 (time.time(), row['id']),
269 )
271 def _finish(self, attempt, successful: bool) -> None:
272 with self._lock:
273 try:
274 blocked = self._checkpoint(
275 attempt.ident, attempt.elapsed(), attempt.limit
276 )
277 if successful and not blocked:
278 with self._connect() as conn:
279 conn.execute(
280 'UPDATE download_policy SET elapsed=0,updated=? WHERE id=?',
281 (time.time(), attempt.ident),
282 )
283 finally:
284 self._active.discard(attempt.ident)
287class Attempt:
288 def __init__(
289 self, policy: DownloadPolicy, ident: str, base: float, limit: int
290 ):
291 self.policy, self.ident, self.base, self.limit = (
292 policy,
293 ident,
294 base,
295 limit,
296 )
297 self.start = policy.clock()
298 self.last_checkpoint = self.start - 10
300 def elapsed(self) -> float:
301 return self.base + max(0.0, self.policy.clock() - self.start)
303 def check(self, *, force: bool = False) -> None:
304 now = self.policy.clock()
305 if not force and now - self.last_checkpoint < 2:
306 return
307 self.last_checkpoint = now
308 if self.policy._checkpoint(self.ident, self.elapsed(), self.limit):
309 raise DownloadPolicyBlocked('download_excluded')
311 def finish(self, *, successful: bool) -> None:
312 self.policy._finish(self, successful)