Coverage for downtify/download_policy.py: 97%

155 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Persistent, reversible exclusions and cumulative download attempt budgets.""" 

2 

3from __future__ import annotations 

4 

5import hashlib 

6import json 

7import re 

8import secrets 

9import threading 

10import time 

11import unicodedata 

12from typing import TYPE_CHECKING, Any, Callable 

13 

14from downtify.db.config import Dialect 

15 

16if TYPE_CHECKING: 

17 from downtify.db.core import Connection 

18 

19 

20def _is_postgres(conn: Connection) -> bool: 

21 # Raw sqlite3 connections (used by tests) carry no ``dialect``. 

22 return getattr(conn, 'dialect', Dialect.SQLITE) is Dialect.POSTGRES 

23 

24 

25def _row_lock(conn: Connection) -> str: 

26 """Lock the rows a read-modify-write is about to change (PostgreSQL).""" 

27 return ' FOR UPDATE' if _is_postgres(conn) else '' 

28 

29 

30def _begin_write(conn: Connection) -> None: 

31 """Open an exclusive write section for find-or-create. 

32 

33 SQLite: ``BEGIN IMMEDIATE`` takes the database write lock. PostgreSQL 

34 has no such statement (the connection already runs in a transaction), 

35 and a row that does not exist yet cannot be locked with ``FOR UPDATE``, 

36 so a transaction-scoped advisory lock serialises the find-or-create 

37 across threads and processes. It is released at commit. 

38 """ 

39 if _is_postgres(conn): 

40 conn.execute( 

41 "SELECT pg_advisory_xact_lock(hashtext('download_policy'))" 

42 ) 

43 else: 

44 conn.execute('BEGIN IMMEDIATE') 

45 

46 

47class DownloadPolicyBlocked(Exception): 

48 def __init__(self, reason: str): 

49 super().__init__(reason) 

50 self.reason = reason 

51 

52 

53def _normalize(value: Any) -> str: 

54 text = unicodedata.normalize('NFKC', str(value or '')).casefold() 

55 # Collaborator suffixes do not change the selected version, but keep 

56 # every other version qualifier (live, remix, radio edit, etc.). 

57 text = re.sub( 

58 r'\s*[\[(]\s*(?:feat\.?|ft\.?|featuring)\s+[^\])]+[\])]\s*$', '', text 

59 ) 

60 return ' '.join(re.sub(r'[^\w]+', ' ', text).split()) 

61 

62 

63def recording(song: dict[str, Any]) -> dict[str, Any]: 

64 raw_id = str(song.get('song_id') or song.get('spotify_id') or '').strip() 

65 # Never persist a URL, query string or a token as a track identity. 

66 track_id = raw_id if re.fullmatch(r'[A-Za-z0-9_:-]{1,100}', raw_id) else '' 

67 artists = song.get('artists') 

68 if isinstance(artists, list) and artists: 

69 artist = artists[0] 

70 else: 

71 artist = str(song.get('artist') or '').split(';', 1)[0] 

72 try: 

73 duration = max(0, min(86400, int(float(song.get('duration') or 0)))) 

74 except (ValueError, TypeError, OverflowError): 

75 duration = 0 

76 return { 

77 'track_id': track_id, 

78 'title': str(song.get('name') or song.get('title') or '')[:500], 

79 'artist': str(artist or '')[:300], 

80 'album': str(song.get('album_name') or song.get('album') or '')[:500], 

81 'duration': duration, 

82 } 

83 

84 

85def _fallback_key(row: dict[str, Any]) -> str: 

86 title, artist, album = ( 

87 _normalize(row[k]) for k in ('title', 'artist', 'album') 

88 ) 

89 if not title or not artist or not album or not row['duration']: 

90 return '' 

91 data = json.dumps( 

92 [title, artist, album, row['duration']], ensure_ascii=True 

93 ) 

94 return 'metadata:' + hashlib.sha256(data.encode()).hexdigest() 

95 

96 

97class DownloadPolicy: 

98 """Tracks only actual attempt time, never waiting between watch checks.""" 

99 

100 def __init__( 

101 self, connect: Callable, *, clock: Callable[[], float] = time.monotonic 

102 ): 

103 self._connect = connect 

104 self.clock = clock 

105 self._lock = threading.RLock() 

106 self._active: set[str] = set() 

107 with self._connect() as conn: 

108 # PostgreSQL migrations own the schema; only SQLite creates it. 

109 if not _is_postgres(conn): 

110 self._init_sqlite(conn) 

111 

112 @staticmethod 

113 def _init_sqlite(conn: Connection) -> None: 

114 conn.execute("""CREATE TABLE IF NOT EXISTS download_policy ( 

115 id TEXT PRIMARY KEY, track_id TEXT NOT NULL DEFAULT '', 

116 title TEXT NOT NULL, artist TEXT NOT NULL, album TEXT NOT NULL, 

117 duration INTEGER NOT NULL, title_key TEXT NOT NULL, 

118 artist_key TEXT NOT NULL, album_key TEXT NOT NULL, 

119 fallback_key TEXT NOT NULL, elapsed REAL NOT NULL DEFAULT 0, 

120 blocked INTEGER NOT NULL DEFAULT 0, reason TEXT NOT NULL DEFAULT '', 

121 requested_by INTEGER NOT NULL DEFAULT 0, updated REAL NOT NULL 

122 )""") 

123 conn.execute( 

124 'CREATE INDEX IF NOT EXISTS idx_policy_track ON download_policy(track_id)' 

125 ) 

126 conn.execute( 

127 'CREATE INDEX IF NOT EXISTS idx_policy_match ON download_policy(title_key,artist_key)' 

128 ) 

129 

130 @staticmethod 

131 def _find(conn, item: dict[str, Any]): 

132 if item['track_id']: 

133 row = conn.execute( 

134 'SELECT * FROM download_policy WHERE track_id=?' 

135 + _row_lock(conn), 

136 (item['track_id'],), 

137 ).fetchone() 

138 if row is not None: 

139 return row 

140 if not item['duration']: 

141 return None 

142 title, artist, album = ( 

143 _normalize(item[k]) for k in ('title', 'artist', 'album') 

144 ) 

145 if not title or not artist: 

146 return None 

147 rows = conn.execute( 

148 """SELECT * FROM download_policy WHERE title_key=? AND artist_key=? 

149 AND duration>0 AND abs(duration-?)<=3""" 

150 + _row_lock(conn), 

151 (title, artist, item['duration']), 

152 ).fetchall() 

153 # Manually excluding a named version may span releases; automatic 

154 # budget accounting also requires the same album to avoid merging 

155 # ambiguous recordings. Never guess between multiple matches. 

156 rows = [ 

157 row 

158 for row in rows 

159 if row['reason'] == 'manual' 

160 or (album and row['album_key'] == album) 

161 ] 

162 return rows[0] if len(rows) == 1 else None 

163 

164 def _get_or_create(self, conn, item: dict[str, Any]): 

165 row = self._find(conn, item) 

166 if row is not None: 

167 return row 

168 fallback = _fallback_key(item) 

169 if not item['track_id'] and not fallback: 

170 raise ValueError('identity_uncertain') 

171 ident = secrets.token_hex(16) 

172 conn.execute( 

173 """INSERT INTO download_policy 

174 (id,track_id,title,artist,album,duration,title_key,artist_key,album_key,fallback_key,updated) 

175 VALUES (?,?,?,?,?,?,?,?,?,?,?)""", 

176 ( 

177 ident, 

178 item['track_id'], 

179 item['title'], 

180 item['artist'], 

181 item['album'], 

182 item['duration'], 

183 _normalize(item['title']), 

184 _normalize(item['artist']), 

185 _normalize(item['album']), 

186 fallback, 

187 time.time(), 

188 ), 

189 ) 

190 return conn.execute( 

191 'SELECT * FROM download_policy WHERE id=?', (ident,) 

192 ).fetchone() 

193 

194 def blocked(self, song: dict[str, Any]) -> bool: 

195 with self._connect() as conn: 

196 row = self._find(conn, recording(song)) 

197 return bool(row and row['blocked']) 

198 

199 def suppress( 

200 self, song: dict[str, Any], *, user_id: int 

201 ) -> dict[str, Any]: 

202 item = recording(song) 

203 with self._lock, self._connect() as conn: 

204 _begin_write(conn) 

205 row = self._get_or_create(conn, item) 

206 conn.execute( 

207 "UPDATE download_policy SET blocked=1,reason='manual',requested_by=?,updated=? WHERE id=?", 

208 (user_id, time.time(), row['id']), 

209 ) 

210 return {'id': row['id'], 'reason': 'manual'} 

211 

212 def resume(self, ident: str) -> None: 

213 with self._lock, self._connect() as conn: 

214 if ident in self._active: 

215 raise ValueError('attempt_still_stopping') 

216 changed = conn.execute( 

217 "UPDATE download_policy SET blocked=0,reason='',elapsed=0,updated=? WHERE id=?", 

218 (time.time(), ident), 

219 ).rowcount 

220 if not changed: 

221 raise ValueError('item_not_found') 

222 

223 def exclusions(self) -> list[dict[str, Any]]: 

224 with self._connect() as conn: 

225 rows = conn.execute("""SELECT id,title,artist,album,duration,elapsed,reason,updated 

226 FROM download_policy WHERE blocked=1 ORDER BY updated DESC LIMIT 1000""").fetchall() 

227 return [dict(row) for row in rows] 

228 

229 def begin(self, song: dict[str, Any], limit_seconds: int): 

230 item = recording(song) 

231 with self._lock, self._connect() as conn: 

232 _begin_write(conn) 

233 row = self._get_or_create(conn, item) 

234 ident = row['id'] 

235 if row['blocked']: 

236 raise DownloadPolicyBlocked('download_excluded') 

237 if ident in self._active: 

238 raise DownloadPolicyBlocked('download_already_active') 

239 self._active.add(ident) 

240 return Attempt( 

241 self, ident, float(row['elapsed']), max(0, limit_seconds) 

242 ) 

243 

244 def _checkpoint(self, ident: str, elapsed: float, limit: int) -> bool: 

245 with self._lock, self._connect() as conn: 

246 row = conn.execute( 

247 'SELECT blocked,reason,elapsed FROM download_policy WHERE id=?' 

248 + _row_lock(conn), 

249 (ident,), 

250 ).fetchone() 

251 if row is None: 

252 return True 

253 total = max(float(row['elapsed']), elapsed) 

254 blocked = bool(row['blocked']) or bool(limit and total >= limit) 

255 reason = row['reason'] or ('timeout' if blocked else '') 

256 conn.execute( 

257 'UPDATE download_policy SET elapsed=?,blocked=?,reason=?,updated=? WHERE id=?', 

258 (total, int(blocked), reason, time.time(), ident), 

259 ) 

260 return blocked 

261 

262 def succeeded(self, song: dict[str, Any]) -> None: 

263 with self._lock, self._connect() as conn: 

264 row = self._find(conn, recording(song)) 

265 if row is not None and not row['blocked']: 

266 conn.execute( 

267 'UPDATE download_policy SET elapsed=0,updated=? WHERE id=?', 

268 (time.time(), row['id']), 

269 ) 

270 

271 def _finish(self, attempt, successful: bool) -> None: 

272 with self._lock: 

273 try: 

274 blocked = self._checkpoint( 

275 attempt.ident, attempt.elapsed(), attempt.limit 

276 ) 

277 if successful and not blocked: 

278 with self._connect() as conn: 

279 conn.execute( 

280 'UPDATE download_policy SET elapsed=0,updated=? WHERE id=?', 

281 (time.time(), attempt.ident), 

282 ) 

283 finally: 

284 self._active.discard(attempt.ident) 

285 

286 

287class Attempt: 

288 def __init__( 

289 self, policy: DownloadPolicy, ident: str, base: float, limit: int 

290 ): 

291 self.policy, self.ident, self.base, self.limit = ( 

292 policy, 

293 ident, 

294 base, 

295 limit, 

296 ) 

297 self.start = policy.clock() 

298 self.last_checkpoint = self.start - 10 

299 

300 def elapsed(self) -> float: 

301 return self.base + max(0.0, self.policy.clock() - self.start) 

302 

303 def check(self, *, force: bool = False) -> None: 

304 now = self.policy.clock() 

305 if not force and now - self.last_checkpoint < 2: 

306 return 

307 self.last_checkpoint = now 

308 if self.policy._checkpoint(self.ident, self.elapsed(), self.limit): 

309 raise DownloadPolicyBlocked('download_excluded') 

310 

311 def finish(self, *, successful: bool) -> None: 

312 self.policy._finish(self, successful)