Coverage for downtify/download_queue.py: 96%

154 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Pausing the download queue, and keeping it across restarts. 

2 

3:data:`queue_pause` is the process-wide switch (``POST 

4/api/queue/pause|resume``): while it is paused no download starts - a 

5waiting job stays where it is, holding its slot if it has one - and the 

6downloads already running finish normally. The queue reads as 

7``pausing`` until the last of them ends, then ``paused``. Playlist 

8Monitor sweeps wait at the same switch before every track. 

9 

10:class:`PendingDownloads` is the database copy of the jobs that haven't 

11finished yet, plus the switch itself, so a restart (or an update) picks 

12the queue back up where it stopped - still paused when it was. 

13""" 

14 

15from __future__ import annotations 

16 

17import asyncio 

18import contextlib 

19import json 

20from collections.abc import AsyncIterator, Awaitable, Callable, Collection 

21from dataclasses import dataclass, field 

22from pathlib import Path 

23from typing import Any, Optional 

24 

25from loguru import logger 

26 

27from .db.config import Dialect 

28from .db.core import Connection, Database, as_database 

29 

30STATE_RUNNING = 'running' 

31STATE_PAUSING = 'pausing' 

32STATE_PAUSED = 'paused' 

33 

34#: Keeps every ``IN (...)`` under SQLite's bound-parameter limit. 

35_CHUNK = 500 

36 

37 

38def _release(waiter: asyncio.Future[None]) -> None: 

39 if not waiter.done(): 

40 waiter.set_result(None) 

41 

42 

43class QueuePause: 

44 """Hold new downloads while paused; count the ones still running.""" 

45 

46 def __init__(self) -> None: 

47 self._paused = False 

48 self._active = 0 

49 self._waiters: list[asyncio.Future[None]] = [] 

50 #: Awaited when the last running download of a paused queue ends 

51 #: (the API broadcasts the new state from it). 

52 self.on_drained: Optional[Callable[[], Awaitable[Any]]] = None 

53 

54 @property 

55 def paused(self) -> bool: 

56 return self._paused 

57 

58 @property 

59 def active(self) -> int: 

60 return self._active 

61 

62 @property 

63 def state(self) -> str: 

64 if not self._paused: 

65 return STATE_RUNNING 

66 return STATE_PAUSING if self._active else STATE_PAUSED 

67 

68 def status(self) -> dict[str, Any]: 

69 return { 

70 'paused': self._paused, 

71 'state': self.state, 

72 'active': self._active, 

73 } 

74 

75 def pause(self) -> None: 

76 self._paused = True 

77 

78 def resume(self) -> None: 

79 self._paused = False 

80 waiters, self._waiters = self._waiters, [] 

81 for waiter in waiters: 

82 # A waiter may belong to a loop that already closed (tests). 

83 with contextlib.suppress(RuntimeError): 

84 waiter.get_loop().call_soon_threadsafe(_release, waiter) 

85 

86 async def wait(self) -> None: 

87 """Return once the queue runs - at once when it isn't paused.""" 

88 

89 while self._paused: 

90 waiter = asyncio.get_running_loop().create_future() 

91 self._waiters.append(waiter) 

92 try: 

93 await waiter 

94 finally: 

95 with contextlib.suppress(ValueError): 

96 self._waiters.remove(waiter) 

97 

98 @contextlib.asynccontextmanager 

99 async def running(self) -> AsyncIterator[None]: 

100 """Count one download as running for as long as the block lasts.""" 

101 

102 await self.wait() 

103 self._active += 1 

104 try: 

105 yield 

106 finally: 

107 self._active -= 1 

108 if self._paused and not self._active and self.on_drained: 

109 try: 

110 await self.on_drained() 

111 except Exception: 

112 logger.opt(exception=True).debug( 

113 'Could not announce the paused queue' 

114 ) 

115 

116 

117queue_pause = QueuePause() 

118 

119 

120@dataclass 

121class PendingDownload: 

122 """One saved job: what ``_run_download`` needs to run it again.""" 

123 

124 song_id: str 

125 song: dict[str, Any] 

126 #: ``subdir``, ``delay_seconds``, ``playlist_name``, 

127 #: ``spotify_playlist_id`` and ``track_order``. 

128 options: dict[str, Any] = field(default_factory=dict) 

129 user_ids: list[int] = field(default_factory=list) 

130 

131 

132class PendingDownloads: 

133 """Database copy of the unfinished download jobs, in queue order.""" 

134 

135 def __init__(self, path_or_db: str | Path | Database) -> None: 

136 self.db = as_database(path_or_db, 'library') 

137 if isinstance(path_or_db, (str, Path)): 

138 self._path: Optional[str] = str(path_or_db) 

139 else: 

140 self._path = str(self.db.path) if self.db.path else None 

141 self._init_db() 

142 

143 def _connect(self) -> Any: 

144 return self.db.connect() 

145 

146 def _init_db(self) -> None: 

147 # PostgreSQL gets its schema from the library migrations. 

148 if self.db.dialect is not Dialect.SQLITE: 

149 return 

150 with self._connect() as conn: 

151 conn.execute(""" 

152 CREATE TABLE IF NOT EXISTS pending_downloads ( 

153 seq INTEGER PRIMARY KEY AUTOINCREMENT, 

154 song_id TEXT NOT NULL UNIQUE, 

155 song TEXT NOT NULL, 

156 options TEXT NOT NULL DEFAULT '{}', 

157 user_ids TEXT NOT NULL DEFAULT '[]' 

158 ) 

159 """) 

160 conn.execute(""" 

161 CREATE TABLE IF NOT EXISTS download_queue_state ( 

162 key TEXT PRIMARY KEY, 

163 value TEXT NOT NULL 

164 ) 

165 """) 

166 

167 def add(self, rows: Collection[PendingDownload]) -> None: 

168 """Save *rows* at the end of the queue. 

169 

170 A job already saved keeps its place (and its options); only the 

171 accounts waiting on it are merged in. 

172 """ 

173 

174 if not rows: 

175 return 

176 with self.db.transaction(lock=True) as conn: 

177 for row in rows: 

178 users = sorted({int(uid) for uid in row.user_ids if uid}) 

179 inserted = conn.execute( 

180 'INSERT INTO pending_downloads ' 

181 '(song_id, song, options, user_ids) ' 

182 'VALUES (?, ?, ?, ?) ' 

183 'ON CONFLICT (song_id) DO NOTHING', 

184 ( 

185 row.song_id, 

186 json.dumps(row.song, default=str), 

187 json.dumps(row.options, default=str), 

188 json.dumps(users), 

189 ), 

190 ).rowcount 

191 if inserted: 

192 continue 

193 saved = conn.execute( 

194 'SELECT user_ids FROM pending_downloads ' 

195 'WHERE song_id = ?' + _row_lock(conn), 

196 (row.song_id,), 

197 ).fetchone() 

198 if saved is None: 

199 continue 

200 merged = sorted(set(_user_ids(saved[0])) | set(users)) 

201 conn.execute( 

202 'UPDATE pending_downloads SET user_ids = ? ' 

203 'WHERE song_id = ?', 

204 (json.dumps(merged), row.song_id), 

205 ) 

206 

207 def forget(self, song_ids: Collection[str]) -> None: 

208 ids = list(dict.fromkeys(str(sid) for sid in song_ids if sid)) 

209 if not ids: 

210 return 

211 with self._connect() as conn: 

212 for start in range(0, len(ids), _CHUNK): 

213 chunk = ids[start : start + _CHUNK] 

214 marks = ','.join('?' * len(chunk)) 

215 conn.execute( 

216 f'DELETE FROM pending_downloads WHERE song_id IN ({marks})', 

217 chunk, 

218 ) 

219 

220 def clear(self) -> None: 

221 with self._connect() as conn: 

222 conn.execute('DELETE FROM pending_downloads') 

223 

224 def rows(self) -> list[PendingDownload]: 

225 """Every saved job, first queued first (unreadable rows skipped).""" 

226 

227 with self._connect() as conn: 

228 found = conn.execute( 

229 'SELECT song_id, song, options, user_ids ' 

230 'FROM pending_downloads ORDER BY seq' 

231 ).fetchall() 

232 rows: list[PendingDownload] = [] 

233 for raw in found: 

234 song_id, song, options, user_ids = tuple(raw) 

235 try: 

236 song_dict = json.loads(song) 

237 options_dict = json.loads(options or '{}') 

238 except ValueError: 

239 logger.warning( 

240 'Skipping unreadable saved download {}', song_id 

241 ) 

242 continue 

243 if not isinstance(song_dict, dict): 

244 continue 

245 rows.append( 

246 PendingDownload( 

247 song_id=str(song_id), 

248 song=song_dict, 

249 options=options_dict 

250 if isinstance(options_dict, dict) 

251 else {}, 

252 user_ids=_user_ids(user_ids), 

253 ) 

254 ) 

255 return rows 

256 

257 def paused(self) -> bool: 

258 with self._connect() as conn: 

259 row = conn.execute( 

260 "SELECT value FROM download_queue_state WHERE key = 'paused'" 

261 ).fetchone() 

262 return bool(row) and tuple(row)[0] == '1' 

263 

264 def set_paused(self, value: bool) -> None: 

265 with self._connect() as conn: 

266 conn.execute( 

267 'INSERT INTO download_queue_state (key, value) ' 

268 "VALUES ('paused', ?) " 

269 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', 

270 ('1' if value else '0',), 

271 ) 

272 

273 

274def _row_lock(conn: Connection) -> str: 

275 """Row lock for read-modify-write on PostgreSQL (SQLite has none).""" 

276 return ' FOR UPDATE' if conn.dialect is Dialect.POSTGRES else '' 

277 

278 

279def _user_ids(raw: Any) -> list[int]: 

280 try: 

281 values = json.loads(raw or '[]') 

282 except (TypeError, ValueError): 

283 return [] 

284 if not isinstance(values, list): 

285 return [] 

286 found: set[int] = set() 

287 for value in values: 

288 try: 

289 uid = int(value) 

290 except (TypeError, ValueError): 

291 continue 

292 if uid: 

293 found.add(uid) 

294 return sorted(found)