Coverage for downtify/download_queue.py: 96%
154 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Pausing the download queue, and keeping it across restarts.
3:data:`queue_pause` is the process-wide switch (``POST
4/api/queue/pause|resume``): while it is paused no download starts - a
5waiting job stays where it is, holding its slot if it has one - and the
6downloads already running finish normally. The queue reads as
7``pausing`` until the last of them ends, then ``paused``. Playlist
8Monitor sweeps wait at the same switch before every track.
10:class:`PendingDownloads` is the database copy of the jobs that haven't
11finished yet, plus the switch itself, so a restart (or an update) picks
12the queue back up where it stopped - still paused when it was.
13"""
15from __future__ import annotations
17import asyncio
18import contextlib
19import json
20from collections.abc import AsyncIterator, Awaitable, Callable, Collection
21from dataclasses import dataclass, field
22from pathlib import Path
23from typing import Any, Optional
25from loguru import logger
27from .db.config import Dialect
28from .db.core import Connection, Database, as_database
30STATE_RUNNING = 'running'
31STATE_PAUSING = 'pausing'
32STATE_PAUSED = 'paused'
34#: Keeps every ``IN (...)`` under SQLite's bound-parameter limit.
35_CHUNK = 500
38def _release(waiter: asyncio.Future[None]) -> None:
39 if not waiter.done():
40 waiter.set_result(None)
43class QueuePause:
44 """Hold new downloads while paused; count the ones still running."""
46 def __init__(self) -> None:
47 self._paused = False
48 self._active = 0
49 self._waiters: list[asyncio.Future[None]] = []
50 #: Awaited when the last running download of a paused queue ends
51 #: (the API broadcasts the new state from it).
52 self.on_drained: Optional[Callable[[], Awaitable[Any]]] = None
54 @property
55 def paused(self) -> bool:
56 return self._paused
58 @property
59 def active(self) -> int:
60 return self._active
62 @property
63 def state(self) -> str:
64 if not self._paused:
65 return STATE_RUNNING
66 return STATE_PAUSING if self._active else STATE_PAUSED
68 def status(self) -> dict[str, Any]:
69 return {
70 'paused': self._paused,
71 'state': self.state,
72 'active': self._active,
73 }
75 def pause(self) -> None:
76 self._paused = True
78 def resume(self) -> None:
79 self._paused = False
80 waiters, self._waiters = self._waiters, []
81 for waiter in waiters:
82 # A waiter may belong to a loop that already closed (tests).
83 with contextlib.suppress(RuntimeError):
84 waiter.get_loop().call_soon_threadsafe(_release, waiter)
86 async def wait(self) -> None:
87 """Return once the queue runs - at once when it isn't paused."""
89 while self._paused:
90 waiter = asyncio.get_running_loop().create_future()
91 self._waiters.append(waiter)
92 try:
93 await waiter
94 finally:
95 with contextlib.suppress(ValueError):
96 self._waiters.remove(waiter)
98 @contextlib.asynccontextmanager
99 async def running(self) -> AsyncIterator[None]:
100 """Count one download as running for as long as the block lasts."""
102 await self.wait()
103 self._active += 1
104 try:
105 yield
106 finally:
107 self._active -= 1
108 if self._paused and not self._active and self.on_drained:
109 try:
110 await self.on_drained()
111 except Exception:
112 logger.opt(exception=True).debug(
113 'Could not announce the paused queue'
114 )
117queue_pause = QueuePause()
120@dataclass
121class PendingDownload:
122 """One saved job: what ``_run_download`` needs to run it again."""
124 song_id: str
125 song: dict[str, Any]
126 #: ``subdir``, ``delay_seconds``, ``playlist_name``,
127 #: ``spotify_playlist_id`` and ``track_order``.
128 options: dict[str, Any] = field(default_factory=dict)
129 user_ids: list[int] = field(default_factory=list)
132class PendingDownloads:
133 """Database copy of the unfinished download jobs, in queue order."""
135 def __init__(self, path_or_db: str | Path | Database) -> None:
136 self.db = as_database(path_or_db, 'library')
137 if isinstance(path_or_db, (str, Path)):
138 self._path: Optional[str] = str(path_or_db)
139 else:
140 self._path = str(self.db.path) if self.db.path else None
141 self._init_db()
143 def _connect(self) -> Any:
144 return self.db.connect()
146 def _init_db(self) -> None:
147 # PostgreSQL gets its schema from the library migrations.
148 if self.db.dialect is not Dialect.SQLITE:
149 return
150 with self._connect() as conn:
151 conn.execute("""
152 CREATE TABLE IF NOT EXISTS pending_downloads (
153 seq INTEGER PRIMARY KEY AUTOINCREMENT,
154 song_id TEXT NOT NULL UNIQUE,
155 song TEXT NOT NULL,
156 options TEXT NOT NULL DEFAULT '{}',
157 user_ids TEXT NOT NULL DEFAULT '[]'
158 )
159 """)
160 conn.execute("""
161 CREATE TABLE IF NOT EXISTS download_queue_state (
162 key TEXT PRIMARY KEY,
163 value TEXT NOT NULL
164 )
165 """)
167 def add(self, rows: Collection[PendingDownload]) -> None:
168 """Save *rows* at the end of the queue.
170 A job already saved keeps its place (and its options); only the
171 accounts waiting on it are merged in.
172 """
174 if not rows:
175 return
176 with self.db.transaction(lock=True) as conn:
177 for row in rows:
178 users = sorted({int(uid) for uid in row.user_ids if uid})
179 inserted = conn.execute(
180 'INSERT INTO pending_downloads '
181 '(song_id, song, options, user_ids) '
182 'VALUES (?, ?, ?, ?) '
183 'ON CONFLICT (song_id) DO NOTHING',
184 (
185 row.song_id,
186 json.dumps(row.song, default=str),
187 json.dumps(row.options, default=str),
188 json.dumps(users),
189 ),
190 ).rowcount
191 if inserted:
192 continue
193 saved = conn.execute(
194 'SELECT user_ids FROM pending_downloads '
195 'WHERE song_id = ?' + _row_lock(conn),
196 (row.song_id,),
197 ).fetchone()
198 if saved is None:
199 continue
200 merged = sorted(set(_user_ids(saved[0])) | set(users))
201 conn.execute(
202 'UPDATE pending_downloads SET user_ids = ? '
203 'WHERE song_id = ?',
204 (json.dumps(merged), row.song_id),
205 )
207 def forget(self, song_ids: Collection[str]) -> None:
208 ids = list(dict.fromkeys(str(sid) for sid in song_ids if sid))
209 if not ids:
210 return
211 with self._connect() as conn:
212 for start in range(0, len(ids), _CHUNK):
213 chunk = ids[start : start + _CHUNK]
214 marks = ','.join('?' * len(chunk))
215 conn.execute(
216 f'DELETE FROM pending_downloads WHERE song_id IN ({marks})',
217 chunk,
218 )
220 def clear(self) -> None:
221 with self._connect() as conn:
222 conn.execute('DELETE FROM pending_downloads')
224 def rows(self) -> list[PendingDownload]:
225 """Every saved job, first queued first (unreadable rows skipped)."""
227 with self._connect() as conn:
228 found = conn.execute(
229 'SELECT song_id, song, options, user_ids '
230 'FROM pending_downloads ORDER BY seq'
231 ).fetchall()
232 rows: list[PendingDownload] = []
233 for raw in found:
234 song_id, song, options, user_ids = tuple(raw)
235 try:
236 song_dict = json.loads(song)
237 options_dict = json.loads(options or '{}')
238 except ValueError:
239 logger.warning(
240 'Skipping unreadable saved download {}', song_id
241 )
242 continue
243 if not isinstance(song_dict, dict):
244 continue
245 rows.append(
246 PendingDownload(
247 song_id=str(song_id),
248 song=song_dict,
249 options=options_dict
250 if isinstance(options_dict, dict)
251 else {},
252 user_ids=_user_ids(user_ids),
253 )
254 )
255 return rows
257 def paused(self) -> bool:
258 with self._connect() as conn:
259 row = conn.execute(
260 "SELECT value FROM download_queue_state WHERE key = 'paused'"
261 ).fetchone()
262 return bool(row) and tuple(row)[0] == '1'
264 def set_paused(self, value: bool) -> None:
265 with self._connect() as conn:
266 conn.execute(
267 'INSERT INTO download_queue_state (key, value) '
268 "VALUES ('paused', ?) "
269 'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
270 ('1' if value else '0',),
271 )
274def _row_lock(conn: Connection) -> str:
275 """Row lock for read-modify-write on PostgreSQL (SQLite has none)."""
276 return ' FOR UPDATE' if conn.dialect is Dialect.POSTGRES else ''
279def _user_ids(raw: Any) -> list[int]:
280 try:
281 values = json.loads(raw or '[]')
282 except (TypeError, ValueError):
283 return []
284 if not isinstance(values, list):
285 return []
286 found: set[int] = set()
287 for value in values:
288 try:
289 uid = int(value)
290 except (TypeError, ValueError):
291 continue
292 if uid:
293 found.add(uid)
294 return sorted(found)