Coverage for downtify/backup.py: 88%
177 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Backups of what a Downtify install knows, so a fresh install can be
2brought back: its settings, every watch with its options, and one record
3per library song (artist, title, album, where it came from).
5The music files themselves are not in a backup. When they are lost, the
6song list tells a restored install what it had, and the songs missing on
7disk can be queued for download again (see
8:func:`missing_songs`).
10A backup is one gzip-compressed JSON file in the backups folder. Writing
11one never leaves the folder without two good copies: the new file is
12written under a temporary name, flushed to disk, read back and checked,
13and only then renamed into place - after which the oldest is deleted, so
14the newest two remain (:func:`write_backup`).
15"""
17from __future__ import annotations
19import gzip
20import json
21import os
22import re
23from collections.abc import Callable, Iterable
24from dataclasses import dataclass
25from datetime import datetime, timedelta, timezone
26from pathlib import Path
27from typing import Any, Optional
29from loguru import logger
31FORMAT = 'downtify-backup'
32VERSION = 1
33#: Backups kept in the folder: a new one bumps the oldest out.
34KEEP = 2
35#: How often scheduled backups may run, in hours (``0``: off).
36INTERVALS = (0, 24, 72, 168)
38_NAME = 'downtify-backup-{stamp}.json.gz'
39_NAME_RE = re.compile(r'^downtify-backup-(\d{8}-\d{6})\.json\.gz$')
40_PARTIAL_SUFFIX = '.partial'
41#: An uploaded backup is refused past this size (compressed).
42MAX_UPLOAD_BYTES = 200 * 1024 * 1024
45class BackupError(ValueError):
46 """Not a Downtify backup, or one this version can't read."""
49@dataclass(frozen=True)
50class BackupInfo:
51 """One backup file in the backups folder."""
53 name: str
54 created_at: str
55 size: int
56 settings: bool
57 watches: int
58 songs: int
60 def to_dict(self) -> dict[str, Any]:
61 return {
62 'name': self.name,
63 'created_at': self.created_at,
64 'size': self.size,
65 'settings': self.settings,
66 'watches': self.watches,
67 'songs': self.songs,
68 }
71def _now() -> datetime:
72 return datetime.now(timezone.utc)
75def clamp_interval(value: Any) -> int:
76 """A valid schedule in hours: one of :data:`INTERVALS`, else off."""
78 try:
79 hours = int(value)
80 except (TypeError, ValueError):
81 return 0
82 return hours if hours in INTERVALS else 0
85# ── Building ────────────────────────────────────────────────────────
88#: Library row fields a backup keeps: enough to find the song again.
89_SONG_FIELDS = (
90 'file',
91 'title',
92 'artist',
93 'artists',
94 'album',
95 'album_artist',
96 'track_number',
97 'year',
98 'release_date',
99 'genre',
100 'duration',
101 'isrc',
102 'explicit',
103)
106def song_record(
107 row: dict[str, Any],
108 *,
109 spotify_id: str = '',
110 provider: str = '',
111) -> dict[str, Any]:
112 """The backup record of one library song (a ``/tracks`` row)."""
114 record = {key: row.get(key) for key in _SONG_FIELDS if key in row}
115 if spotify_id:
116 record['spotify_id'] = spotify_id
117 if provider:
118 record['provider'] = provider
119 return record
122def build(
123 *,
124 settings: dict[str, Any],
125 watches: Iterable[dict[str, Any]],
126 songs: Iterable[dict[str, Any]],
127 app_version: str = '',
128 now: Optional[datetime] = None,
129) -> dict[str, Any]:
130 """The backup document."""
132 return {
133 'format': FORMAT,
134 'version': VERSION,
135 'created_at': (now or _now()).isoformat(timespec='seconds'),
136 'app_version': app_version,
137 'settings': dict(settings),
138 'watches': list(watches),
139 'songs': list(songs),
140 }
143# ── Reading ─────────────────────────────────────────────────────────
146def parse(raw: bytes) -> dict[str, Any]:
147 """A backup document from a file's bytes (gzip or plain JSON).
149 Raises :class:`BackupError` when it isn't a readable Downtify backup.
150 """
152 try:
153 text = gzip.decompress(raw) if raw[:2] == b'\x1f\x8b' else raw
154 data = json.loads(text)
155 except (OSError, EOFError, ValueError) as exc:
156 raise BackupError('Not a Downtify backup file') from exc
157 if not isinstance(data, dict) or data.get('format') != FORMAT:
158 raise BackupError('Not a Downtify backup file')
159 version = data.get('version')
160 if not isinstance(version, int) or version > VERSION:
161 raise BackupError(
162 'This backup was made by a newer Downtify; update first'
163 )
164 if not isinstance(data.get('settings'), dict):
165 raise BackupError('The backup has no settings')
166 for key in ('watches', 'songs'):
167 if not isinstance(data.get(key), list):
168 raise BackupError(f'The backup has no {key} list')
169 return data
172def read(path: Path) -> dict[str, Any]:
173 return parse(Path(path).read_bytes())
176# ── The folder ──────────────────────────────────────────────────────
179def _info(path: Path, data: dict[str, Any]) -> BackupInfo:
180 return BackupInfo(
181 name=path.name,
182 created_at=str(data.get('created_at') or ''),
183 size=path.stat().st_size,
184 settings=bool(data.get('settings')),
185 watches=len(data.get('watches') or []),
186 songs=len(data.get('songs') or []),
187 )
190def backup_names(folder: Path) -> list[str]:
191 """Backup file names in *folder*, newest first."""
193 try:
194 names = [p.name for p in Path(folder).iterdir() if p.is_file()]
195 except OSError:
196 return []
197 return sorted((n for n in names if _NAME_RE.match(n)), reverse=True)
200def path_of(folder: Path, name: str) -> Optional[Path]:
201 """The backup called *name* in *folder* (``None``: not one of them)."""
203 if not _NAME_RE.match(str(name or '')):
204 return None
205 path = Path(folder) / name
206 return path if path.is_file() else None
209def list_backups(folder: Path) -> list[BackupInfo]:
210 """The backups in *folder*, newest first. An unreadable file is
211 left out (and logged) rather than failing the list."""
213 found: list[BackupInfo] = []
214 for name in backup_names(folder):
215 path = Path(folder) / name
216 try:
217 found.append(_info(path, read(path)))
218 except (OSError, BackupError):
219 logger.warning('Unreadable backup {}', path)
220 return found
223def _fsync_dir(folder: Path) -> None:
224 # Makes the rename itself durable; not possible on Windows.
225 try:
226 fd = os.open(folder, os.O_RDONLY)
227 except OSError:
228 return
229 try:
230 os.fsync(fd)
231 except OSError:
232 pass
233 finally:
234 os.close(fd)
237def write_backup(
238 folder: Path,
239 data: dict[str, Any],
240 *,
241 keep: int = KEEP,
242 now: Optional[datetime] = None,
243) -> BackupInfo:
244 """Save *data* as a new backup in *folder* and keep the newest *keep*.
246 The file is written under a temporary name, flushed to disk, read
247 back and compared with *data*; only a backup that passed is renamed
248 into place, and only after that are older ones deleted. A failure at
249 any point leaves every existing backup as it was.
250 """
252 folder = Path(folder)
253 folder.mkdir(parents=True, exist_ok=True)
254 when = now or _now()
255 name = _NAME.format(stamp=when.strftime('%Y%m%d-%H%M%S'))
256 final = folder / name
257 if final.exists():
258 # Two backups within one second: the second waits its turn.
259 when += timedelta(seconds=1)
260 return write_backup(folder, data, keep=keep, now=when)
261 partial = folder / (name + _PARTIAL_SUFFIX)
262 payload = json.dumps(data, ensure_ascii=False, default=str).encode()
263 try:
264 with (
265 open(partial, 'wb') as raw,
266 gzip.GzipFile(
267 filename=name, mode='wb', fileobj=raw, mtime=0
268 ) as out,
269 ):
270 out.write(payload)
271 with open(partial, 'rb+') as raw:
272 os.fsync(raw.fileno())
273 written = read(partial)
274 if written != json.loads(payload):
275 raise BackupError('The written backup does not match')
276 os.replace(partial, final)
277 _fsync_dir(folder)
278 except BaseException:
279 partial.unlink(missing_ok=True)
280 raise
281 info = _info(final, written)
282 prune(folder, keep=keep)
283 logger.info(
284 'Backup saved: {} ({} watches, {} songs, {} bytes)',
285 info.name,
286 info.watches,
287 info.songs,
288 info.size,
289 )
290 return info
293def prune(folder: Path, *, keep: int = KEEP) -> list[str]:
294 """Delete all but the newest *keep* backups (and leftover temporary
295 files from an interrupted backup). Returns the names deleted."""
297 folder = Path(folder)
298 removed: list[str] = []
299 for name in backup_names(folder)[max(1, keep) :]:
300 try:
301 (folder / name).unlink()
302 removed.append(name)
303 except OSError:
304 logger.warning('Could not delete the old backup {}', name)
305 try:
306 leftovers = [
307 p for p in folder.iterdir() if p.name.endswith(_PARTIAL_SUFFIX)
308 ]
309 except OSError:
310 leftovers = []
311 for path in leftovers:
312 path.unlink(missing_ok=True)
313 return removed
316def due(
317 last: Optional[str], interval_hours: int, now: Optional[datetime] = None
318) -> bool:
319 """Whether a scheduled backup should run (``last``: ISO time of the
320 newest backup, ``None`` when there is none)."""
322 if interval_hours <= 0:
323 return False
324 if not last:
325 return True
326 try:
327 then = datetime.fromisoformat(last)
328 except ValueError:
329 return True
330 if then.tzinfo is None:
331 then = then.replace(tzinfo=timezone.utc)
332 return (now or _now()) - then >= timedelta(hours=interval_hours)
335# ── Restoring ───────────────────────────────────────────────────────
338def missing_songs(
339 songs: Iterable[dict[str, Any]], on_disk: Callable[[str], bool]
340) -> list[dict[str, Any]]:
341 """The backup's songs whose file isn't there any more, as songs the
342 download pipeline can look up again (title, artists, album; the
343 Spotify track when the backup knew it)."""
345 found: list[dict[str, Any]] = []
346 for record in songs:
347 if not isinstance(record, dict):
348 continue
349 title = str(record.get('title') or '').strip()
350 if not title:
351 continue
352 stored = str(record.get('file') or '')
353 if stored and on_disk(stored):
354 continue
355 artists = [
356 str(a).strip() for a in record.get('artists') or [] if str(a)
357 ] or [
358 a.strip()
359 for a in str(record.get('artist') or '').split(',')
360 if a.strip()
361 ]
362 song: dict[str, Any] = {
363 'name': title,
364 'artists': artists,
365 'artist': ', '.join(artists),
366 'album_name': str(record.get('album') or ''),
367 'year': str(record.get('year') or ''),
368 'release_date': str(record.get('release_date') or ''),
369 'duration': int(float(record.get('duration') or 0)),
370 }
371 if record.get('isrc'):
372 song['isrc'] = str(record['isrc'])
373 spotify_id = str(record.get('spotify_id') or '')
374 if spotify_id:
375 song['song_id'] = spotify_id
376 song['url'] = f'https://open.spotify.com/track/{spotify_id}'
377 found.append(song)
378 return found
381#: Watch fields a restore writes after adding the watch.
382WATCH_OPTIONS = (
383 'enabled',
384 'deezer_id',
385 'musicbrainz_id',
386 'musicbrainz_kinds',
387)