Coverage for downtify/backup.py: 88%

177 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Backups of what a Downtify install knows, so a fresh install can be 

2brought back: its settings, every watch with its options, and one record 

3per library song (artist, title, album, where it came from). 

4 

5The music files themselves are not in a backup. When they are lost, the 

6song list tells a restored install what it had, and the songs missing on 

7disk can be queued for download again (see 

8:func:`missing_songs`). 

9 

10A backup is one gzip-compressed JSON file in the backups folder. Writing 

11one never leaves the folder without two good copies: the new file is 

12written under a temporary name, flushed to disk, read back and checked, 

13and only then renamed into place - after which the oldest is deleted, so 

14the newest two remain (:func:`write_backup`). 

15""" 

16 

17from __future__ import annotations 

18 

19import gzip 

20import json 

21import os 

22import re 

23from collections.abc import Callable, Iterable 

24from dataclasses import dataclass 

25from datetime import datetime, timedelta, timezone 

26from pathlib import Path 

27from typing import Any, Optional 

28 

29from loguru import logger 

30 

31FORMAT = 'downtify-backup' 

32VERSION = 1 

33#: Backups kept in the folder: a new one bumps the oldest out. 

34KEEP = 2 

35#: How often scheduled backups may run, in hours (``0``: off). 

36INTERVALS = (0, 24, 72, 168) 

37 

38_NAME = 'downtify-backup-{stamp}.json.gz' 

39_NAME_RE = re.compile(r'^downtify-backup-(\d{8}-\d{6})\.json\.gz$') 

40_PARTIAL_SUFFIX = '.partial' 

41#: An uploaded backup is refused past this size (compressed). 

42MAX_UPLOAD_BYTES = 200 * 1024 * 1024 

43 

44 

45class BackupError(ValueError): 

46 """Not a Downtify backup, or one this version can't read.""" 

47 

48 

49@dataclass(frozen=True) 

50class BackupInfo: 

51 """One backup file in the backups folder.""" 

52 

53 name: str 

54 created_at: str 

55 size: int 

56 settings: bool 

57 watches: int 

58 songs: int 

59 

60 def to_dict(self) -> dict[str, Any]: 

61 return { 

62 'name': self.name, 

63 'created_at': self.created_at, 

64 'size': self.size, 

65 'settings': self.settings, 

66 'watches': self.watches, 

67 'songs': self.songs, 

68 } 

69 

70 

71def _now() -> datetime: 

72 return datetime.now(timezone.utc) 

73 

74 

75def clamp_interval(value: Any) -> int: 

76 """A valid schedule in hours: one of :data:`INTERVALS`, else off.""" 

77 

78 try: 

79 hours = int(value) 

80 except (TypeError, ValueError): 

81 return 0 

82 return hours if hours in INTERVALS else 0 

83 

84 

85# ── Building ──────────────────────────────────────────────────────── 

86 

87 

88#: Library row fields a backup keeps: enough to find the song again. 

89_SONG_FIELDS = ( 

90 'file', 

91 'title', 

92 'artist', 

93 'artists', 

94 'album', 

95 'album_artist', 

96 'track_number', 

97 'year', 

98 'release_date', 

99 'genre', 

100 'duration', 

101 'isrc', 

102 'explicit', 

103) 

104 

105 

106def song_record( 

107 row: dict[str, Any], 

108 *, 

109 spotify_id: str = '', 

110 provider: str = '', 

111) -> dict[str, Any]: 

112 """The backup record of one library song (a ``/tracks`` row).""" 

113 

114 record = {key: row.get(key) for key in _SONG_FIELDS if key in row} 

115 if spotify_id: 

116 record['spotify_id'] = spotify_id 

117 if provider: 

118 record['provider'] = provider 

119 return record 

120 

121 

122def build( 

123 *, 

124 settings: dict[str, Any], 

125 watches: Iterable[dict[str, Any]], 

126 songs: Iterable[dict[str, Any]], 

127 app_version: str = '', 

128 now: Optional[datetime] = None, 

129) -> dict[str, Any]: 

130 """The backup document.""" 

131 

132 return { 

133 'format': FORMAT, 

134 'version': VERSION, 

135 'created_at': (now or _now()).isoformat(timespec='seconds'), 

136 'app_version': app_version, 

137 'settings': dict(settings), 

138 'watches': list(watches), 

139 'songs': list(songs), 

140 } 

141 

142 

143# ── Reading ───────────────────────────────────────────────────────── 

144 

145 

146def parse(raw: bytes) -> dict[str, Any]: 

147 """A backup document from a file's bytes (gzip or plain JSON). 

148 

149 Raises :class:`BackupError` when it isn't a readable Downtify backup. 

150 """ 

151 

152 try: 

153 text = gzip.decompress(raw) if raw[:2] == b'\x1f\x8b' else raw 

154 data = json.loads(text) 

155 except (OSError, EOFError, ValueError) as exc: 

156 raise BackupError('Not a Downtify backup file') from exc 

157 if not isinstance(data, dict) or data.get('format') != FORMAT: 

158 raise BackupError('Not a Downtify backup file') 

159 version = data.get('version') 

160 if not isinstance(version, int) or version > VERSION: 

161 raise BackupError( 

162 'This backup was made by a newer Downtify; update first' 

163 ) 

164 if not isinstance(data.get('settings'), dict): 

165 raise BackupError('The backup has no settings') 

166 for key in ('watches', 'songs'): 

167 if not isinstance(data.get(key), list): 

168 raise BackupError(f'The backup has no {key} list') 

169 return data 

170 

171 

172def read(path: Path) -> dict[str, Any]: 

173 return parse(Path(path).read_bytes()) 

174 

175 

176# ── The folder ────────────────────────────────────────────────────── 

177 

178 

179def _info(path: Path, data: dict[str, Any]) -> BackupInfo: 

180 return BackupInfo( 

181 name=path.name, 

182 created_at=str(data.get('created_at') or ''), 

183 size=path.stat().st_size, 

184 settings=bool(data.get('settings')), 

185 watches=len(data.get('watches') or []), 

186 songs=len(data.get('songs') or []), 

187 ) 

188 

189 

190def backup_names(folder: Path) -> list[str]: 

191 """Backup file names in *folder*, newest first.""" 

192 

193 try: 

194 names = [p.name for p in Path(folder).iterdir() if p.is_file()] 

195 except OSError: 

196 return [] 

197 return sorted((n for n in names if _NAME_RE.match(n)), reverse=True) 

198 

199 

200def path_of(folder: Path, name: str) -> Optional[Path]: 

201 """The backup called *name* in *folder* (``None``: not one of them).""" 

202 

203 if not _NAME_RE.match(str(name or '')): 

204 return None 

205 path = Path(folder) / name 

206 return path if path.is_file() else None 

207 

208 

209def list_backups(folder: Path) -> list[BackupInfo]: 

210 """The backups in *folder*, newest first. An unreadable file is 

211 left out (and logged) rather than failing the list.""" 

212 

213 found: list[BackupInfo] = [] 

214 for name in backup_names(folder): 

215 path = Path(folder) / name 

216 try: 

217 found.append(_info(path, read(path))) 

218 except (OSError, BackupError): 

219 logger.warning('Unreadable backup {}', path) 

220 return found 

221 

222 

223def _fsync_dir(folder: Path) -> None: 

224 # Makes the rename itself durable; not possible on Windows. 

225 try: 

226 fd = os.open(folder, os.O_RDONLY) 

227 except OSError: 

228 return 

229 try: 

230 os.fsync(fd) 

231 except OSError: 

232 pass 

233 finally: 

234 os.close(fd) 

235 

236 

237def write_backup( 

238 folder: Path, 

239 data: dict[str, Any], 

240 *, 

241 keep: int = KEEP, 

242 now: Optional[datetime] = None, 

243) -> BackupInfo: 

244 """Save *data* as a new backup in *folder* and keep the newest *keep*. 

245 

246 The file is written under a temporary name, flushed to disk, read 

247 back and compared with *data*; only a backup that passed is renamed 

248 into place, and only after that are older ones deleted. A failure at 

249 any point leaves every existing backup as it was. 

250 """ 

251 

252 folder = Path(folder) 

253 folder.mkdir(parents=True, exist_ok=True) 

254 when = now or _now() 

255 name = _NAME.format(stamp=when.strftime('%Y%m%d-%H%M%S')) 

256 final = folder / name 

257 if final.exists(): 

258 # Two backups within one second: the second waits its turn. 

259 when += timedelta(seconds=1) 

260 return write_backup(folder, data, keep=keep, now=when) 

261 partial = folder / (name + _PARTIAL_SUFFIX) 

262 payload = json.dumps(data, ensure_ascii=False, default=str).encode() 

263 try: 

264 with ( 

265 open(partial, 'wb') as raw, 

266 gzip.GzipFile( 

267 filename=name, mode='wb', fileobj=raw, mtime=0 

268 ) as out, 

269 ): 

270 out.write(payload) 

271 with open(partial, 'rb+') as raw: 

272 os.fsync(raw.fileno()) 

273 written = read(partial) 

274 if written != json.loads(payload): 

275 raise BackupError('The written backup does not match') 

276 os.replace(partial, final) 

277 _fsync_dir(folder) 

278 except BaseException: 

279 partial.unlink(missing_ok=True) 

280 raise 

281 info = _info(final, written) 

282 prune(folder, keep=keep) 

283 logger.info( 

284 'Backup saved: {} ({} watches, {} songs, {} bytes)', 

285 info.name, 

286 info.watches, 

287 info.songs, 

288 info.size, 

289 ) 

290 return info 

291 

292 

293def prune(folder: Path, *, keep: int = KEEP) -> list[str]: 

294 """Delete all but the newest *keep* backups (and leftover temporary 

295 files from an interrupted backup). Returns the names deleted.""" 

296 

297 folder = Path(folder) 

298 removed: list[str] = [] 

299 for name in backup_names(folder)[max(1, keep) :]: 

300 try: 

301 (folder / name).unlink() 

302 removed.append(name) 

303 except OSError: 

304 logger.warning('Could not delete the old backup {}', name) 

305 try: 

306 leftovers = [ 

307 p for p in folder.iterdir() if p.name.endswith(_PARTIAL_SUFFIX) 

308 ] 

309 except OSError: 

310 leftovers = [] 

311 for path in leftovers: 

312 path.unlink(missing_ok=True) 

313 return removed 

314 

315 

316def due( 

317 last: Optional[str], interval_hours: int, now: Optional[datetime] = None 

318) -> bool: 

319 """Whether a scheduled backup should run (``last``: ISO time of the 

320 newest backup, ``None`` when there is none).""" 

321 

322 if interval_hours <= 0: 

323 return False 

324 if not last: 

325 return True 

326 try: 

327 then = datetime.fromisoformat(last) 

328 except ValueError: 

329 return True 

330 if then.tzinfo is None: 

331 then = then.replace(tzinfo=timezone.utc) 

332 return (now or _now()) - then >= timedelta(hours=interval_hours) 

333 

334 

335# ── Restoring ─────────────────────────────────────────────────────── 

336 

337 

338def missing_songs( 

339 songs: Iterable[dict[str, Any]], on_disk: Callable[[str], bool] 

340) -> list[dict[str, Any]]: 

341 """The backup's songs whose file isn't there any more, as songs the 

342 download pipeline can look up again (title, artists, album; the 

343 Spotify track when the backup knew it).""" 

344 

345 found: list[dict[str, Any]] = [] 

346 for record in songs: 

347 if not isinstance(record, dict): 

348 continue 

349 title = str(record.get('title') or '').strip() 

350 if not title: 

351 continue 

352 stored = str(record.get('file') or '') 

353 if stored and on_disk(stored): 

354 continue 

355 artists = [ 

356 str(a).strip() for a in record.get('artists') or [] if str(a) 

357 ] or [ 

358 a.strip() 

359 for a in str(record.get('artist') or '').split(',') 

360 if a.strip() 

361 ] 

362 song: dict[str, Any] = { 

363 'name': title, 

364 'artists': artists, 

365 'artist': ', '.join(artists), 

366 'album_name': str(record.get('album') or ''), 

367 'year': str(record.get('year') or ''), 

368 'release_date': str(record.get('release_date') or ''), 

369 'duration': int(float(record.get('duration') or 0)), 

370 } 

371 if record.get('isrc'): 

372 song['isrc'] = str(record['isrc']) 

373 spotify_id = str(record.get('spotify_id') or '') 

374 if spotify_id: 

375 song['song_id'] = spotify_id 

376 song['url'] = f'https://open.spotify.com/track/{spotify_id}' 

377 found.append(song) 

378 return found 

379 

380 

381#: Watch fields a restore writes after adding the watch. 

382WATCH_OPTIONS = ( 

383 'enabled', 

384 'deezer_id', 

385 'musicbrainz_id', 

386 'musicbrainz_kinds', 

387)