Coverage for downtify/library_paths_cache.py: 88%

272 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""In-memory + sqlite cache for library path scans and ``GET /tracks`` rows. 

2 

3UI reads return the last snapshot without walking the disk. A background 

4refresh (or a caller that needs a live scan) still fingerprints the 

5tree and rebuilds when files change. ``invalidate_library_paths_cache`` 

6drops the cheap playlist list (M3U) but keeps the track snapshot so a 

7download does not freeze Home / Library. 

8""" 

9 

10from __future__ import annotations 

11 

12import threading 

13import time 

14from collections.abc import Callable, Sequence 

15from pathlib import Path 

16from typing import TYPE_CHECKING, Any, Optional 

17 

18from loguru import logger 

19 

20from .db.core import Database 

21from .library_fingerprint import ( 

22 library_tree_fingerprint, 

23 playlist_listing_fingerprint, 

24) 

25from .library_listing_store import LibraryListingStore 

26 

27if TYPE_CHECKING: 

28 from .library_catalog import LibraryContext 

29 

30_PATH_CACHE: dict[str, tuple[str, list[tuple[str, str]]]] = {} 

31_PLAYLIST_CACHE: dict[str, tuple[str, list[dict[str, Any]]]] = {} 

32_ENTRIES_CACHE: dict[str, tuple[str, list[dict[str, Any]]]] = {} 

33_DATA_LOCK = threading.RLock() 

34_LISTING_LOCK = threading.RLock() 

35_PLAYLIST_LOCK = threading.RLock() 

36_STORE: Optional[LibraryListingStore] = None 

37_REFRESH_LOCK = threading.Lock() 

38_REFRESH_THREAD: Optional[threading.Thread] = None 

39_REFRESH_AGAIN = False 

40_REFRESH_FN: Optional[Callable[[], None]] = None 

41_REFRESH_GATE: Optional[Callable[[], bool]] = None 

42# Seconds to wait after the last invalidation before walking the tree, 

43# so a playlist queue does not rescan extra folders after every track. 

44_REFRESH_DEBOUNCE_S = 5.0 

45#: Seconds a checked ``GET /playlists`` listing is served without walking 

46#: the downloads folder for M3U files again (slow on a NAS). Downtify's 

47#: own M3U writes drop it at once (:func:`forget_cached_playlists`); this 

48#: bounds how late an M3U changed by hand shows up. 

49PLAYLIST_RECHECK_SECONDS = 60.0 

50#: When each playlist listing was last checked against the disk. 

51_PLAYLIST_CHECKED: dict[str, float] = {} 

52#: Listings being checked in the background: a burst of page loads starts 

53#: one walk of the downloads folder, not one each. 

54_PLAYLIST_RECHECKING: set[str] = set() 

55#: Bumped when a listing is dropped, so a walk that began before an M3U was 

56#: written can't store what it read over the drop. 

57_PLAYLIST_GENERATION = 0 

58 

59 

60def set_listing_refresh_fn(fn: Callable[[], None]) -> None: 

61 """Remember how to rebuild listings after an invalidation.""" 

62 

63 global _REFRESH_FN 

64 _REFRESH_FN = fn 

65 

66 

67def set_listing_refresh_gate(fn: Optional[Callable[[], bool]]) -> None: 

68 """*fn* True means a download is using the disk; delay the tree walk.""" 

69 

70 global _REFRESH_GATE 

71 _REFRESH_GATE = fn 

72 

73 

74def listing_lock() -> threading.RLock: 

75 return _LISTING_LOCK 

76 

77 

78def bind_listing_store(target: Optional[Path | Database]) -> None: 

79 """Persist listings in the library database (SQLite file or PostgreSQL). 

80 

81 Pass ``None`` to disable persistence (tests). 

82 """ 

83 

84 global _STORE 

85 _STORE = LibraryListingStore(target) if target is not None else None 

86 

87 

88def cache_key_for_dirs( 

89 download_dir: Path, 

90 slskd_dir: Optional[Path] = None, 

91 extra_dirs: Sequence[Path] = (), 

92) -> str: 

93 parts = [str(download_dir.resolve())] 

94 if slskd_dir is not None: 

95 try: 

96 parts.append(str(slskd_dir.resolve())) 

97 except OSError: 

98 parts.append(str(slskd_dir)) 

99 else: 

100 parts.append('') 

101 extra: list[str] = [] 

102 for root in extra_dirs or (): 

103 try: 

104 extra.append(str(root.resolve())) 

105 except OSError: 

106 extra.append(str(root)) 

107 extra.sort() 

108 parts.append(','.join(extra)) 

109 return '|'.join(parts) 

110 

111 

112def _cache_key(ctx: LibraryContext) -> str: 

113 key = cache_key_for_dirs( 

114 ctx.download_dir, 

115 ctx.slskd_dir, 

116 getattr(ctx, 'extra_dirs', ()) or (), 

117 ) 

118 primary = getattr(ctx, 'primary_extra_id', '') 

119 return f'{key}|primary={primary}' if primary else key 

120 

121 

122def listing_fingerprint( 

123 download_dir: Path, 

124 slskd_dir: Optional[Path] = None, 

125 extra_dirs: Sequence[Path] = (), 

126) -> str: 

127 """Directory-tree fingerprint (folder mtimes, not per-file stats).""" 

128 

129 return library_tree_fingerprint(download_dir, slskd_dir, extra_dirs) 

130 

131 

132def _ctx_fingerprint(ctx: LibraryContext) -> str: 

133 return listing_fingerprint( 

134 ctx.download_dir, 

135 ctx.slskd_dir, 

136 getattr(ctx, 'extra_dirs', ()) or (), 

137 ) 

138 

139 

140def drop_listing_memory_caches() -> None: 

141 """Drop RAM copies only (sqlite snapshot stays). Used to test restart.""" 

142 

143 with _DATA_LOCK: 

144 _PATH_CACHE.clear() 

145 _drop_playlist_listings() 

146 _ENTRIES_CACHE.clear() 

147 

148 

149def _drop_playlist_listings() -> None: 

150 """Forget every playlist listing. Call with ``_DATA_LOCK`` held.""" 

151 

152 global _PLAYLIST_GENERATION 

153 _PLAYLIST_CACHE.clear() 

154 _PLAYLIST_CHECKED.clear() 

155 _PLAYLIST_GENERATION += 1 

156 

157 

158def forget_cached_playlists() -> None: 

159 """Drop the playlist listing (an M3U was just written); the next 

160 ``GET /playlists`` reads the M3U files again.""" 

161 

162 with _DATA_LOCK: 

163 _drop_playlist_listings() 

164 

165 

166def _hydrate_snapshot(key: str, snap: dict[str, Any]) -> None: 

167 fingerprint = str(snap['fingerprint']) 

168 paths = list(snap['paths']) 

169 entries = list(snap['entries']) 

170 _PATH_CACHE[key] = (fingerprint, paths) 

171 _ENTRIES_CACHE[key] = (fingerprint, entries) 

172 playlists = list(snap.get('playlists') or []) 

173 if playlists: 

174 _PLAYLIST_CACHE[key] = (fingerprint, playlists) 

175 

176 

177def _load_sqlite(key: str) -> Optional[dict[str, Any]]: 

178 if _STORE is None: 

179 return None 

180 try: 

181 return _STORE.load(key) 

182 except Exception: 

183 logger.opt(exception=True).debug('Could not load listing snapshot') 

184 return None 

185 

186 

187def _try_sqlite(key: str, fingerprint: str) -> Optional[dict[str, Any]]: 

188 snap = _load_sqlite(key) 

189 if snap is None or str(snap.get('fingerprint') or '') != fingerprint: 

190 return None 

191 with _DATA_LOCK: 

192 _hydrate_snapshot(key, snap) 

193 return snap 

194 

195 

196def peek_cached_path_pairs( 

197 ctx: LibraryContext, 

198) -> Optional[list[tuple[str, str]]]: 

199 """Last known path pairs, with no directory walk.""" 

200 

201 key = _cache_key(ctx) 

202 with _DATA_LOCK: 

203 hit = _PATH_CACHE.get(key) 

204 if hit is not None: 

205 return list(hit[1]) 

206 snap = _load_sqlite(key) 

207 if snap is None: 

208 return None 

209 with _DATA_LOCK: 

210 _hydrate_snapshot(key, snap) 

211 hit = _PATH_CACHE.get(key) 

212 return list(hit[1]) if hit is not None else None 

213 

214 

215def peek_cached_track_entries( 

216 ctx: LibraryContext, 

217) -> Optional[list[dict[str, Any]]]: 

218 """Last folded ``GET /tracks`` rows, with no directory walk.""" 

219 

220 key = _cache_key(ctx) 

221 with _DATA_LOCK: 

222 hit = _ENTRIES_CACHE.get(key) 

223 if hit is not None: 

224 return list(hit[1]) 

225 snap = _load_sqlite(key) 

226 if snap is None: 

227 return None 

228 with _DATA_LOCK: 

229 _hydrate_snapshot(key, snap) 

230 hit = _ENTRIES_CACHE.get(key) 

231 return list(hit[1]) if hit is not None else None 

232 

233 

234def start_background_listing_refresh(fn: Callable[[], None]) -> None: 

235 """Run *fn* on a daemon thread; coalesce overlapping requests.""" 

236 

237 global _REFRESH_THREAD, _REFRESH_AGAIN, _REFRESH_FN 

238 _REFRESH_FN = fn 

239 

240 def worker() -> None: 

241 global _REFRESH_AGAIN, _REFRESH_THREAD 

242 while True: 

243 time.sleep(_REFRESH_DEBOUNCE_S) 

244 with _REFRESH_LOCK: 

245 again = _REFRESH_AGAIN 

246 _REFRESH_AGAIN = False 

247 if again: 

248 continue 

249 gate = _REFRESH_GATE 

250 if gate is not None: 

251 try: 

252 if gate(): 

253 with _REFRESH_LOCK: 

254 _REFRESH_AGAIN = True 

255 continue 

256 except Exception: 

257 pass 

258 task = _REFRESH_FN 

259 try: 

260 if task is not None: 

261 task() 

262 except Exception: 

263 logger.opt(exception=True).debug( 

264 'Background library listing refresh failed' 

265 ) 

266 with _REFRESH_LOCK: 

267 if _REFRESH_AGAIN: 

268 _REFRESH_AGAIN = False 

269 continue 

270 _REFRESH_THREAD = None 

271 return 

272 

273 with _REFRESH_LOCK: 

274 if _REFRESH_THREAD is not None and _REFRESH_THREAD.is_alive(): 

275 _REFRESH_AGAIN = True 

276 return 

277 _REFRESH_THREAD = threading.Thread( 

278 target=worker, 

279 name='downtify-listing-refresh', 

280 daemon=True, 

281 ) 

282 _REFRESH_THREAD.start() 

283 

284 

285def get_cached_path_pairs( 

286 ctx: LibraryContext, 

287 scan_fn, 

288) -> list[tuple[str, str]]: 

289 """Cached ``(stored, resolved)`` pairs, or scan and store the result.""" 

290 

291 key = _cache_key(ctx) 

292 fingerprint = _ctx_fingerprint(ctx) 

293 with _DATA_LOCK: 

294 hit = _PATH_CACHE.get(key) 

295 if hit is not None and hit[0] == fingerprint: 

296 return list(hit[1]) 

297 snap = _try_sqlite(key, fingerprint) 

298 if snap is not None: 

299 return list(snap['paths']) 

300 pairs = scan_fn(ctx) 

301 with _DATA_LOCK: 

302 _PATH_CACHE[key] = (fingerprint, pairs) 

303 return list(pairs) 

304 

305 

306def get_cached_playlists( 

307 download_dir: Path, 

308 slskd_dir: Optional[Path], 

309 extra_dirs: Sequence[Path], 

310 build_fn: Callable[[], list[dict[str, Any]]], 

311 *, 

312 stale_ok: bool = False, 

313) -> list[dict[str, Any]]: 

314 """``GET /playlists`` rows. 

315 

316 *stale_ok* (UI reads) always serves the RAM listing when there is one, 

317 so no page waits on a walk of the downloads folder; once it is older 

318 than :data:`PLAYLIST_RECHECK_SECONDS` a background check brings it up 

319 to date for the next read. Without it, every call checks the M3U 

320 files, so tests and writes see a new M3U on the next listing. 

321 """ 

322 

323 key = cache_key_for_dirs(download_dir, slskd_dir, extra_dirs) 

324 with _DATA_LOCK: 

325 hit = _PLAYLIST_CACHE.get(key) 

326 cached = list(hit[1]) if hit is not None else None 

327 fresh = _playlists_checked_recently(key) 

328 if cached is not None and stale_ok: 

329 if not fresh: 

330 _recheck_playlists_in_background(key, download_dir, build_fn) 

331 return cached 

332 return _check_playlists(key, download_dir, build_fn, stale_ok=stale_ok) 

333 

334 

335def _playlists_checked_recently(key: str) -> bool: 

336 """Call with ``_DATA_LOCK`` held.""" 

337 

338 checked = _PLAYLIST_CHECKED.get(key) 

339 return ( 

340 checked is not None 

341 and time.monotonic() - checked < PLAYLIST_RECHECK_SECONDS 

342 ) 

343 

344 

345def _check_playlists( 

346 key: str, 

347 download_dir: Path, 

348 build_fn: Callable[[], list[dict[str, Any]]], 

349 *, 

350 stale_ok: bool = False, 

351) -> list[dict[str, Any]]: 

352 """The listing, rebuilt when the M3U files on disk have changed.""" 

353 

354 with _PLAYLIST_LOCK: 

355 with _DATA_LOCK: 

356 generation = _PLAYLIST_GENERATION 

357 hit = _PLAYLIST_CACHE.get(key) 

358 # Whoever held the lock just walked: don't walk again after it. 

359 if ( 

360 stale_ok 

361 and hit is not None 

362 and _playlists_checked_recently(key) 

363 ): 

364 return list(hit[1]) 

365 fingerprint = playlist_listing_fingerprint(download_dir) 

366 with _DATA_LOCK: 

367 hit = _PLAYLIST_CACHE.get(key) 

368 if hit is not None and hit[0] == fingerprint: 

369 _PLAYLIST_CHECKED[key] = time.monotonic() 

370 return list(hit[1]) 

371 playlists = build_fn() 

372 with _DATA_LOCK: 

373 if generation == _PLAYLIST_GENERATION: 

374 _PLAYLIST_CACHE[key] = (fingerprint, playlists) 

375 _PLAYLIST_CHECKED[key] = time.monotonic() 

376 return list(playlists) 

377 

378 

379def _recheck_playlists_in_background( 

380 key: str, 

381 download_dir: Path, 

382 build_fn: Callable[[], list[dict[str, Any]]], 

383) -> None: 

384 """Run :func:`_check_playlists` on a daemon thread, one per listing.""" 

385 

386 with _DATA_LOCK: 

387 if key in _PLAYLIST_RECHECKING: 

388 return 

389 _PLAYLIST_RECHECKING.add(key) 

390 

391 def worker() -> None: 

392 try: 

393 _check_playlists(key, download_dir, build_fn) 

394 except Exception: 

395 logger.opt(exception=True).debug( 

396 'Background playlist listing check failed' 

397 ) 

398 finally: 

399 with _DATA_LOCK: 

400 _PLAYLIST_RECHECKING.discard(key) 

401 

402 threading.Thread( 

403 target=worker, name='downtify-playlist-recheck', daemon=True 

404 ).start() 

405 

406 

407def get_cached_track_entries( 

408 ctx: LibraryContext, 

409) -> Optional[list[dict[str, Any]]]: 

410 """Folded ``GET /tracks`` rows when the tree fingerprint still matches.""" 

411 

412 key = _cache_key(ctx) 

413 fingerprint = _ctx_fingerprint(ctx) 

414 with _DATA_LOCK: 

415 hit = _ENTRIES_CACHE.get(key) 

416 if hit is not None and hit[0] == fingerprint: 

417 return list(hit[1]) 

418 snap = _try_sqlite(key, fingerprint) 

419 if snap is not None: 

420 return list(snap['entries']) 

421 return None 

422 

423 

424#: Fingerprint of a RAM listing patched in place (one file added or 

425#: removed): never equal to a real one, so the next live read rebuilds. 

426PROVISIONAL_FINGERPRINT = '' 

427 

428 

429def store_cached_track_entries( 

430 ctx: LibraryContext, 

431 entries: list[dict[str, Any]], 

432 playlists: Optional[list[dict[str, Any]]] = None, 

433 paths: Optional[list[tuple[str, str]]] = None, 

434 *, 

435 provisional: bool = False, 

436) -> None: 

437 """Remember a folded listing in RAM and in sqlite. 

438 

439 *provisional* (a download or delete patching the listing) keeps it in 

440 RAM only, without fingerprinting the tree: walking every library 

441 folder and rewriting the whole snapshot for each downloaded track 

442 was slow on a NAS. The refresh that follows the change (see 

443 :func:`invalidate_library_paths_cache`) stores the real one. 

444 """ 

445 

446 key = _cache_key(ctx) 

447 fingerprint = ( 

448 PROVISIONAL_FINGERPRINT if provisional else _ctx_fingerprint(ctx) 

449 ) 

450 copied = list(entries) 

451 with _DATA_LOCK: 

452 _ENTRIES_CACHE[key] = (fingerprint, copied) 

453 if paths is not None: 

454 _PATH_CACHE[key] = (fingerprint, list(paths)) 

455 path_rows = ( 

456 paths 

457 if paths is not None 

458 else list(_PATH_CACHE.get(key, (fingerprint, []))[1]) 

459 ) 

460 playlist_rows = ( 

461 playlists 

462 if playlists is not None 

463 else list(_PLAYLIST_CACHE.get(key, (fingerprint, []))[1]) 

464 ) 

465 if _STORE is None or provisional: 

466 return 

467 try: 

468 _STORE.save(key, fingerprint, path_rows, playlist_rows, copied) 

469 except Exception: 

470 logger.opt(exception=True).debug( 

471 'Could not persist library listing snapshot' 

472 ) 

473 

474 

475#: Called on every invalidation - i.e. whenever the library changed - with 

476#: no arguments, from whichever thread invalidated. See 

477#: :func:`add_invalidation_listener`. 

478_LISTENERS: list[Callable[[], None]] = [] 

479 

480 

481def add_invalidation_listener(fn: Callable[[], None]) -> None: 

482 """Call *fn* whenever the library changes (e.g. to tell connected 

483 apps to sync). It must be quick and thread-safe.""" 

484 

485 if fn not in _LISTENERS: 

486 _LISTENERS.append(fn) 

487 

488 

489def invalidate_library_paths_cache( 

490 *, notify: bool = True, drop_entries: bool = False 

491) -> None: 

492 """Mark listings stale after downloads, deletes, or path reconcile. 

493 

494 Playlist RAM is always dropped (an M3U scan is cheap). The folded 

495 track snapshot stays unless *drop_entries* is set, so UI routes can 

496 keep serving it while a background refresh catches up. 

497 

498 *notify* ``False`` is for a rescan that isn't a change in itself (a 

499 client asking for a fresh scan): the listeners aren't told. 

500 """ 

501 

502 with _DATA_LOCK: 

503 _drop_playlist_listings() 

504 if drop_entries: 

505 _PATH_CACHE.clear() 

506 _ENTRIES_CACHE.clear() 

507 if drop_entries and _STORE is not None: 

508 try: 

509 _STORE.clear() 

510 except Exception: 

511 logger.opt(exception=True).debug( 

512 'Could not clear library listing snapshot' 

513 ) 

514 refresh = _REFRESH_FN 

515 if refresh is not None: 

516 start_background_listing_refresh(refresh) 

517 if not notify: 

518 return 

519 for listener in list(_LISTENERS): 

520 try: 

521 listener() 

522 except Exception: 

523 logger.opt(exception=True).debug('Library change listener failed')