Coverage for downtify/library_paths_cache.py: 88%
272 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""In-memory + sqlite cache for library path scans and ``GET /tracks`` rows.
3UI reads return the last snapshot without walking the disk. A background
4refresh (or a caller that needs a live scan) still fingerprints the
5tree and rebuilds when files change. ``invalidate_library_paths_cache``
6drops the cheap playlist list (M3U) but keeps the track snapshot so a
7download does not freeze Home / Library.
8"""
10from __future__ import annotations
12import threading
13import time
14from collections.abc import Callable, Sequence
15from pathlib import Path
16from typing import TYPE_CHECKING, Any, Optional
18from loguru import logger
20from .db.core import Database
21from .library_fingerprint import (
22 library_tree_fingerprint,
23 playlist_listing_fingerprint,
24)
25from .library_listing_store import LibraryListingStore
27if TYPE_CHECKING:
28 from .library_catalog import LibraryContext
30_PATH_CACHE: dict[str, tuple[str, list[tuple[str, str]]]] = {}
31_PLAYLIST_CACHE: dict[str, tuple[str, list[dict[str, Any]]]] = {}
32_ENTRIES_CACHE: dict[str, tuple[str, list[dict[str, Any]]]] = {}
33_DATA_LOCK = threading.RLock()
34_LISTING_LOCK = threading.RLock()
35_PLAYLIST_LOCK = threading.RLock()
36_STORE: Optional[LibraryListingStore] = None
37_REFRESH_LOCK = threading.Lock()
38_REFRESH_THREAD: Optional[threading.Thread] = None
39_REFRESH_AGAIN = False
40_REFRESH_FN: Optional[Callable[[], None]] = None
41_REFRESH_GATE: Optional[Callable[[], bool]] = None
42# Seconds to wait after the last invalidation before walking the tree,
43# so a playlist queue does not rescan extra folders after every track.
44_REFRESH_DEBOUNCE_S = 5.0
45#: Seconds a checked ``GET /playlists`` listing is served without walking
46#: the downloads folder for M3U files again (slow on a NAS). Downtify's
47#: own M3U writes drop it at once (:func:`forget_cached_playlists`); this
48#: bounds how late an M3U changed by hand shows up.
49PLAYLIST_RECHECK_SECONDS = 60.0
50#: When each playlist listing was last checked against the disk.
51_PLAYLIST_CHECKED: dict[str, float] = {}
52#: Listings being checked in the background: a burst of page loads starts
53#: one walk of the downloads folder, not one each.
54_PLAYLIST_RECHECKING: set[str] = set()
55#: Bumped when a listing is dropped, so a walk that began before an M3U was
56#: written can't store what it read over the drop.
57_PLAYLIST_GENERATION = 0
60def set_listing_refresh_fn(fn: Callable[[], None]) -> None:
61 """Remember how to rebuild listings after an invalidation."""
63 global _REFRESH_FN
64 _REFRESH_FN = fn
67def set_listing_refresh_gate(fn: Optional[Callable[[], bool]]) -> None:
68 """*fn* True means a download is using the disk; delay the tree walk."""
70 global _REFRESH_GATE
71 _REFRESH_GATE = fn
74def listing_lock() -> threading.RLock:
75 return _LISTING_LOCK
78def bind_listing_store(target: Optional[Path | Database]) -> None:
79 """Persist listings in the library database (SQLite file or PostgreSQL).
81 Pass ``None`` to disable persistence (tests).
82 """
84 global _STORE
85 _STORE = LibraryListingStore(target) if target is not None else None
88def cache_key_for_dirs(
89 download_dir: Path,
90 slskd_dir: Optional[Path] = None,
91 extra_dirs: Sequence[Path] = (),
92) -> str:
93 parts = [str(download_dir.resolve())]
94 if slskd_dir is not None:
95 try:
96 parts.append(str(slskd_dir.resolve()))
97 except OSError:
98 parts.append(str(slskd_dir))
99 else:
100 parts.append('')
101 extra: list[str] = []
102 for root in extra_dirs or ():
103 try:
104 extra.append(str(root.resolve()))
105 except OSError:
106 extra.append(str(root))
107 extra.sort()
108 parts.append(','.join(extra))
109 return '|'.join(parts)
112def _cache_key(ctx: LibraryContext) -> str:
113 key = cache_key_for_dirs(
114 ctx.download_dir,
115 ctx.slskd_dir,
116 getattr(ctx, 'extra_dirs', ()) or (),
117 )
118 primary = getattr(ctx, 'primary_extra_id', '')
119 return f'{key}|primary={primary}' if primary else key
122def listing_fingerprint(
123 download_dir: Path,
124 slskd_dir: Optional[Path] = None,
125 extra_dirs: Sequence[Path] = (),
126) -> str:
127 """Directory-tree fingerprint (folder mtimes, not per-file stats)."""
129 return library_tree_fingerprint(download_dir, slskd_dir, extra_dirs)
132def _ctx_fingerprint(ctx: LibraryContext) -> str:
133 return listing_fingerprint(
134 ctx.download_dir,
135 ctx.slskd_dir,
136 getattr(ctx, 'extra_dirs', ()) or (),
137 )
140def drop_listing_memory_caches() -> None:
141 """Drop RAM copies only (sqlite snapshot stays). Used to test restart."""
143 with _DATA_LOCK:
144 _PATH_CACHE.clear()
145 _drop_playlist_listings()
146 _ENTRIES_CACHE.clear()
149def _drop_playlist_listings() -> None:
150 """Forget every playlist listing. Call with ``_DATA_LOCK`` held."""
152 global _PLAYLIST_GENERATION
153 _PLAYLIST_CACHE.clear()
154 _PLAYLIST_CHECKED.clear()
155 _PLAYLIST_GENERATION += 1
158def forget_cached_playlists() -> None:
159 """Drop the playlist listing (an M3U was just written); the next
160 ``GET /playlists`` reads the M3U files again."""
162 with _DATA_LOCK:
163 _drop_playlist_listings()
166def _hydrate_snapshot(key: str, snap: dict[str, Any]) -> None:
167 fingerprint = str(snap['fingerprint'])
168 paths = list(snap['paths'])
169 entries = list(snap['entries'])
170 _PATH_CACHE[key] = (fingerprint, paths)
171 _ENTRIES_CACHE[key] = (fingerprint, entries)
172 playlists = list(snap.get('playlists') or [])
173 if playlists:
174 _PLAYLIST_CACHE[key] = (fingerprint, playlists)
177def _load_sqlite(key: str) -> Optional[dict[str, Any]]:
178 if _STORE is None:
179 return None
180 try:
181 return _STORE.load(key)
182 except Exception:
183 logger.opt(exception=True).debug('Could not load listing snapshot')
184 return None
187def _try_sqlite(key: str, fingerprint: str) -> Optional[dict[str, Any]]:
188 snap = _load_sqlite(key)
189 if snap is None or str(snap.get('fingerprint') or '') != fingerprint:
190 return None
191 with _DATA_LOCK:
192 _hydrate_snapshot(key, snap)
193 return snap
196def peek_cached_path_pairs(
197 ctx: LibraryContext,
198) -> Optional[list[tuple[str, str]]]:
199 """Last known path pairs, with no directory walk."""
201 key = _cache_key(ctx)
202 with _DATA_LOCK:
203 hit = _PATH_CACHE.get(key)
204 if hit is not None:
205 return list(hit[1])
206 snap = _load_sqlite(key)
207 if snap is None:
208 return None
209 with _DATA_LOCK:
210 _hydrate_snapshot(key, snap)
211 hit = _PATH_CACHE.get(key)
212 return list(hit[1]) if hit is not None else None
215def peek_cached_track_entries(
216 ctx: LibraryContext,
217) -> Optional[list[dict[str, Any]]]:
218 """Last folded ``GET /tracks`` rows, with no directory walk."""
220 key = _cache_key(ctx)
221 with _DATA_LOCK:
222 hit = _ENTRIES_CACHE.get(key)
223 if hit is not None:
224 return list(hit[1])
225 snap = _load_sqlite(key)
226 if snap is None:
227 return None
228 with _DATA_LOCK:
229 _hydrate_snapshot(key, snap)
230 hit = _ENTRIES_CACHE.get(key)
231 return list(hit[1]) if hit is not None else None
234def start_background_listing_refresh(fn: Callable[[], None]) -> None:
235 """Run *fn* on a daemon thread; coalesce overlapping requests."""
237 global _REFRESH_THREAD, _REFRESH_AGAIN, _REFRESH_FN
238 _REFRESH_FN = fn
240 def worker() -> None:
241 global _REFRESH_AGAIN, _REFRESH_THREAD
242 while True:
243 time.sleep(_REFRESH_DEBOUNCE_S)
244 with _REFRESH_LOCK:
245 again = _REFRESH_AGAIN
246 _REFRESH_AGAIN = False
247 if again:
248 continue
249 gate = _REFRESH_GATE
250 if gate is not None:
251 try:
252 if gate():
253 with _REFRESH_LOCK:
254 _REFRESH_AGAIN = True
255 continue
256 except Exception:
257 pass
258 task = _REFRESH_FN
259 try:
260 if task is not None:
261 task()
262 except Exception:
263 logger.opt(exception=True).debug(
264 'Background library listing refresh failed'
265 )
266 with _REFRESH_LOCK:
267 if _REFRESH_AGAIN:
268 _REFRESH_AGAIN = False
269 continue
270 _REFRESH_THREAD = None
271 return
273 with _REFRESH_LOCK:
274 if _REFRESH_THREAD is not None and _REFRESH_THREAD.is_alive():
275 _REFRESH_AGAIN = True
276 return
277 _REFRESH_THREAD = threading.Thread(
278 target=worker,
279 name='downtify-listing-refresh',
280 daemon=True,
281 )
282 _REFRESH_THREAD.start()
285def get_cached_path_pairs(
286 ctx: LibraryContext,
287 scan_fn,
288) -> list[tuple[str, str]]:
289 """Cached ``(stored, resolved)`` pairs, or scan and store the result."""
291 key = _cache_key(ctx)
292 fingerprint = _ctx_fingerprint(ctx)
293 with _DATA_LOCK:
294 hit = _PATH_CACHE.get(key)
295 if hit is not None and hit[0] == fingerprint:
296 return list(hit[1])
297 snap = _try_sqlite(key, fingerprint)
298 if snap is not None:
299 return list(snap['paths'])
300 pairs = scan_fn(ctx)
301 with _DATA_LOCK:
302 _PATH_CACHE[key] = (fingerprint, pairs)
303 return list(pairs)
306def get_cached_playlists(
307 download_dir: Path,
308 slskd_dir: Optional[Path],
309 extra_dirs: Sequence[Path],
310 build_fn: Callable[[], list[dict[str, Any]]],
311 *,
312 stale_ok: bool = False,
313) -> list[dict[str, Any]]:
314 """``GET /playlists`` rows.
316 *stale_ok* (UI reads) always serves the RAM listing when there is one,
317 so no page waits on a walk of the downloads folder; once it is older
318 than :data:`PLAYLIST_RECHECK_SECONDS` a background check brings it up
319 to date for the next read. Without it, every call checks the M3U
320 files, so tests and writes see a new M3U on the next listing.
321 """
323 key = cache_key_for_dirs(download_dir, slskd_dir, extra_dirs)
324 with _DATA_LOCK:
325 hit = _PLAYLIST_CACHE.get(key)
326 cached = list(hit[1]) if hit is not None else None
327 fresh = _playlists_checked_recently(key)
328 if cached is not None and stale_ok:
329 if not fresh:
330 _recheck_playlists_in_background(key, download_dir, build_fn)
331 return cached
332 return _check_playlists(key, download_dir, build_fn, stale_ok=stale_ok)
335def _playlists_checked_recently(key: str) -> bool:
336 """Call with ``_DATA_LOCK`` held."""
338 checked = _PLAYLIST_CHECKED.get(key)
339 return (
340 checked is not None
341 and time.monotonic() - checked < PLAYLIST_RECHECK_SECONDS
342 )
345def _check_playlists(
346 key: str,
347 download_dir: Path,
348 build_fn: Callable[[], list[dict[str, Any]]],
349 *,
350 stale_ok: bool = False,
351) -> list[dict[str, Any]]:
352 """The listing, rebuilt when the M3U files on disk have changed."""
354 with _PLAYLIST_LOCK:
355 with _DATA_LOCK:
356 generation = _PLAYLIST_GENERATION
357 hit = _PLAYLIST_CACHE.get(key)
358 # Whoever held the lock just walked: don't walk again after it.
359 if (
360 stale_ok
361 and hit is not None
362 and _playlists_checked_recently(key)
363 ):
364 return list(hit[1])
365 fingerprint = playlist_listing_fingerprint(download_dir)
366 with _DATA_LOCK:
367 hit = _PLAYLIST_CACHE.get(key)
368 if hit is not None and hit[0] == fingerprint:
369 _PLAYLIST_CHECKED[key] = time.monotonic()
370 return list(hit[1])
371 playlists = build_fn()
372 with _DATA_LOCK:
373 if generation == _PLAYLIST_GENERATION:
374 _PLAYLIST_CACHE[key] = (fingerprint, playlists)
375 _PLAYLIST_CHECKED[key] = time.monotonic()
376 return list(playlists)
379def _recheck_playlists_in_background(
380 key: str,
381 download_dir: Path,
382 build_fn: Callable[[], list[dict[str, Any]]],
383) -> None:
384 """Run :func:`_check_playlists` on a daemon thread, one per listing."""
386 with _DATA_LOCK:
387 if key in _PLAYLIST_RECHECKING:
388 return
389 _PLAYLIST_RECHECKING.add(key)
391 def worker() -> None:
392 try:
393 _check_playlists(key, download_dir, build_fn)
394 except Exception:
395 logger.opt(exception=True).debug(
396 'Background playlist listing check failed'
397 )
398 finally:
399 with _DATA_LOCK:
400 _PLAYLIST_RECHECKING.discard(key)
402 threading.Thread(
403 target=worker, name='downtify-playlist-recheck', daemon=True
404 ).start()
407def get_cached_track_entries(
408 ctx: LibraryContext,
409) -> Optional[list[dict[str, Any]]]:
410 """Folded ``GET /tracks`` rows when the tree fingerprint still matches."""
412 key = _cache_key(ctx)
413 fingerprint = _ctx_fingerprint(ctx)
414 with _DATA_LOCK:
415 hit = _ENTRIES_CACHE.get(key)
416 if hit is not None and hit[0] == fingerprint:
417 return list(hit[1])
418 snap = _try_sqlite(key, fingerprint)
419 if snap is not None:
420 return list(snap['entries'])
421 return None
424#: Fingerprint of a RAM listing patched in place (one file added or
425#: removed): never equal to a real one, so the next live read rebuilds.
426PROVISIONAL_FINGERPRINT = ''
429def store_cached_track_entries(
430 ctx: LibraryContext,
431 entries: list[dict[str, Any]],
432 playlists: Optional[list[dict[str, Any]]] = None,
433 paths: Optional[list[tuple[str, str]]] = None,
434 *,
435 provisional: bool = False,
436) -> None:
437 """Remember a folded listing in RAM and in sqlite.
439 *provisional* (a download or delete patching the listing) keeps it in
440 RAM only, without fingerprinting the tree: walking every library
441 folder and rewriting the whole snapshot for each downloaded track
442 was slow on a NAS. The refresh that follows the change (see
443 :func:`invalidate_library_paths_cache`) stores the real one.
444 """
446 key = _cache_key(ctx)
447 fingerprint = (
448 PROVISIONAL_FINGERPRINT if provisional else _ctx_fingerprint(ctx)
449 )
450 copied = list(entries)
451 with _DATA_LOCK:
452 _ENTRIES_CACHE[key] = (fingerprint, copied)
453 if paths is not None:
454 _PATH_CACHE[key] = (fingerprint, list(paths))
455 path_rows = (
456 paths
457 if paths is not None
458 else list(_PATH_CACHE.get(key, (fingerprint, []))[1])
459 )
460 playlist_rows = (
461 playlists
462 if playlists is not None
463 else list(_PLAYLIST_CACHE.get(key, (fingerprint, []))[1])
464 )
465 if _STORE is None or provisional:
466 return
467 try:
468 _STORE.save(key, fingerprint, path_rows, playlist_rows, copied)
469 except Exception:
470 logger.opt(exception=True).debug(
471 'Could not persist library listing snapshot'
472 )
475#: Called on every invalidation - i.e. whenever the library changed - with
476#: no arguments, from whichever thread invalidated. See
477#: :func:`add_invalidation_listener`.
478_LISTENERS: list[Callable[[], None]] = []
481def add_invalidation_listener(fn: Callable[[], None]) -> None:
482 """Call *fn* whenever the library changes (e.g. to tell connected
483 apps to sync). It must be quick and thread-safe."""
485 if fn not in _LISTENERS:
486 _LISTENERS.append(fn)
489def invalidate_library_paths_cache(
490 *, notify: bool = True, drop_entries: bool = False
491) -> None:
492 """Mark listings stale after downloads, deletes, or path reconcile.
494 Playlist RAM is always dropped (an M3U scan is cheap). The folded
495 track snapshot stays unless *drop_entries* is set, so UI routes can
496 keep serving it while a background refresh catches up.
498 *notify* ``False`` is for a rescan that isn't a change in itself (a
499 client asking for a fresh scan): the listeners aren't told.
500 """
502 with _DATA_LOCK:
503 _drop_playlist_listings()
504 if drop_entries:
505 _PATH_CACHE.clear()
506 _ENTRIES_CACHE.clear()
507 if drop_entries and _STORE is not None:
508 try:
509 _STORE.clear()
510 except Exception:
511 logger.opt(exception=True).debug(
512 'Could not clear library listing snapshot'
513 )
514 refresh = _REFRESH_FN
515 if refresh is not None:
516 start_background_listing_refresh(refresh)
517 if not notify:
518 return
519 for listener in list(_LISTENERS):
520 try:
521 listener()
522 except Exception:
523 logger.opt(exception=True).debug('Library change listener failed')