Coverage for downtify/library_reconcile.py: 42%

229 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Detect library file moves and refresh playlist paths + M3U / Navidrome.""" 

2 

3from __future__ import annotations 

4 

5from pathlib import Path 

6from typing import TYPE_CHECKING, Any, Optional 

7 

8import httpx 

9from loguru import logger 

10 

11from . import m3u 

12from .downloader import Downloader 

13from .library_cache_keys import file_content_key 

14from .library_catalog import ( 

15 AUDIO_EXTENSIONS, 

16 PODCASTS_DIRNAME, 

17 LibraryContext, 

18 library_context_from_state, 

19) 

20from .library_delete import ( 

21 TagMismatchDeleteContext, 

22 delete_if_spotify_tag_mismatch, 

23) 

24from .library_paths import library_stored_path, locate_library_file 

25from .library_paths_cache import invalidate_library_paths_cache 

26from .navidrome import ( 

27 _effective_navidrome_settings, 

28 enrich_song_from_library_file, 

29 remove_navidrome_playlist_by_name, 

30 sync_playlist_to_navidrome, 

31) 

32from .playlist_catalog import PlaylistCatalog 

33from .playlist_spotify_cache import PlaylistSpotifyCache, fetch_playlist_tracks 

34from .track_index import TrackIndex, normalize_spotify_track_id 

35from .track_tag_match import spotify_aligns_with_file_tags 

36 

37if TYPE_CHECKING: 

38 from .monitor import PlaylistMonitorDB 

39 

40 

41def playlist_refresh_enabled(settings: dict[str, Any]) -> tuple[bool, bool]: 

42 """Return ``(generate_m3u, sync_navidrome)`` from current settings.""" 

43 

44 generate_m3u = settings.get('generate_m3u', True) is not False 

45 sync_navidrome = settings.get( 

46 'sync_navidrome', True 

47 ) is not False and _effective_navidrome_settings(settings).get('enabled') 

48 return generate_m3u, sync_navidrome 

49 

50 

51def build_disk_content_index(ctx: LibraryContext) -> dict[str, str]: 

52 """Scan library trees: ``content_key`` → current stored relative path.""" 

53 

54 index: dict[str, str] = {} 

55 

56 def _add(file_path: Path) -> None: 

57 if not file_path.is_file(): 

58 return 

59 if file_path.suffix.lower() not in AUDIO_EXTENSIONS: 

60 return 

61 try: 

62 file_path.relative_to(ctx.download_dir / PODCASTS_DIRNAME) 

63 return # podcast episode: not part of the music library 

64 except ValueError: 

65 pass 

66 ck = file_content_key(file_path) 

67 if not ck: 

68 return 

69 stored = library_stored_path( 

70 file_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

71 ) 

72 index[ck] = stored 

73 

74 if ctx.download_dir.is_dir(): 

75 for path in ctx.download_dir.rglob('*'): 

76 _add(path) 

77 

78 if ctx.slskd_dir is not None and ctx.slskd_dir.is_dir(): 

79 try: 

80 same = ctx.slskd_dir.resolve() == ctx.download_dir.resolve() 

81 except OSError: 

82 same = False 

83 if not same: 

84 for path in ctx.slskd_dir.rglob('*'): 

85 _add(path) 

86 

87 for extra in ctx.extra_dirs: 

88 if not extra.is_dir(): 

89 continue 

90 try: 

91 extra_resolved = extra.resolve() 

92 except OSError: 

93 extra_resolved = extra 

94 try: 

95 download_resolved = ctx.download_dir.resolve() 

96 except OSError: 

97 download_resolved = ctx.download_dir 

98 if extra_resolved == download_resolved: 

99 continue 

100 for path in extra.rglob('*'): 

101 _add(path) 

102 

103 return index 

104 

105 

106def reconcile_library_paths( 

107 ctx: LibraryContext, 

108 *, 

109 track_index: Optional[TrackIndex] = None, 

110 playlist_catalog: Optional[PlaylistCatalog] = None, 

111 monitor_db: Optional['PlaylistMonitorDB'] = None, 

112) -> tuple[int, set[str]]: 

113 """Update stored paths when files moved but ``content_key`` is unchanged. 

114 

115 Returns ``(paths_updated, affected_playlist_names)``. 

116 """ 

117 

118 disk = build_disk_content_index(ctx) 

119 if not disk: 

120 return 0, set() 

121 

122 updated = 0 

123 affected: set[str] = set() 

124 

125 if track_index is not None: 

126 for row in track_index.all_rows(): 

127 ck = str(row.get('content_key') or '').strip() 

128 old_path = str(row.get('filename') or '').strip() 

129 tid = str(row.get('spotify_track_id') or '').strip() 

130 if not ck or not tid: 

131 continue 

132 new_path = disk.get(ck) 

133 if not new_path or new_path == old_path: 

134 continue 

135 if locate_library_file( 

136 old_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

137 ): 

138 continue 

139 if track_index.update_filename(tid, new_path, content_key=ck): 

140 updated += 1 

141 if monitor_db is not None: 

142 affected |= monitor_db.update_filename_for_spotify( 

143 tid, new_path 

144 ) 

145 

146 if playlist_catalog is not None: 

147 for row in playlist_catalog.all_track_rows(): 

148 ck = str(row.get('content_key') or '').strip() 

149 old_path = str(row.get('filename') or '').strip() 

150 if not ck: 

151 continue 

152 new_path = disk.get(ck) 

153 if not new_path or new_path == old_path: 

154 continue 

155 if locate_library_file( 

156 old_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

157 ): 

158 continue 

159 names = playlist_catalog.update_filename_by_content_key( 

160 ck, new_path 

161 ) 

162 if names: 

163 updated += 1 

164 affected.update(names) 

165 

166 if updated: 

167 logger.info( 

168 'library reconcile: updated {} path(s), {} playlist(s) affected', 

169 updated, 

170 len(affected), 

171 ) 

172 return updated, affected 

173 

174 

175def prune_stale_and_backfill( 

176 ctx: LibraryContext, 

177 *, 

178 track_index: Optional[TrackIndex] = None, 

179 playlist_catalog: Optional[PlaylistCatalog] = None, 

180 navidrome_index: Optional[Any] = None, 

181) -> tuple[int, int, set[str]]: 

182 """Remove DB rows for missing files; fill missing ``content_key`` values.""" 

183 

184 pruned = 0 

185 backfilled = 0 

186 affected: set[str] = set() 

187 

188 if playlist_catalog is not None: 

189 for row in list(playlist_catalog.all_track_rows()): 

190 old_path = str(row.get('filename') or '').strip() 

191 pl_name = str(row.get('playlist_name') or '').strip() 

192 tid = str(row.get('track_spotify_id') or '').strip() 

193 if not old_path or not pl_name or not tid: 

194 continue 

195 full = locate_library_file( 

196 old_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

197 ) 

198 if full is None: 

199 if playlist_catalog.remove_track(pl_name, tid): 

200 pruned += 1 

201 affected.add(pl_name) 

202 if navidrome_index is not None: 

203 navidrome_index.forget_filename(old_path) 

204 continue 

205 ck = str(row.get('content_key') or '').strip() 

206 if not ck: 

207 new_ck = file_content_key(full) 

208 if new_ck: 

209 playlist_catalog.set_content_key(pl_name, tid, new_ck) 

210 backfilled += 1 

211 

212 if track_index is not None: 

213 for row in list(track_index.all_rows()): 

214 old_path = str(row.get('filename') or '').strip() 

215 tid = str(row.get('spotify_track_id') or '').strip() 

216 if not old_path or not tid: 

217 continue 

218 full = locate_library_file( 

219 old_path, ctx.download_dir, ctx.slskd_dir, ctx.extra_dirs 

220 ) 

221 if full is None: 

222 if track_index.remove_by_filename(old_path): 

223 pruned += 1 

224 if navidrome_index is not None: 

225 navidrome_index.forget_filename(old_path) 

226 continue 

227 ck = str(row.get('content_key') or '').strip() 

228 if not ck: 

229 new_ck = file_content_key(full) 

230 if new_ck and track_index.set_content_key(tid, new_ck): 

231 backfilled += 1 

232 

233 if pruned or backfilled: 

234 logger.info( 

235 'library reconcile: pruned {} stale row(s), backfilled {} content_key(s)', 

236 pruned, 

237 backfilled, 

238 ) 

239 return pruned, backfilled, affected 

240 

241 

242def refresh_playlists_after_moves( # noqa: PLR0914 

243 playlist_names: set[str], 

244 *, 

245 settings: dict[str, Any], 

246 downloader: Downloader, 

247 playlist_catalog: PlaylistCatalog, 

248 track_index: Optional[TrackIndex] = None, 

249 monitor_db: Optional['PlaylistMonitorDB'] = None, 

250 navidrome_index: Optional[Any] = None, 

251 navidrome_scan: bool = False, 

252 playlist_spotify_cache: Optional[PlaylistSpotifyCache] = None, 

253 cover_cache: Optional[Any] = None, 

254 metadata_cache: Optional[Any] = None, 

255 delete_tag_mismatches: bool = True, 

256) -> None: 

257 """Regenerate local M3U and optionally Navidrome for *playlist_names*. 

258 

259 Library scan is off by default here (deletes / reconcile). New downloads 

260 call ``sync_playlist_to_navidrome`` directly with scan from settings. 

261 """ 

262 

263 if not playlist_names: 

264 return 

265 

266 generate_m3u, sync_navidrome = playlist_refresh_enabled(settings) 

267 if not generate_m3u and not sync_navidrome: 

268 logger.info( 

269 'library reconcile: skip playlist refresh (M3U and Navidrome sync off)' 

270 ) 

271 return 

272 

273 logger.info( 

274 'library reconcile: refreshing {} playlist(s) after library change: {}', 

275 len(playlist_names), 

276 ', '.join(sorted(playlist_names)[:8]) 

277 + ('; ...' if len(playlist_names) > 8 else ''), 

278 ) 

279 

280 download_dir = Path(downloader.download_dir) 

281 slskd_dir = ( 

282 Path(str(settings.get('slskd', {}).get('source_dir') or '')) 

283 if isinstance(settings.get('slskd'), dict) 

284 and settings.get('slskd', {}).get('source_dir') 

285 else None 

286 ) 

287 organize = bool( 

288 settings.get('organize_by_artist', False) 

289 or settings.get('organize_by_album', False) 

290 ) 

291 mismatch_delete: Optional[TagMismatchDeleteContext] = None 

292 if delete_tag_mismatches: 

293 mismatch_delete = TagMismatchDeleteContext( 

294 ctx=library_context_from_state( 

295 download_dir, 

296 settings, 

297 track_index=track_index, 

298 metadata_cache=metadata_cache, 

299 playlist_catalog=playlist_catalog, 

300 ), 

301 cover_cache=cover_cache, 

302 metadata_cache=metadata_cache, 

303 playlist_catalog=playlist_catalog, 

304 track_index=track_index, 

305 navidrome_index=navidrome_index, 

306 download_sources=getattr(downloader, 'download_sources', None), 

307 ) 

308 

309 for name in sorted(playlist_names): 

310 catalog_rows = playlist_catalog.list_tracks(name) 

311 if not catalog_rows: 

312 if sync_navidrome: 

313 try: 

314 remove_navidrome_playlist_by_name(name, settings) 

315 except Exception: 

316 logger.exception( 

317 'library reconcile: Navidrome remove failed for {!r}', 

318 name, 

319 ) 

320 continue 

321 

322 spotify_tracks: list[dict[str, Any]] = [] 

323 spotify_id = playlist_catalog.spotify_id_for_playlist(name) 

324 if spotify_id: 

325 try: 

326 _pl_name, spotify_tracks = fetch_playlist_tracks( 

327 spotify_id, 

328 cache=playlist_spotify_cache, 

329 ) 

330 except httpx.HTTPStatusError as exc: 

331 status = exc.response.status_code 

332 logger.warning( 

333 'Playlist refresh: Spotify embed HTTP {} for {!r} ' 

334 '(id={}); using catalog metadata only', 

335 status, 

336 name, 

337 spotify_id, 

338 ) 

339 spotify_tracks = [] 

340 except Exception: 

341 logger.opt(exception=True).warning( 

342 'Playlist refresh: Spotify fetch failed for {!r}', name 

343 ) 

344 spotify_tracks = [] 

345 

346 by_id = { 

347 normalize_spotify_track_id(t): t 

348 for t in spotify_tracks 

349 if normalize_spotify_track_id(t) 

350 } 

351 

352 entries: list[dict[str, Any]] = [] 

353 songs_for_nav: list[dict[str, Any]] = [] 

354 for row in catalog_rows: 

355 tid = row['track_spotify_id'] 

356 filename = row['filename'] 

357 if not locate_library_file(filename, download_dir, slskd_dir): 

358 continue 

359 meta = by_id.get(tid) or {'song_id': tid} 

360 song = dict(meta) 

361 song['filename'] = filename 

362 song = enrich_song_from_library_file(song, download_dir, slskd_dir) 

363 if not spotify_aligns_with_file_tags(song): 

364 if ( 

365 mismatch_delete is not None 

366 and delete_if_spotify_tag_mismatch( 

367 song, 

368 mismatch_delete, 

369 playlist_name=name, 

370 ) 

371 ): 

372 continue 

373 entries.append({ 

374 'filename': filename, 

375 'title': song.get('name') or '', 

376 'artist': ', '.join(song.get('artists') or []), 

377 'duration': song.get('duration') or 0, 

378 }) 

379 songs_for_nav.append(song) 

380 

381 if generate_m3u and entries: 

382 pl_subdir = None if organize else m3u.sanitize_playlist_name(name) 

383 try: 

384 m3u.write_m3u( 

385 download_dir, 

386 name, 

387 entries, 

388 playlist_subdir=pl_subdir, 

389 slskd_dir=slskd_dir, 

390 ) 

391 logger.info( 

392 'library reconcile: rewrote M3U for playlist {!r} ({} tracks)', 

393 name, 

394 len(entries), 

395 ) 

396 except Exception: 

397 logger.exception( 

398 'library reconcile: M3U failed for playlist {!r}', name 

399 ) 

400 

401 if sync_navidrome and songs_for_nav: 

402 try: 

403 sync_playlist_to_navidrome( 

404 name, 

405 songs_for_nav, 

406 settings, 

407 navidrome_index=navidrome_index, 

408 download_dir=download_dir, 

409 trigger_scan=navidrome_scan, 

410 delete_tag_mismatches=mismatch_delete, 

411 ) 

412 except Exception: 

413 logger.exception( 

414 'library reconcile: Navidrome sync failed for {!r}', name 

415 ) 

416 

417 

418def reconcile_and_refresh( 

419 download_dir: Path, 

420 settings: dict[str, Any], 

421 downloader: Downloader, 

422 *, 

423 track_index: Optional[TrackIndex] = None, 

424 playlist_catalog: Optional[PlaylistCatalog] = None, 

425 monitor_db: Optional['PlaylistMonitorDB'] = None, 

426 navidrome_index: Optional[Any] = None, 

427 refresh_playlists: bool = True, 

428 playlist_spotify_cache: Optional[PlaylistSpotifyCache] = None, 

429 cover_cache: Optional[Any] = None, 

430 metadata_cache: Optional[Any] = None, 

431) -> dict[str, Any]: 

432 """Run path reconciliation and optional playlist / Navidrome refresh.""" 

433 

434 ctx = library_context_from_state( 

435 download_dir, settings, track_index=track_index 

436 ) 

437 pruned, backfilled, prune_affected = prune_stale_and_backfill( 

438 ctx, 

439 track_index=track_index, 

440 playlist_catalog=playlist_catalog, 

441 navidrome_index=navidrome_index, 

442 ) 

443 count, move_affected = reconcile_library_paths( 

444 ctx, 

445 track_index=track_index, 

446 playlist_catalog=playlist_catalog, 

447 monitor_db=monitor_db, 

448 ) 

449 affected = prune_affected | move_affected 

450 generate_m3u, sync_navidrome = playlist_refresh_enabled(settings) 

451 did_refresh = False 

452 if ( 

453 refresh_playlists 

454 and affected 

455 and playlist_catalog is not None 

456 and (generate_m3u or sync_navidrome) 

457 ): 

458 refresh_playlists_after_moves( 

459 affected, 

460 settings=settings, 

461 downloader=downloader, 

462 playlist_catalog=playlist_catalog, 

463 track_index=track_index, 

464 monitor_db=monitor_db, 

465 navidrome_index=navidrome_index, 

466 playlist_spotify_cache=playlist_spotify_cache, 

467 cover_cache=cover_cache, 

468 metadata_cache=metadata_cache, 

469 ) 

470 did_refresh = True 

471 invalidate_library_paths_cache() 

472 if not (count or pruned or backfilled): 

473 logger.info( 

474 'library reconcile: no path updates, stale rows, or backfills ' 

475 '(DB already matches disk, or delete was done in the Library UI)' 

476 ) 

477 return { 

478 'paths_updated': count, 

479 'pruned_stale': pruned, 

480 'content_keys_backfilled': backfilled, 

481 'playlists_affected': sorted(affected), 

482 'refresh_m3u': did_refresh and generate_m3u, 

483 'refresh_navidrome': did_refresh and sync_navidrome, 

484 }