Coverage for downtify/external_sync.py: 68%

173 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Background extra-folder sync and unmapping a folder from the library.""" 

2 

3from __future__ import annotations 

4 

5import json 

6import threading 

7import time 

8from collections.abc import Callable, Sequence 

9from datetime import datetime, timezone 

10from pathlib import Path 

11from typing import Any, Optional 

12 

13from loguru import logger 

14 

15from .external_library import ( 

16 AUDIO_EXTENSIONS, 

17 effective_external_library, 

18 sync_external_library, 

19) 

20from .library_catalog import LibraryContext 

21from .library_cleanup import delete_lrc_sidecar 

22from .library_paths import ( 

23 EXTERNAL_LIBRARY_PREFIX, 

24 extra_dir_id, 

25 library_stored_path, 

26 locate_library_file, 

27) 

28from .library_paths_cache import invalidate_library_paths_cache 

29from .manual_playlists import prune_files_from_manual_playlists 

30 

31STATE_IDLE = 'idle' 

32STATE_RUNNING = 'running' 

33STATE_DONE = 'done' 

34STATE_ERROR = 'error' 

35 

36 

37def _now_iso() -> str: 

38 return datetime.now(timezone.utc).replace(microsecond=0).isoformat() 

39 

40 

41def _empty_progress() -> dict[str, Any]: 

42 return {'done': 0, 'total': 0, 'current': ''} 

43 

44 

45class ExternalSyncJob: 

46 """One extra-folder sync at a time, with the last finished log on disk.""" 

47 

48 def __init__(self, path: Path) -> None: 

49 self.path = path 

50 self._lock = threading.Lock() 

51 self._live: dict[str, Any] = { 

52 'state': STATE_IDLE, 

53 'started_at': '', 

54 'finished_at': '', 

55 'progress': _empty_progress(), 

56 'error': '', 

57 'result': None, 

58 } 

59 self._load() 

60 if self._live.get('state') == STATE_RUNNING: 

61 self._live['state'] = STATE_ERROR 

62 self._live['error'] = 'interrupted' 

63 self._live['finished_at'] = _now_iso() 

64 self._save_locked() 

65 

66 def snapshot(self) -> dict[str, Any]: 

67 with self._lock: 

68 return json.loads(json.dumps(self._live)) 

69 

70 def try_begin(self) -> bool: 

71 with self._lock: 

72 if self._live.get('state') == STATE_RUNNING: 

73 return False 

74 self._live['state'] = STATE_RUNNING 

75 self._live['started_at'] = _now_iso() 

76 self._live['finished_at'] = '' 

77 self._live['error'] = '' 

78 self._live['progress'] = _empty_progress() 

79 self._save_locked() 

80 return True 

81 

82 def set_progress(self, done: int, total: int, current: str = '') -> None: 

83 with self._lock: 

84 self._live['progress'] = { 

85 'done': int(done), 

86 'total': int(total), 

87 'current': str(current or ''), 

88 } 

89 

90 def finish( 

91 self, 

92 *, 

93 result: Optional[dict[str, Any]] = None, 

94 error: str = '', 

95 ) -> None: 

96 with self._lock: 

97 self._live['finished_at'] = _now_iso() 

98 self._live['error'] = str(error or '') 

99 if result is not None: 

100 self._live['result'] = result 

101 progress = self._live.get('progress') or _empty_progress() 

102 total = int(progress.get('total') or 0) 

103 if total: 

104 progress['done'] = total 

105 progress['current'] = '' 

106 self._live['progress'] = progress 

107 self._live['state'] = STATE_ERROR if error else STATE_DONE 

108 self._save_locked() 

109 

110 def _load(self) -> None: 

111 try: 

112 raw = json.loads(self.path.read_text(encoding='utf-8')) 

113 except (OSError, json.JSONDecodeError): 

114 return 

115 if not isinstance(raw, dict): 

116 return 

117 with self._lock: 

118 for key in ( 

119 'state', 

120 'started_at', 

121 'finished_at', 

122 'error', 

123 'result', 

124 ): 

125 if key in raw: 

126 self._live[key] = raw[key] 

127 progress = raw.get('progress') 

128 if isinstance(progress, dict): 

129 self._live['progress'] = { 

130 'done': int(progress.get('done') or 0), 

131 'total': int(progress.get('total') or 0), 

132 'current': str(progress.get('current') or ''), 

133 } 

134 

135 def _save_locked(self) -> None: 

136 payload = { 

137 'state': self._live.get('state') or STATE_IDLE, 

138 'started_at': self._live.get('started_at') or '', 

139 'finished_at': self._live.get('finished_at') or '', 

140 'error': self._live.get('error') or '', 

141 'result': self._live.get('result'), 

142 'progress': self._live.get('progress') or _empty_progress(), 

143 } 

144 try: 

145 self.path.parent.mkdir(parents=True, exist_ok=True) 

146 self.path.write_text( 

147 json.dumps(payload, indent=2), 

148 encoding='utf-8', 

149 ) 

150 except OSError: 

151 logger.opt(exception=True).warning( 

152 'Could not persist extra-folder sync status' 

153 ) 

154 

155 

156def run_external_sync( 

157 job: ExternalSyncJob, 

158 *, 

159 ctx: Any, 

160 download_dir: Path, 

161 settings: dict[str, Any], 

162 lang: str, 

163 image_kinds: tuple[str, ...], 

164 lyrics_cache: Optional[Any], 

165 cover_cache: Optional[Any] = None, 

166 on_update: Optional[Callable[[dict[str, Any]], None]] = None, 

167) -> dict[str, Any]: 

168 """Run a sync on the worker thread and keep *job* up to date.""" 

169 

170 last_push = 0.0 

171 

172 def _progress(done: int, total: int, current: str = '') -> None: 

173 nonlocal last_push 

174 job.set_progress(done, total, current) 

175 now = time.monotonic() 

176 if on_update is None: 

177 return 

178 if now - last_push < 0.4 and done < total: 

179 return 

180 last_push = now 

181 on_update(job.snapshot()) 

182 

183 try: 

184 result = sync_external_library( 

185 ctx, 

186 download_dir=download_dir, 

187 settings=settings, 

188 lang=lang, 

189 image_kinds=image_kinds, 

190 lyrics_cache=lyrics_cache, 

191 cover_cache=cover_cache, 

192 on_progress=_progress, 

193 ) 

194 except Exception as exc: 

195 logger.opt(exception=True).error('External library sync failed') 

196 job.finish(error=str(exc)[:300]) 

197 snapshot = job.snapshot() 

198 if on_update is not None: 

199 on_update(snapshot) 

200 return snapshot 

201 job.finish(result=result) 

202 snapshot = job.snapshot() 

203 if on_update is not None: 

204 on_update(snapshot) 

205 return snapshot 

206 

207 

208def extra_audio_paths(root: Path) -> list[Path]: 

209 if not root.is_dir(): 

210 return [] 

211 found: list[Path] = [] 

212 for path in root.rglob('*'): 

213 try: 

214 if path.is_file() and path.suffix.lower() in AUDIO_EXTENSIONS: 

215 found.append(path) 

216 except OSError: 

217 continue 

218 return found 

219 

220 

221def unmap_extra_folder( 

222 root: Path, 

223 *, 

224 download_dir: Path, 

225 slskd_dir: Optional[Path] = None, 

226 extra_dirs: Sequence[Path] = (), 

227 cover_cache: Optional[Any] = None, 

228 metadata_cache: Optional[Any] = None, 

229 playlist_catalog: Optional[Any] = None, 

230 track_index: Optional[Any] = None, 

231 likes: Optional[Any] = None, 

232 navidrome_index: Optional[Any] = None, 

233) -> dict[str, Any]: 

234 """Drop a mapped extra folder from the library without deleting audio.""" 

235 

236 extras = list(extra_dirs) 

237 try: 

238 want = root.resolve() 

239 except OSError: 

240 want = root 

241 if not any(_same_path(item, want) for item in extras): 

242 extras.append(root) 

243 folder_id = extra_dir_id(root) 

244 prefix = f'{EXTERNAL_LIBRARY_PREFIX}{folder_id}/' 

245 stored_set: set[str] = set() 

246 if track_index is not None: 

247 for name in track_index.list_filenames(): 

248 if str(name).startswith(prefix): 

249 stored_set.add(str(name)) 

250 for full in extra_audio_paths(root): 

251 stored_set.add( 

252 library_stored_path(full, download_dir, slskd_dir, extras) 

253 ) 

254 

255 unmapped = 0 

256 for stored in sorted(stored_set): 

257 full = locate_library_file(stored, download_dir, slskd_dir, extras) 

258 if full is not None: 

259 delete_lrc_sidecar(full) 

260 if cover_cache is not None: 

261 if full is not None: 

262 cover_cache.forget(stored, full_path=full) 

263 else: 

264 cover_cache.forget_by_stored_path(stored) 

265 if metadata_cache is not None: 

266 metadata_cache.forget(stored, full_path=full) 

267 if playlist_catalog is not None: 

268 playlist_catalog.remove_tracks_for_filename(stored) 

269 if track_index is not None: 

270 track_index.remove_by_filename(stored) 

271 if likes is not None: 

272 likes.unlike(stored) 

273 if navidrome_index is not None: 

274 navidrome_index.forget_filename(stored, full_path=full) 

275 unmapped += 1 

276 

277 ctx = LibraryContext( 

278 download_dir=download_dir, 

279 slskd_dir=slskd_dir, 

280 extra_dirs=tuple(extras), 

281 ) 

282 prune_files_from_manual_playlists(ctx, sorted(stored_set)) 

283 

284 invalidate_library_paths_cache() 

285 logger.info( 

286 'Unmapped extra folder {} ({} track{})', 

287 root, 

288 unmapped, 

289 '' if unmapped == 1 else 's', 

290 ) 

291 return { 

292 'folder': str(root), 

293 'folder_id': folder_id, 

294 'unmapped': unmapped, 

295 } 

296 

297 

298def _same_path(path: Path, other: Path) -> bool: 

299 try: 

300 return path.resolve() == other 

301 except OSError: 

302 return path == other 

303 

304 

305def folders_removed( 

306 before: dict[str, Any], after: dict[str, Any] 

307) -> list[Path]: 

308 """Absolute folders present in *before* and gone from *after*.""" 

309 

310 old_text = set(effective_external_library(before)['folders']) 

311 new_text = set(effective_external_library(after)['folders']) 

312 return [Path(item) for item in sorted(old_text - new_text)]