Coverage for downtify/storage_scan.py: 96%

165 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""What Downtify's folders hold besides library songs. 

2 

3The library listing only sees audio files. This walks the folders 

4Downtify writes to - the downloads folder, slskd's folder and the 

5dedicated lyrics folder - once, and sorts every file into a kind 

6(:data:`KINDS`), so the library usage page can say what else takes room 

7and what is safe to clean up: 

8 

9* *leftovers* - temporary files an interrupted download, conversion or 

10 library upgrade left behind (:func:`is_temporary`), only once they are 

11 older than :data:`LEFTOVER_MIN_AGE` (a younger one may be in use); 

12* *orphans* - sidecars whose song is gone: an ``.lrc`` with no audio 

13 file of the same name beside it (or, in the lyrics folder, no library 

14 song at its path) and an album ``cover.jpg`` in a folder with no audio 

15 file and no playlist left. 

16 

17Nothing is deleted. ``GET /api/library/usage/scan`` (``downtify/api.py``) 

18runs it in a thread and keeps the result for a few minutes. 

19""" 

20 

21from __future__ import annotations 

22 

23import os 

24import re 

25import time 

26from collections.abc import Collection, Iterator 

27from dataclasses import dataclass, field 

28from pathlib import Path 

29from typing import Any, Optional 

30 

31from .library_catalog import ( 

32 AUDIO_EXTENSIONS, 

33 IMAGE_EXTENSIONS, 

34 PODCASTS_DIRNAME, 

35 UPGRADE_STAGING_MARKER, 

36) 

37from .library_cleanup import ALBUM_COVER_FILENAME 

38 

39#: Every kind a file is sorted into. 

40KINDS = ( 

41 'audio', 

42 'podcasts', 

43 'lyrics', 

44 'images', 

45 'playlists', 

46 'metadata', 

47 'temporary', 

48 'other', 

49) 

50#: A temporary file younger than this may still be written to. 

51LEFTOVER_MIN_AGE = 3600.0 

52#: How many leftovers / orphans are listed by name (biggest first). 

53EXAMPLES = 50 

54#: Saved artist photos, banners and profiles (``.metadata/...``). 

55METADATA_DIRNAME = '.metadata' 

56 

57PLAYLIST_EXTENSIONS = frozenset({'.m3u', '.m3u8'}) 

58#: yt-dlp's partial downloads and fragments (``.part``, ``.part-Frag3``), 

59#: its resume notes (``.ytdl``), ffmpeg's ``name.temp.mp3`` while it 

60#: converts, and the ``.tmp`` of atomic writes. 

61_TEMPORARY = re.compile( 

62 r'(\.part(-frag\d+)?|\.ytdl|\.tmp|\.temp(\.\w{1,5})?)$', re.IGNORECASE 

63) 

64 

65 

66def is_temporary(name: str) -> bool: 

67 """Whether *name* is a file only ever meant to exist for a while.""" 

68 

69 return UPGRADE_STAGING_MARKER in name or bool(_TEMPORARY.search(name)) 

70 

71 

72@dataclass(frozen=True) 

73class Folder: 

74 """One folder to walk: ``id`` names it, ``skip`` are folders inside 

75 it walked on their own (or not at all).""" 

76 

77 id: str 

78 path: Path 

79 skip: tuple[Path, ...] = () 

80 

81 

82@dataclass 

83class _File: 

84 rel: str 

85 name: str 

86 size: int 

87 mtime: float 

88 

89 

90@dataclass 

91class _Tally: 

92 files: int = 0 

93 bytes: int = 0 

94 kinds: dict[str, list[int]] = field( 

95 default_factory=lambda: {kind: [0, 0] for kind in KINDS} 

96 ) 

97 

98 def add(self, kind: str, size: int) -> None: 

99 self.files += 1 

100 self.bytes += size 

101 self.kinds[kind][0] += 1 

102 self.kinds[kind][1] += size 

103 

104 

105def _walk( 

106 root: Path, skip: Collection[Path] 

107) -> Iterator[tuple[str, list[_File]]]: 

108 """``(folder relative to root, [_File])`` for every folder under 

109 *root*, symlinks not followed.""" 

110 

111 pending = [root] 

112 while pending: 

113 current = pending.pop() 

114 files: list[_File] = [] 

115 try: 

116 entries = list(os.scandir(current)) 

117 except OSError: 

118 continue 

119 rel_dir = '' if current == root else current.relative_to(root) 

120 for entry in entries: 

121 try: 

122 if entry.is_dir(follow_symlinks=False): 

123 path = Path(entry.path) 

124 if path not in skip: 

125 pending.append(path) 

126 continue 

127 if not entry.is_file(follow_symlinks=False): 

128 continue 

129 stat = entry.stat(follow_symlinks=False) 

130 except OSError: 

131 continue 

132 rel = f'{rel_dir}/{entry.name}' if rel_dir else entry.name 

133 files.append( 

134 _File( 

135 str(rel).replace('\\', '/'), 

136 entry.name, 

137 stat.st_size, 

138 stat.st_mtime, 

139 ) 

140 ) 

141 yield str(rel_dir).replace('\\', '/'), files 

142 

143 

144def _suffix(name: str) -> str: 

145 return os.path.splitext(name)[1].lower() 

146 

147 

148def _stem(name: str) -> str: 

149 return os.path.splitext(name)[0] 

150 

151 

152def _kind(folder_rel: str, item: _File, is_downloads: bool) -> str: 

153 if is_temporary(item.name): 

154 return 'temporary' 

155 top = folder_rel.split('/', 1)[0] 

156 if is_downloads and top == PODCASTS_DIRNAME: 

157 return 'podcasts' 

158 if top == METADATA_DIRNAME: 

159 return 'metadata' 

160 suffix = _suffix(item.name) 

161 if suffix in AUDIO_EXTENSIONS: 

162 return 'audio' 

163 if suffix == '.lrc': 

164 return 'lyrics' 

165 if suffix in IMAGE_EXTENSIONS: 

166 return 'images' 

167 if suffix in PLAYLIST_EXTENSIONS: 

168 return 'playlists' 

169 return 'other' 

170 

171 

172@dataclass 

173class _Found: 

174 files: int = 0 

175 bytes: int = 0 

176 by_kind: dict[str, int] = field(default_factory=dict) 

177 items: list[dict[str, Any]] = field(default_factory=list) 

178 

179 def add(self, folder: str, item: _File, kind: str) -> None: 

180 self.files += 1 

181 self.bytes += item.size 

182 self.by_kind[kind] = self.by_kind.get(kind, 0) + 1 

183 self.items.append({ 

184 'folder': folder, 

185 'file': item.rel, 

186 'bytes': item.size, 

187 'kind': kind, 

188 'modified': int(item.mtime), 

189 }) 

190 

191 def report(self) -> dict[str, Any]: 

192 largest = sorted( 

193 self.items, key=lambda i: (-i['bytes'], i['folder'], i['file']) 

194 ) 

195 return { 

196 'files': self.files, 

197 'bytes': self.bytes, 

198 'by_kind': dict(sorted(self.by_kind.items())), 

199 'examples': largest[:EXAMPLES], 

200 } 

201 

202 

203def _orphans_in( 

204 folder_rel: str, 

205 files: list[_File], 

206 library_stems: Optional[Collection[str]], 

207) -> list[tuple[_File, str]]: 

208 """The sidecars in one folder whose song is gone. 

209 

210 In the lyrics folder (*library_stems* given) an ``.lrc`` belongs to 

211 the library song at its own path; elsewhere, to the audio beside it. 

212 """ 

213 

214 stems = { 

215 _stem(item.name) 

216 for item in files 

217 if _suffix(item.name) in AUDIO_EXTENSIONS 

218 and not is_temporary(item.name) 

219 } 

220 playlists = any(_suffix(i.name) in PLAYLIST_EXTENSIONS for i in files) 

221 found: list[tuple[_File, str]] = [] 

222 for item in files: 

223 suffix = _suffix(item.name) 

224 if suffix == '.lrc': 

225 if library_stems is not None: 

226 if _stem(item.rel) not in library_stems: 

227 found.append((item, 'lyrics')) 

228 elif _stem(item.name) not in stems: 

229 found.append((item, 'lyrics')) 

230 elif ( 

231 library_stems is None 

232 and item.name == ALBUM_COVER_FILENAME 

233 and not stems 

234 and not playlists 

235 and folder_rel 

236 ): 

237 found.append((item, 'cover')) 

238 return found 

239 

240 

241def scan_folders( 

242 folders: list[Folder], 

243 *, 

244 lyrics: Optional[Folder] = None, 

245 library_stems: Collection[str] = (), 

246 now: Optional[float] = None, 

247) -> dict[str, Any]: 

248 """Walk *folders* (and the *lyrics* folder, whose ``.lrc`` files 

249 belong to the library songs at *library_stems* - their library 

250 paths without the extension). 

251 

252 ``folders`` lists what each holds (``{id, path, files, bytes, 

253 kinds: {kind: {files, bytes}}}``); ``leftovers`` and ``orphans`` 

254 count what could go, biggest first. 

255 """ 

256 

257 moment = time.time() if now is None else now 

258 leftovers = _Found() 

259 orphans = _Found() 

260 report: list[dict[str, Any]] = [] 

261 stems = set(library_stems) 

262 walks = [(folder, None) for folder in folders] 

263 if lyrics is not None: 

264 walks.append((lyrics, stems)) 

265 started = time.monotonic() 

266 for folder, owned in walks: 

267 tally = _Tally() 

268 is_downloads = folder.id == 'downloads' 

269 for folder_rel, files in _walk(folder.path, folder.skip): 

270 for item in files: 

271 kind = _kind(folder_rel, item, is_downloads) 

272 tally.add(kind, item.size) 

273 if kind == 'temporary' and ( 

274 moment - item.mtime >= LEFTOVER_MIN_AGE 

275 ): 

276 leftovers.add(folder.id, item, 'temporary') 

277 in_podcasts = ( 

278 is_downloads 

279 and folder_rel.split('/', 1)[0] == PODCASTS_DIRNAME 

280 ) 

281 if folder_rel.split('/', 1)[0] == METADATA_DIRNAME or in_podcasts: 

282 continue 

283 for item, kind in _orphans_in(folder_rel, files, owned): 

284 orphans.add(folder.id, item, kind) 

285 report.append({ 

286 'id': folder.id, 

287 'path': str(folder.path), 

288 'files': tally.files, 

289 'bytes': tally.bytes, 

290 'kinds': { 

291 kind: {'files': files, 'bytes': size} 

292 for kind, (files, size) in tally.kinds.items() 

293 if files 

294 }, 

295 }) 

296 return { 

297 'folders': report, 

298 'leftovers': leftovers.report(), 

299 'orphans': orphans.report(), 

300 'seconds': round(time.monotonic() - started, 2), 

301 } 

302 

303 

304def _resolved(path: Optional[Path]) -> Optional[Path]: 

305 if path is None: 

306 return None 

307 try: 

308 resolved = path.resolve() 

309 except OSError: 

310 return None 

311 return resolved if resolved.is_dir() else None 

312 

313 

314def _inside(path: Path, root: Path) -> bool: 

315 return path == root or root in path.parents 

316 

317 

318def plan_folders( 

319 download_dir: Path, 

320 slskd_dir: Optional[Path], 

321 lyrics_dir: Optional[Path], 

322) -> tuple[list[Folder], Optional[Folder]]: 

323 """``(folders, lyrics folder)`` to walk: each folder once, one inside 

324 another walked on its own, and the lyrics folder only when it holds 

325 none of the others (``.lrc`` files beside the songs are found 

326 walking the songs' folders).""" 

327 

328 downloads = _resolved(download_dir) 

329 if downloads is None: 

330 return [], None 

331 roots = {'downloads': downloads} 

332 slskd = _resolved(slskd_dir) 

333 if slskd is not None and slskd != downloads: 

334 roots['slskd'] = slskd 

335 lyrics = _resolved(lyrics_dir) 

336 if lyrics is not None and not any( 

337 _inside(root, lyrics) for root in roots.values() 

338 ): 

339 roots['lyrics'] = lyrics 

340 folders = { 

341 key: Folder( 

342 key, 

343 path, 

344 tuple( 

345 other 

346 for name, other in roots.items() 

347 if name != key and other != path and _inside(other, path) 

348 ), 

349 ) 

350 for key, path in roots.items() 

351 } 

352 lyrics_folder = folders.pop('lyrics', None) 

353 return list(folders.values()), lyrics_folder