Coverage for downtify/storage_scan.py: 96%
165 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""What Downtify's folders hold besides library songs.
3The library listing only sees audio files. This walks the folders
4Downtify writes to - the downloads folder, slskd's folder and the
5dedicated lyrics folder - once, and sorts every file into a kind
6(:data:`KINDS`), so the library usage page can say what else takes room
7and what is safe to clean up:
9* *leftovers* - temporary files an interrupted download, conversion or
10 library upgrade left behind (:func:`is_temporary`), only once they are
11 older than :data:`LEFTOVER_MIN_AGE` (a younger one may be in use);
12* *orphans* - sidecars whose song is gone: an ``.lrc`` with no audio
13 file of the same name beside it (or, in the lyrics folder, no library
14 song at its path) and an album ``cover.jpg`` in a folder with no audio
15 file and no playlist left.
17Nothing is deleted. ``GET /api/library/usage/scan`` (``downtify/api.py``)
18runs it in a thread and keeps the result for a few minutes.
19"""
21from __future__ import annotations
23import os
24import re
25import time
26from collections.abc import Collection, Iterator
27from dataclasses import dataclass, field
28from pathlib import Path
29from typing import Any, Optional
31from .library_catalog import (
32 AUDIO_EXTENSIONS,
33 IMAGE_EXTENSIONS,
34 PODCASTS_DIRNAME,
35 UPGRADE_STAGING_MARKER,
36)
37from .library_cleanup import ALBUM_COVER_FILENAME
39#: Every kind a file is sorted into.
40KINDS = (
41 'audio',
42 'podcasts',
43 'lyrics',
44 'images',
45 'playlists',
46 'metadata',
47 'temporary',
48 'other',
49)
50#: A temporary file younger than this may still be written to.
51LEFTOVER_MIN_AGE = 3600.0
52#: How many leftovers / orphans are listed by name (biggest first).
53EXAMPLES = 50
54#: Saved artist photos, banners and profiles (``.metadata/...``).
55METADATA_DIRNAME = '.metadata'
57PLAYLIST_EXTENSIONS = frozenset({'.m3u', '.m3u8'})
58#: yt-dlp's partial downloads and fragments (``.part``, ``.part-Frag3``),
59#: its resume notes (``.ytdl``), ffmpeg's ``name.temp.mp3`` while it
60#: converts, and the ``.tmp`` of atomic writes.
61_TEMPORARY = re.compile(
62 r'(\.part(-frag\d+)?|\.ytdl|\.tmp|\.temp(\.\w{1,5})?)$', re.IGNORECASE
63)
66def is_temporary(name: str) -> bool:
67 """Whether *name* is a file only ever meant to exist for a while."""
69 return UPGRADE_STAGING_MARKER in name or bool(_TEMPORARY.search(name))
72@dataclass(frozen=True)
73class Folder:
74 """One folder to walk: ``id`` names it, ``skip`` are folders inside
75 it walked on their own (or not at all)."""
77 id: str
78 path: Path
79 skip: tuple[Path, ...] = ()
82@dataclass
83class _File:
84 rel: str
85 name: str
86 size: int
87 mtime: float
90@dataclass
91class _Tally:
92 files: int = 0
93 bytes: int = 0
94 kinds: dict[str, list[int]] = field(
95 default_factory=lambda: {kind: [0, 0] for kind in KINDS}
96 )
98 def add(self, kind: str, size: int) -> None:
99 self.files += 1
100 self.bytes += size
101 self.kinds[kind][0] += 1
102 self.kinds[kind][1] += size
105def _walk(
106 root: Path, skip: Collection[Path]
107) -> Iterator[tuple[str, list[_File]]]:
108 """``(folder relative to root, [_File])`` for every folder under
109 *root*, symlinks not followed."""
111 pending = [root]
112 while pending:
113 current = pending.pop()
114 files: list[_File] = []
115 try:
116 entries = list(os.scandir(current))
117 except OSError:
118 continue
119 rel_dir = '' if current == root else current.relative_to(root)
120 for entry in entries:
121 try:
122 if entry.is_dir(follow_symlinks=False):
123 path = Path(entry.path)
124 if path not in skip:
125 pending.append(path)
126 continue
127 if not entry.is_file(follow_symlinks=False):
128 continue
129 stat = entry.stat(follow_symlinks=False)
130 except OSError:
131 continue
132 rel = f'{rel_dir}/{entry.name}' if rel_dir else entry.name
133 files.append(
134 _File(
135 str(rel).replace('\\', '/'),
136 entry.name,
137 stat.st_size,
138 stat.st_mtime,
139 )
140 )
141 yield str(rel_dir).replace('\\', '/'), files
144def _suffix(name: str) -> str:
145 return os.path.splitext(name)[1].lower()
148def _stem(name: str) -> str:
149 return os.path.splitext(name)[0]
152def _kind(folder_rel: str, item: _File, is_downloads: bool) -> str:
153 if is_temporary(item.name):
154 return 'temporary'
155 top = folder_rel.split('/', 1)[0]
156 if is_downloads and top == PODCASTS_DIRNAME:
157 return 'podcasts'
158 if top == METADATA_DIRNAME:
159 return 'metadata'
160 suffix = _suffix(item.name)
161 if suffix in AUDIO_EXTENSIONS:
162 return 'audio'
163 if suffix == '.lrc':
164 return 'lyrics'
165 if suffix in IMAGE_EXTENSIONS:
166 return 'images'
167 if suffix in PLAYLIST_EXTENSIONS:
168 return 'playlists'
169 return 'other'
172@dataclass
173class _Found:
174 files: int = 0
175 bytes: int = 0
176 by_kind: dict[str, int] = field(default_factory=dict)
177 items: list[dict[str, Any]] = field(default_factory=list)
179 def add(self, folder: str, item: _File, kind: str) -> None:
180 self.files += 1
181 self.bytes += item.size
182 self.by_kind[kind] = self.by_kind.get(kind, 0) + 1
183 self.items.append({
184 'folder': folder,
185 'file': item.rel,
186 'bytes': item.size,
187 'kind': kind,
188 'modified': int(item.mtime),
189 })
191 def report(self) -> dict[str, Any]:
192 largest = sorted(
193 self.items, key=lambda i: (-i['bytes'], i['folder'], i['file'])
194 )
195 return {
196 'files': self.files,
197 'bytes': self.bytes,
198 'by_kind': dict(sorted(self.by_kind.items())),
199 'examples': largest[:EXAMPLES],
200 }
203def _orphans_in(
204 folder_rel: str,
205 files: list[_File],
206 library_stems: Optional[Collection[str]],
207) -> list[tuple[_File, str]]:
208 """The sidecars in one folder whose song is gone.
210 In the lyrics folder (*library_stems* given) an ``.lrc`` belongs to
211 the library song at its own path; elsewhere, to the audio beside it.
212 """
214 stems = {
215 _stem(item.name)
216 for item in files
217 if _suffix(item.name) in AUDIO_EXTENSIONS
218 and not is_temporary(item.name)
219 }
220 playlists = any(_suffix(i.name) in PLAYLIST_EXTENSIONS for i in files)
221 found: list[tuple[_File, str]] = []
222 for item in files:
223 suffix = _suffix(item.name)
224 if suffix == '.lrc':
225 if library_stems is not None:
226 if _stem(item.rel) not in library_stems:
227 found.append((item, 'lyrics'))
228 elif _stem(item.name) not in stems:
229 found.append((item, 'lyrics'))
230 elif (
231 library_stems is None
232 and item.name == ALBUM_COVER_FILENAME
233 and not stems
234 and not playlists
235 and folder_rel
236 ):
237 found.append((item, 'cover'))
238 return found
241def scan_folders(
242 folders: list[Folder],
243 *,
244 lyrics: Optional[Folder] = None,
245 library_stems: Collection[str] = (),
246 now: Optional[float] = None,
247) -> dict[str, Any]:
248 """Walk *folders* (and the *lyrics* folder, whose ``.lrc`` files
249 belong to the library songs at *library_stems* - their library
250 paths without the extension).
252 ``folders`` lists what each holds (``{id, path, files, bytes,
253 kinds: {kind: {files, bytes}}}``); ``leftovers`` and ``orphans``
254 count what could go, biggest first.
255 """
257 moment = time.time() if now is None else now
258 leftovers = _Found()
259 orphans = _Found()
260 report: list[dict[str, Any]] = []
261 stems = set(library_stems)
262 walks = [(folder, None) for folder in folders]
263 if lyrics is not None:
264 walks.append((lyrics, stems))
265 started = time.monotonic()
266 for folder, owned in walks:
267 tally = _Tally()
268 is_downloads = folder.id == 'downloads'
269 for folder_rel, files in _walk(folder.path, folder.skip):
270 for item in files:
271 kind = _kind(folder_rel, item, is_downloads)
272 tally.add(kind, item.size)
273 if kind == 'temporary' and (
274 moment - item.mtime >= LEFTOVER_MIN_AGE
275 ):
276 leftovers.add(folder.id, item, 'temporary')
277 in_podcasts = (
278 is_downloads
279 and folder_rel.split('/', 1)[0] == PODCASTS_DIRNAME
280 )
281 if folder_rel.split('/', 1)[0] == METADATA_DIRNAME or in_podcasts:
282 continue
283 for item, kind in _orphans_in(folder_rel, files, owned):
284 orphans.add(folder.id, item, kind)
285 report.append({
286 'id': folder.id,
287 'path': str(folder.path),
288 'files': tally.files,
289 'bytes': tally.bytes,
290 'kinds': {
291 kind: {'files': files, 'bytes': size}
292 for kind, (files, size) in tally.kinds.items()
293 if files
294 },
295 })
296 return {
297 'folders': report,
298 'leftovers': leftovers.report(),
299 'orphans': orphans.report(),
300 'seconds': round(time.monotonic() - started, 2),
301 }
304def _resolved(path: Optional[Path]) -> Optional[Path]:
305 if path is None:
306 return None
307 try:
308 resolved = path.resolve()
309 except OSError:
310 return None
311 return resolved if resolved.is_dir() else None
314def _inside(path: Path, root: Path) -> bool:
315 return path == root or root in path.parents
318def plan_folders(
319 download_dir: Path,
320 slskd_dir: Optional[Path],
321 lyrics_dir: Optional[Path],
322) -> tuple[list[Folder], Optional[Folder]]:
323 """``(folders, lyrics folder)`` to walk: each folder once, one inside
324 another walked on its own, and the lyrics folder only when it holds
325 none of the others (``.lrc`` files beside the songs are found
326 walking the songs' folders)."""
328 downloads = _resolved(download_dir)
329 if downloads is None:
330 return [], None
331 roots = {'downloads': downloads}
332 slskd = _resolved(slskd_dir)
333 if slskd is not None and slskd != downloads:
334 roots['slskd'] = slskd
335 lyrics = _resolved(lyrics_dir)
336 if lyrics is not None and not any(
337 _inside(root, lyrics) for root in roots.values()
338 ):
339 roots['lyrics'] = lyrics
340 folders = {
341 key: Folder(
342 key,
343 path,
344 tuple(
345 other
346 for name, other in roots.items()
347 if name != key and other != path and _inside(other, path)
348 ),
349 )
350 for key, path in roots.items()
351 }
352 lyrics_folder = folders.pop('lyrics', None)
353 return list(folders.values()), lyrics_folder