Coverage for downtify/library_usage.py: 96%
265 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Library storage and download usage (``GET /api/library/usage``).
3Pure functions over what the server already knows - the library listing
4rows (their ``size`` and file date), which provider fetched each file
5and when (:mod:`downtify.file_providers`), the download outcomes
6(:mod:`downtify.download_log`) and the daily size snapshots kept here
7(:class:`UsageSnapshots`). ``downtify/api.py`` does the I/O.
9What counts as the library: every file the Library lists, minus the
10*working copies* - transfers slskd left in its own folder after
11Downtify copied them into the library (recorded since this was kept),
12and, while slskd files aren't left in place, any other file in slskd's
13folder nothing records as a library song. Those copies are listed apart
14so a source copy is never taken for unique library content.
16Growth comes from the snapshots; for the days before the first one, the
17curve is *estimated* from the dates of the files still in the library
18(when each was downloaded, else its modified time), so it never shows
19what was deleted since. Duplicates are files holding the same song:
20*exact* ones share a name and size, *likely* ones the same artist and
21title (see :func:`downtify.library_catalog.song_key`) and a length
22within :data:`DURATION_TOLERANCE` seconds.
23"""
25from __future__ import annotations
27import bisect
28import json
29import math
30from collections import defaultdict
31from collections.abc import Collection, Iterable
32from contextlib import AbstractContextManager
33from datetime import date, datetime, timedelta, timezone
34from pathlib import Path, PurePosixPath
35from typing import Any, Optional
37from .db.config import Dialect
38from .db.core import Connection, Database, as_database
39from .download_log import OUTCOME_COMPLETED, OUTCOMES
40from .file_providers import guess_provider
41from .library_catalog import entry_content_key, song_key
42from .library_paths import EXTERNAL_LIBRARY_PREFIX, SLSKD_LIBRARY_PREFIX
43from .watch_stats import (
44 DEFAULT_DAYS,
45 bucket_days_for,
46 clamp_days,
47 norm_path,
48 parse_time,
49)
51#: Where a library file lives.
52LOCATION_DOWNLOADS = 'downloads'
53LOCATION_SLSKD = 'slskd'
54LOCATION_FOLDERS = 'folders'
56#: Provider of a file nothing recorded and its path doesn't give away.
57UNKNOWN = 'unknown'
59#: Two files of one song may differ this much in length (seconds).
60DURATION_TOLERANCE = 3.0
61#: How many duplicate groups / working copies are listed by name.
62EXAMPLES = 50
63#: The growth curve has a point per day up to this many days, then one a
64#: week.
65DAILY_POINTS_UP_TO = 90
68def location_of(path: Any) -> str:
69 """``downloads``, ``slskd`` or ``folders`` (extra music folders)."""
71 name = norm_path(path)
72 if name.startswith(SLSKD_LIBRARY_PREFIX):
73 return LOCATION_SLSKD
74 if name.startswith(EXTERNAL_LIBRARY_PREFIX):
75 return LOCATION_FOLDERS
76 return LOCATION_DOWNLOADS
79def _size(entry: dict[str, Any]) -> int:
80 try:
81 return max(0, int(entry.get('size') or 0))
82 except (TypeError, ValueError):
83 return 0
86def _format(path: str) -> str:
87 return PurePosixPath(path).suffix.lstrip('.').lower() or UNKNOWN
90def provider_of(path: str, providers: dict[str, str]) -> str:
91 return providers.get(path) or guess_provider(path) or UNKNOWN
94def split_working_copies(
95 entries: Iterable[dict[str, Any]],
96 *,
97 sources: dict[str, int],
98 slskd_in_place: bool,
99 referenced: Collection[str],
100) -> tuple[list[dict[str, Any]], dict[str, int]]:
101 """``(library rows, {working copy: size})``.
103 *sources* are the recorded transfers still on disk, ``{library path:
104 size}`` (a path outside the library folders stays as given). While
105 slskd files aren't left in place (*slskd_in_place* false), an
106 ``slskd/`` file is a working copy too unless *referenced* - the
107 track index or the provider records name it as a library song.
108 """
110 library: list[dict[str, Any]] = []
111 working: dict[str, int] = {}
112 for entry in entries:
113 name = norm_path(entry.get('file'))
114 if not name:
115 continue
116 if name in sources or (
117 not slskd_in_place
118 and name.startswith(SLSKD_LIBRARY_PREFIX)
119 and name not in referenced
120 ):
121 working[name] = _size(entry)
122 else:
123 library.append(entry)
124 for name, size in sources.items():
125 working.setdefault(norm_path(name), max(0, int(size)))
126 return library, working
129def _breakdown(
130 rows: Iterable[tuple[str, int]],
131) -> list[dict[str, Any]]:
132 """``[{id, files, bytes}]``, biggest first, from ``(id, size)``."""
134 files: dict[str, int] = defaultdict(int)
135 sizes: dict[str, int] = defaultdict(int)
136 for key, size in rows:
137 files[key] += 1
138 sizes[key] += size
139 return sorted(
140 (
141 {'id': key, 'files': files[key], 'bytes': sizes[key]}
142 for key in files
143 ),
144 key=lambda row: (-row['bytes'], -row['files'], row['id']),
145 )
148def library_totals(
149 entries: list[dict[str, Any]], providers: dict[str, str]
150) -> dict[str, Any]:
151 """Size of the library, by provider, location and format."""
153 sized = [(norm_path(e.get('file')), _size(e)) for e in entries]
154 return {
155 'files': len(sized),
156 'bytes': sum(size for _name, size in sized),
157 'by_provider': _breakdown(
158 (provider_of(name, providers), size) for name, size in sized
159 ),
160 'by_location': _breakdown(
161 (location_of(name), size) for name, size in sized
162 ),
163 'by_format': _breakdown((_format(name), size) for name, size in sized),
164 }
167def added_times(
168 entries: Iterable[dict[str, Any]], recorded: dict[str, str]
169) -> dict[str, datetime]:
170 """When each file came into the library: the download time the
171 provider records keep, else the file's modified time."""
173 times: dict[str, datetime] = {}
174 for entry in entries:
175 name = norm_path(entry.get('file'))
176 moment = parse_time(recorded.get(name)) if name in recorded else None
177 if moment is None:
178 try:
179 stamp = float(entry.get('added') or 0)
180 except (TypeError, ValueError):
181 stamp = 0.0
182 if stamp > 0:
183 moment = datetime.fromtimestamp(stamp, timezone.utc)
184 if moment is not None:
185 times[name] = moment
186 return times
189def _bucket(
190 moment: datetime, now: datetime, span: timedelta, count: int
191) -> Optional[int]:
192 """Which of *count* buckets of *span* ending *now* holds *moment*
193 (oldest first), or ``None``."""
195 if moment > now:
196 return None
197 index = count - 1 - math.floor((now - moment) / span)
198 return index if 0 <= index < count else None
201def period_added(
202 entries: list[dict[str, Any]],
203 times: dict[str, datetime],
204 providers: dict[str, str],
205 *,
206 now: datetime,
207 days: int,
208) -> dict[str, Any]:
209 """Songs still in the library that came in during the last *days*:
210 how many, their size, by provider and per day or week."""
212 bucket_days = bucket_days_for(days)
213 span = timedelta(days=bucket_days)
214 count = math.ceil(days / bucket_days)
215 series = [{'files': 0, 'bytes': 0} for _ in range(count)]
216 start = now - timedelta(days=days)
217 added: list[tuple[str, int]] = []
218 for entry in entries:
219 name = norm_path(entry.get('file'))
220 moment = times.get(name)
221 if moment is None or moment < start or moment > now:
222 continue
223 size = _size(entry)
224 added.append((name, size))
225 index = _bucket(moment, now, span, count)
226 if index is not None:
227 series[index]['files'] += 1
228 series[index]['bytes'] += size
229 return {
230 'files': len(added),
231 'bytes': sum(size for _name, size in added),
232 'by_provider': _breakdown(
233 (provider_of(name, providers), size) for name, size in added
234 ),
235 'bucket_days': bucket_days,
236 'series': series,
237 }
240class _Estimate:
241 """How big today's files added up to at a given moment."""
243 def __init__(
244 self, entries: list[dict[str, Any]], times: dict[str, datetime]
245 ) -> None:
246 dated = sorted(
247 (moment, _size(entry))
248 for entry in entries
249 if (moment := times.get(norm_path(entry.get('file')))) is not None
250 )
251 self._moments = [moment for moment, _size_ in dated]
252 self._running = [0]
253 for _moment, size in dated:
254 self._running.append(self._running[-1] + size)
256 def before(self, end: datetime) -> tuple[int, int]:
257 """``(files, bytes)`` of the files that came in before *end*."""
259 count = bisect.bisect_left(self._moments, end)
260 return count, self._running[count]
263def _point(
264 day: date, files: int, size: int, estimated: bool
265) -> dict[str, Any]:
266 return {
267 'day': day.isoformat(),
268 'files': int(files),
269 'bytes': int(size),
270 'estimated': estimated,
271 }
274def growth(
275 snapshots: list[dict[str, Any]],
276 entries: list[dict[str, Any]],
277 times: dict[str, datetime],
278 *,
279 now: datetime,
280 days: int,
281 current: dict[str, int],
282) -> dict[str, Any]:
283 """The library's size at the end of each day (or week) of the last
284 *days*, oldest first, ending with *current* (``{files, bytes}``).
286 A day takes the latest snapshot up to it; a day before the first
287 snapshot is ``estimated`` from the dates of today's files.
288 """
290 step = 1 if days <= DAILY_POINTS_UP_TO else 7
291 count = days // step + 1
292 today = now.date()
293 by_day = sorted(
294 (str(row['day']), row) for row in snapshots if row.get('day')
295 )
296 estimate = _Estimate(entries, times)
297 points: list[dict[str, Any]] = []
298 for index in range(count):
299 day = today - timedelta(days=(count - 1 - index) * step)
300 snapshot = _latest_on(by_day, day.isoformat())
301 if day == today:
302 points.append(
303 _point(day, current['files'], current['bytes'], False)
304 )
305 elif snapshot is not None:
306 points.append(
307 _point(day, snapshot['files'], snapshot['bytes'], False)
308 )
309 else:
310 end = datetime.combine(
311 day + timedelta(days=1), datetime.min.time(), timezone.utc
312 )
313 points.append(_point(day, *estimate.before(end), True))
314 first, last = points[0], points[-1]
315 change = last['bytes'] - first['bytes']
316 return {
317 'step_days': step,
318 'points': points,
319 'change': {
320 'files': last['files'] - first['files'],
321 'bytes': change,
322 },
323 'per_day_bytes': round(change / max(1, (count - 1) * step)),
324 'since': by_day[0][0] if by_day else None,
325 }
328def _latest_on(
329 by_day: list[tuple[str, dict[str, Any]]], day: str
330) -> Optional[dict[str, Any]]:
331 found = None
332 for key, row in by_day:
333 if key > day:
334 break
335 found = row
336 return found
339def forecast_days(
340 disk: Optional[dict[str, int]], per_day_bytes: int
341) -> Optional[int]:
342 """Days until the disk fills at *per_day_bytes*, if it grows."""
344 if not disk or per_day_bytes <= 0:
345 return None
346 return int(max(0, int(disk.get('free') or 0)) // per_day_bytes)
349class _Groups:
350 """Union-find over file names."""
352 def __init__(self) -> None:
353 self._parent: dict[str, str] = {}
355 def find(self, name: str) -> str:
356 parent = self._parent.setdefault(name, name)
357 while parent != self._parent[parent]:
358 self._parent[parent] = self._parent[self._parent[parent]]
359 parent = self._parent[parent]
360 self._parent[name] = parent
361 return parent
363 def join(self, names: list[str]) -> None:
364 roots = [self.find(name) for name in names]
365 for root in roots[1:]:
366 self._parent[root] = roots[0]
369def _duration(entry: dict[str, Any]) -> float:
370 try:
371 return float(entry.get('duration') or 0)
372 except (TypeError, ValueError):
373 return 0.0
376def _same_length_runs(rows: list[dict[str, Any]]) -> list[list[str]]:
377 """Files of one artist/title whose lengths chain within the
378 tolerance; files of unknown length never match on it."""
380 timed = sorted(
381 ((d, norm_path(r.get('file'))) for r in rows if (d := _duration(r))),
382 )
383 runs: list[list[str]] = []
384 run: list[str] = []
385 previous = 0.0
386 for length, name in timed:
387 if run and length - previous > DURATION_TOLERANCE:
388 runs.append(run)
389 run = []
390 run.append(name)
391 previous = length
392 if run:
393 runs.append(run)
394 return [names for names in runs if len(names) > 1]
397def duplicates(
398 entries: list[dict[str, Any]],
399 providers: dict[str, str],
400 *,
401 limit: Optional[int] = EXAMPLES,
402) -> dict[str, Any]:
403 """Library files holding the same song more than once.
405 ``bytes`` is what keeping only the biggest file of each group would
406 free. ``examples`` lists the *limit* groups that would free most
407 (every group with ``None``).
408 """
410 by_file = {norm_path(e.get('file')): e for e in entries}
411 by_file.pop('', None)
412 groups = _Groups()
413 by_key: dict[str, list[str]] = defaultdict(list)
414 by_song: dict[str, list[dict[str, Any]]] = defaultdict(list)
415 for name, entry in by_file.items():
416 key = entry_content_key(entry)
417 if key:
418 by_key[key].append(name)
419 song = song_key(
420 str(entry.get('artist') or ''), str(entry.get('title') or '')
421 )
422 if song != '|':
423 by_song[song].append(entry)
424 for names in by_key.values():
425 if len(names) > 1:
426 groups.join(names)
427 for rows in by_song.values():
428 if len(rows) > 1:
429 for run in _same_length_runs(rows):
430 groups.join(run)
431 members: dict[str, list[str]] = defaultdict(list)
432 for name in by_file:
433 members[groups.find(name)].append(name)
434 found = [
435 _duplicate_group(sorted(names), by_file, providers)
436 for names in members.values()
437 if len(names) > 1
438 ]
439 found.sort(key=lambda g: (-g['bytes'], g['files'][0]['file']))
440 totals = {
441 kind: {
442 'groups': sum(1 for g in found if g['kind'] == kind),
443 'files': sum(
444 len(g['files']) - 1 for g in found if g['kind'] == kind
445 ),
446 'bytes': sum(g['bytes'] for g in found if g['kind'] == kind),
447 }
448 for kind in ('exact', 'likely')
449 }
450 return {
451 'groups': len(found),
452 'files': sum(len(g['files']) - 1 for g in found),
453 'bytes': sum(g['bytes'] for g in found),
454 'exact': totals['exact'],
455 'likely': totals['likely'],
456 'examples': found if limit is None else found[:limit],
457 }
460def _duplicate_group(
461 names: list[str],
462 by_file: dict[str, dict[str, Any]],
463 providers: dict[str, str],
464) -> dict[str, Any]:
465 rows = [by_file[name] for name in names]
466 sizes = [_size(row) for row in rows]
467 keys = {entry_content_key(row) for row in rows}
468 first = rows[0]
469 return {
470 'kind': 'exact' if len(keys) == 1 and '' not in keys else 'likely',
471 'title': str(first.get('title') or ''),
472 'artist': str(first.get('artist') or ''),
473 'bytes': sum(sizes) - max(sizes),
474 'files': sorted(
475 (
476 {
477 'file': name,
478 'bytes': size,
479 'provider': provider_of(name, providers),
480 'codec': str(row.get('codec') or ''),
481 'bitrate': int(row.get('bitrate') or 0),
482 # Each copy's own tags: look-alikes can differ there
483 # (a parody uploaded under the original's name).
484 'title': str(row.get('title') or ''),
485 'artist': str(row.get('artist') or ''),
486 'album': str(row.get('album') or ''),
487 'duration': _duration(row),
488 }
489 for name, row, size in zip(names, rows, sizes)
490 ),
491 key=lambda item: (-item['bytes'], item['file']),
492 ),
493 }
496def download_totals(
497 rows: list[dict[str, Any]],
498 *,
499 now: datetime,
500 days: int,
501 first_at: Optional[str] = None,
502) -> dict[str, Any]:
503 """How the downloads of the last *days* ended, per day or week."""
505 bucket_days = bucket_days_for(days)
506 span = timedelta(days=bucket_days)
507 count = math.ceil(days / bucket_days)
508 series = [dict.fromkeys(OUTCOMES, 0) for _ in range(count)]
509 totals = dict.fromkeys(OUTCOMES, 0)
510 completed: list[tuple[str, int]] = []
511 start = now - timedelta(days=days)
512 for row in rows:
513 moment = parse_time(row.get('at'))
514 outcome = row.get('outcome')
515 if moment is None or outcome not in totals or moment < start:
516 continue
517 totals[outcome] += 1
518 index = _bucket(moment, now, span, count)
519 if index is not None:
520 series[index][outcome] += 1
521 if outcome == OUTCOME_COMPLETED:
522 completed.append((
523 str(row.get('provider') or '') or UNKNOWN,
524 max(0, int(row.get('bytes') or 0)),
525 ))
526 return {
527 **totals,
528 'bytes': sum(size for _provider, size in completed),
529 'by_provider': _breakdown(completed),
530 'bucket_days': bucket_days,
531 'series': series,
532 'since': first_at,
533 }
536def usage_report(
537 entries: list[dict[str, Any]],
538 *,
539 providers: dict[str, str],
540 recorded: dict[str, str],
541 sources: dict[str, int],
542 slskd_in_place: bool,
543 referenced: Collection[str],
544 snapshots: list[dict[str, Any]],
545 downloads: list[dict[str, Any]],
546 downloads_since: Optional[str],
547 disk: Optional[dict[str, int]],
548 now: datetime,
549 days: int = DEFAULT_DAYS,
550) -> dict[str, Any]:
551 """Everything the library usage page shows but the folder scan.
553 *entries* are the listing rows, *providers* ``{path: provider}``,
554 *recorded* ``{path: download time}``, *sources* the recorded slskd
555 transfers still on disk (see :func:`split_working_copies`),
556 *snapshots* the daily sizes kept (:class:`UsageSnapshots`),
557 *downloads* the download log rows of the period and *disk* the
558 downloads folder's volume (``{total, used, free}``).
559 """
561 days = clamp_days(days)
562 library, working = split_working_copies(
563 entries,
564 sources=sources,
565 slskd_in_place=slskd_in_place,
566 referenced=referenced,
567 )
568 totals = library_totals(library, providers)
569 times = added_times(library, recorded)
570 grown = growth(
571 snapshots,
572 library,
573 times,
574 now=now,
575 days=days,
576 current={'files': totals['files'], 'bytes': totals['bytes']},
577 )
578 largest = sorted(working.items(), key=lambda item: (-item[1], item[0]))
579 return {
580 'days': days,
581 'generated_at': now.isoformat(),
582 'library': totals,
583 'period': period_added(library, times, providers, now=now, days=days),
584 'growth': grown,
585 'downloads': download_totals(
586 downloads, now=now, days=days, first_at=downloads_since
587 ),
588 'duplicates': duplicates(library, providers),
589 'working': {
590 'files': len(working),
591 'bytes': sum(working.values()),
592 'slskd_in_place': slskd_in_place,
593 'examples': [
594 {'file': name, 'bytes': size}
595 for name, size in largest[:EXAMPLES]
596 ],
597 },
598 'disk': (
599 {**disk, 'days_left': forecast_days(disk, grown['per_day_bytes'])}
600 if disk
601 else None
602 ),
603 }
606def snapshot_of(report: dict[str, Any]) -> dict[str, Any]:
607 """What :class:`UsageSnapshots` keeps of a :func:`usage_report`."""
609 library = report['library']
610 return {
611 'files': library['files'],
612 'bytes': library['bytes'],
613 'working_files': report['working']['files'],
614 'working_bytes': report['working']['bytes'],
615 'by_provider': library['by_provider'],
616 'by_location': library['by_location'],
617 }
620def library_snapshot(
621 entries: list[dict[str, Any]],
622 *,
623 providers: dict[str, str],
624 sources: dict[str, int],
625 slskd_in_place: bool,
626 referenced: Collection[str],
627) -> dict[str, Any]:
628 """The same as :func:`snapshot_of`, without the rest of the report
629 (what the hourly reading needs)."""
631 library, working = split_working_copies(
632 entries,
633 sources=sources,
634 slskd_in_place=slskd_in_place,
635 referenced=referenced,
636 )
637 totals = library_totals(library, providers)
638 return {
639 'files': totals['files'],
640 'bytes': totals['bytes'],
641 'working_files': len(working),
642 'working_bytes': sum(working.values()),
643 'by_provider': totals['by_provider'],
644 'by_location': totals['by_location'],
645 }
648class UsageSnapshots:
649 """The library's size once a day (the last reading of the day wins)."""
651 def __init__(self, db_path: str | Path | Database) -> None:
652 self.db = as_database(db_path, 'library')
653 self._path = None if self.db.path is None else str(self.db.path)
654 if self.db.dialect is Dialect.SQLITE:
655 self._init_db()
657 def _init_db(self) -> None:
658 # SQLite only: on PostgreSQL the migration made this schema.
659 with self._connect() as conn:
660 conn.execute("""
661 CREATE TABLE IF NOT EXISTS library_usage_snapshots (
662 day TEXT PRIMARY KEY,
663 taken_at TEXT NOT NULL,
664 files INTEGER NOT NULL DEFAULT 0,
665 bytes INTEGER NOT NULL DEFAULT 0,
666 working_files INTEGER NOT NULL DEFAULT 0,
667 working_bytes INTEGER NOT NULL DEFAULT 0,
668 detail_json TEXT NOT NULL DEFAULT '{}'
669 )
670 """)
672 def _connect(self) -> AbstractContextManager[Connection]:
673 return self.db.connect()
675 def record(self, snapshot: dict[str, Any], now: datetime) -> None:
676 detail = {
677 key: snapshot.get(key) or []
678 for key in ('by_provider', 'by_location')
679 }
680 with self._connect() as conn:
681 conn.execute(
682 """INSERT INTO library_usage_snapshots
683 (day, taken_at, files, bytes, working_files,
684 working_bytes, detail_json)
685 VALUES (?, ?, ?, ?, ?, ?, ?)
686 ON CONFLICT(day) DO UPDATE SET
687 taken_at = excluded.taken_at,
688 files = excluded.files,
689 bytes = excluded.bytes,
690 working_files = excluded.working_files,
691 working_bytes = excluded.working_bytes,
692 detail_json = excluded.detail_json""",
693 (
694 now.date().isoformat(),
695 now.isoformat(),
696 int(snapshot.get('files') or 0),
697 int(snapshot.get('bytes') or 0),
698 int(snapshot.get('working_files') or 0),
699 int(snapshot.get('working_bytes') or 0),
700 json.dumps(detail),
701 ),
702 )
704 def since(self, day: str) -> list[dict[str, Any]]:
705 """Snapshots from *day* (``YYYY-MM-DD``) on, oldest first - plus
706 the last one before it, which still stands for that day."""
708 with self._connect() as conn:
709 before = conn.execute(
710 'SELECT day, files, bytes, working_files, working_bytes '
711 'FROM library_usage_snapshots WHERE day < ? '
712 'ORDER BY day DESC LIMIT 1',
713 (day,),
714 ).fetchall()
715 rows = conn.execute(
716 'SELECT day, files, bytes, working_files, working_bytes '
717 'FROM library_usage_snapshots WHERE day >= ? ORDER BY day',
718 (day,),
719 ).fetchall()
720 return [dict(row) for row in (*before, *rows)]