Coverage for downtify/watch_stats.py: 97%
187 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Playlist Monitor provenance and per-watch statistics.
3Pure functions over what the monitor database records - the
4``downloaded_tracks`` each watch holds and its ``watch_events`` (see
5:mod:`downtify.monitor`) - plus ``{library path: size}`` for the files
6still on disk. ``GET /api/monitor/stats`` and
7``GET /api/monitor/provenance`` (``downtify/api.py``) do the I/O.
9A file several watches hold is still one file: each of them lists it,
10but the totals count it once, and a watch's *unique* storage is only
11what no other watch also holds (``share`` splits a shared file evenly).
13Activity (rates and the trend) counts the tracks a watch actually
14added: the ones it downloaded, not those it found already in the
15library (``linked``), and never its first check's backfill
16(``initial``). Tracks recorded before outcomes were kept are
17``unknown`` and counted as added, flagged by ``period.unknown``.
18"""
20from __future__ import annotations
22import math
23from collections import Counter
24from dataclasses import dataclass, field
25from datetime import datetime, timedelta, timezone
26from typing import Any, Collection, Iterable, Optional
28from .monitor import MonitoredPlaylist
29from .watch_history import (
30 OUTCOME_DOWNLOADED,
31 OUTCOME_LINKED,
32 OUTCOME_UNKNOWN,
33)
35DEFAULT_DAYS = 30
36MAX_DAYS = 365
37#: Periods up to this many days are bucketed per day, longer per week.
38DAILY_BUCKETS_UP_TO = 14
39#: A trend moves when the fitted line changes by more than this share of
40#: the average bucket (and by at least one song) over the period.
41TREND_THRESHOLD = 0.25
42MIN_TREND_BUCKETS = 3
44#: Outcomes that grew the library.
45ADDED = frozenset({OUTCOME_DOWNLOADED, OUTCOME_UNKNOWN})
48def parse_time(value: Any) -> Optional[datetime]:
49 """An ISO timestamp as an aware datetime (UTC when it has no zone)."""
51 try:
52 moment = datetime.fromisoformat(str(value or ''))
53 except ValueError:
54 return None
55 if moment.tzinfo is None:
56 moment = moment.replace(tzinfo=timezone.utc)
57 return moment
60def clamp_days(value: Any) -> int:
61 """A requested period in days, kept between 1 and :data:`MAX_DAYS`."""
63 try:
64 days = int(value)
65 except (TypeError, ValueError):
66 return DEFAULT_DAYS
67 return max(1, min(MAX_DAYS, days))
70def bucket_days_for(days: int) -> int:
71 return 1 if days <= DAILY_BUCKETS_UP_TO else 7
74def norm_path(path: Any) -> str:
75 return str(path or '').strip().replace('\\', '/')
78def trend(series: list[int]) -> dict[str, Any]:
79 """Which way *series* (added songs per bucket, oldest first) goes.
81 ``direction`` is ``new`` (fewer than three buckets to tell from: a
82 line through two points says little), ``idle`` (nothing added),
83 ``up``, ``down`` or ``steady``; ``slope`` is the least-squares change
84 in songs per bucket, per bucket.
85 """
87 count = len(series)
88 if count < MIN_TREND_BUCKETS:
89 return {'direction': 'new', 'slope': 0.0}
90 total = sum(series)
91 if not total:
92 return {'direction': 'idle', 'slope': 0.0}
93 mean_x = (count - 1) / 2
94 mean_y = total / count
95 spread = sum((x - mean_x) ** 2 for x in range(count))
96 slope = (
97 sum((x - mean_x) * (y - mean_y) for x, y in enumerate(series)) / spread
98 )
99 change = slope * (count - 1)
100 threshold = max(1.0, TREND_THRESHOLD * mean_y)
101 if change > threshold:
102 direction = 'up'
103 elif change < -threshold:
104 direction = 'down'
105 else:
106 direction = 'steady'
107 return {'direction': direction, 'slope': round(slope, 3)}
110def activity_series(
111 times: Iterable[datetime], now: datetime, start: datetime, days: int
112) -> list[int]:
113 """Songs per bucket over the whole buckets between *start* (the
114 later of the period's start and when the watch began) and *now*,
115 oldest first."""
117 span = timedelta(days=bucket_days_for(days))
118 first = max(start, now - timedelta(days=days))
119 count = max(0, math.floor((now - first) / span))
120 series = [0] * count
121 for moment in times:
122 if moment > now:
123 continue
124 index = count - 1 - math.floor((now - moment) / span)
125 if 0 <= index < count:
126 series[index] += 1
127 return series
130def holders_by_file(
131 associations: Iterable[dict[str, Any]],
132 on_disk: Optional[Collection[str]] = None,
133) -> dict[str, set[int]]:
134 """``{file: ids of the watches holding it}`` - only files *on_disk*
135 when given."""
137 holders: dict[str, set[int]] = {}
138 for row in associations:
139 name = norm_path(row.get('filename'))
140 if name and (on_disk is None or name in on_disk):
141 holders.setdefault(name, set()).add(int(row['playlist_id']))
142 return holders
145def providers_from_events(
146 events: Iterable[dict[str, Any]],
147) -> dict[str, str]:
148 """``{file: provider}`` as the watches' downloads recorded it (the
149 latest wins) - what fetched a file another watch only linked."""
151 found: dict[str, str] = {}
152 for event in events:
153 name = norm_path(event.get('filename'))
154 if (
155 name
156 and event['outcome'] == OUTCOME_DOWNLOADED
157 and event['provider']
158 ):
159 found[name] = str(event['provider'])
160 return found
163def _event_key(event: dict[str, Any]) -> tuple[int, str, str]:
164 return (
165 int(event['playlist_id']),
166 str(event['watch_key']),
167 str(event['track_id']),
168 )
171def first_events(
172 events: Iterable[dict[str, Any]],
173) -> dict[tuple[int, str, str], dict[str, Any]]:
174 """The earliest event per ``(watch id, watch key, track id)``: how
175 the watch got each track. *events* come oldest first."""
177 firsts: dict[tuple[int, str, str], dict[str, Any]] = {}
178 for event in events:
179 firsts.setdefault(_event_key(event), event)
180 return firsts
183@dataclass
184class _WatchFacts:
185 """What one watch holds and did, gathered in one pass."""
187 watch: MonitoredPlaylist
188 rows: list[dict[str, Any]] = field(default_factory=list)
189 events: list[dict[str, Any]] = field(default_factory=list)
192def _gather(
193 watches: list[MonitoredPlaylist],
194 associations: Iterable[dict[str, Any]],
195 events: Iterable[dict[str, Any]],
196) -> dict[int, _WatchFacts]:
197 facts = {watch.id: _WatchFacts(watch) for watch in watches}
198 for row in associations:
199 fact = facts.get(int(row['playlist_id']))
200 if fact is not None and norm_path(row.get('filename')):
201 fact.rows.append(row)
202 for event in events:
203 fact = facts.get(int(event['playlist_id']))
204 # A retargeted watch's events of its old target are history.
205 if fact is not None and event['watch_key'] == fact.watch.spotify_id:
206 fact.events.append(event)
207 return facts
210def _storage(
211 fact: _WatchFacts,
212 sizes: dict[str, int],
213 holders: dict[str, set[int]],
214) -> dict[str, Any]:
215 files = {norm_path(row['filename']) for row in fact.rows}
216 present = sorted(name for name in files if name in sizes)
217 unique = [name for name in present if len(holders[name]) == 1]
218 shared = [name for name in present if len(holders[name]) > 1]
219 return {
220 'files': len(present),
221 'missing': len(files) - len(present),
222 'unique_files': len(unique),
223 'shared_files': len(shared),
224 'bytes': sum(sizes[name] for name in present),
225 'unique_bytes': sum(sizes[name] for name in unique),
226 'shared_bytes': sum(sizes[name] for name in shared),
227 'share_bytes': round(
228 sum(sizes[name] / len(holders[name]) for name in present)
229 ),
230 }
233def _status(
234 fact: _WatchFacts,
235 firsts: dict[tuple[int, str, str], dict[str, Any]],
236 sizes: dict[str, int],
237 providers: dict[str, str],
238) -> tuple[dict[str, int], dict[str, int]]:
239 """How the tracks on disk got into the watch, and which provider
240 fetched their files."""
242 outcomes: Counter[str] = Counter()
243 by_provider: Counter[str] = Counter()
244 seen: set[str] = set()
245 for row in fact.rows:
246 name = norm_path(row['filename'])
247 if name not in sizes or name in seen:
248 continue
249 seen.add(name)
250 first = firsts.get((
251 fact.watch.id,
252 fact.watch.spotify_id,
253 str(row['track_spotify_id']),
254 ))
255 outcomes[first['outcome'] if first else OUTCOME_UNKNOWN] += 1
256 provider = providers.get(name) or (first or {}).get('provider')
257 by_provider[provider or 'unknown'] += 1
258 status = {
259 outcome: outcomes.get(outcome, 0)
260 for outcome in (OUTCOME_DOWNLOADED, OUTCOME_LINKED, OUTCOME_UNKNOWN)
261 }
262 return status, dict(by_provider.most_common())
265def _activity(
266 fact: _WatchFacts,
267 sizes: dict[str, int],
268 now: datetime,
269 days: int,
270) -> dict[str, Any]:
271 period_start = now - timedelta(days=days)
272 counts: Counter[str] = Counter()
273 added_times: list[datetime] = []
274 added_files: set[str] = set()
275 last_added: Optional[datetime] = None
276 for event in fact.events:
277 moment = parse_time(event['happened_at'])
278 if moment is None:
279 continue
280 added = event['outcome'] in ADDED and not event['initial']
281 if added:
282 added_times.append(moment)
283 last_added = (
284 moment if last_added is None else max(last_added, moment)
285 )
286 if moment < period_start:
287 continue
288 if event['initial']:
289 counts['initial'] += 1
290 continue
291 counts[event['outcome']] += 1
292 if added:
293 added_files.add(norm_path(event['filename']))
295 # The watch began when its target was first checked (a retargeted
296 # watch, at its new target), else when it was added.
297 began = parse_time(fact.events[0]['happened_at']) if fact.events else None
298 began = began or parse_time(fact.watch.created_at) or now
299 series = activity_series(added_times, now, began, days)
300 added_count = counts[OUTCOME_DOWNLOADED] + counts[OUTCOME_UNKNOWN]
301 span_days = max(
302 1.0, (now - max(began, period_start)).total_seconds() / 86400
303 )
304 per_day = added_count / span_days
305 return {
306 'period': {
307 'added': added_count,
308 'downloaded': counts[OUTCOME_DOWNLOADED],
309 'linked': counts[OUTCOME_LINKED],
310 'unknown': counts[OUTCOME_UNKNOWN],
311 'initial': counts['initial'],
312 'bytes': sum(sizes.get(name, 0) for name in added_files),
313 },
314 'rate': {
315 'per_day': round(per_day, 3),
316 'per_week': round(per_day * 7, 2),
317 'per_month': round(per_day * 30, 2),
318 },
319 'series': series,
320 'trend': trend(series),
321 'last_added_at': last_added.isoformat() if last_added else None,
322 }
325def watch_stats(
326 watches: list[MonitoredPlaylist],
327 associations: list[dict[str, Any]],
328 events: list[dict[str, Any]],
329 sizes: dict[str, int],
330 *,
331 now: datetime,
332 days: int = DEFAULT_DAYS,
333 providers: Optional[dict[str, str]] = None,
334 failures: Optional[dict[int, int]] = None,
335) -> dict[str, Any]:
336 """Storage, status and activity of each of *watches*.
338 *associations* and *events* are every watch's rows (other watches'
339 tell which files are shared); *sizes* has the library files on disk.
340 """
342 days = clamp_days(days)
343 # What the library recorded for the file wins over the events.
344 providers = {**providers_from_events(events), **(providers or {})}
345 failures = failures or {}
346 holders = holders_by_file(associations, sizes)
347 firsts = first_events(events)
348 facts = _gather(watches, associations, events)
349 rows = []
350 for watch in watches:
351 fact = facts[watch.id]
352 status, by_provider = _status(fact, firsts, sizes, providers)
353 rows.append({
354 'id': watch.id,
355 'storage': _storage(fact, sizes, holders),
356 'status': {**status, 'failing': int(failures.get(watch.id, 0))},
357 'providers': by_provider,
358 **_activity(fact, sizes, now, days),
359 })
360 files = {
361 norm_path(row['filename'])
362 for fact in facts.values()
363 for row in fact.rows
364 if norm_path(row['filename']) in sizes
365 }
366 return {
367 'days': days,
368 'bucket_days': bucket_days_for(days),
369 'generated_at': now.isoformat(),
370 'totals': {
371 'watches': len(watches),
372 'files': len(files),
373 'bytes': sum(sizes[name] for name in files),
374 'shared_files': sum(1 for name in files if len(holders[name]) > 1),
375 },
376 'watches': rows,
377 }
380def provenance(
381 associations: list[dict[str, Any]],
382 events: list[dict[str, Any]],
383 watches_by_id: dict[int, MonitoredPlaylist],
384) -> dict[str, Any]:
385 """Where one library file came from.
387 *associations* are the ``downloaded_tracks`` rows holding it and
388 *events* every event about it, oldest first. ``origin`` is the first
389 watch that downloaded it (``estimated`` when only an event recorded
390 before outcomes were kept says so); ``holders`` every watch that
391 holds it now, with how and when it got there.
392 """
394 firsts = first_events(events)
395 holders: dict[int, dict[str, Any]] = {}
396 for row in associations:
397 watch = watches_by_id.get(int(row['playlist_id']))
398 if watch is None:
399 continue
400 first = firsts.get((
401 watch.id,
402 watch.spotify_id,
403 str(row['track_spotify_id']),
404 ))
405 entry = {
406 'watch': watch,
407 'added_at': first['happened_at']
408 if first
409 else row['downloaded_at'],
410 'outcome': first['outcome'] if first else OUTCOME_UNKNOWN,
411 'initial': bool(first and first['initial']),
412 }
413 known = holders.get(watch.id)
414 if known is None or str(entry['added_at']) < str(known['added_at']):
415 holders[watch.id] = entry
416 return {
417 'origin': _origin(events, watches_by_id),
418 'holders': sorted(holders.values(), key=lambda h: str(h['added_at'])),
419 }
422def _origin(
423 events: list[dict[str, Any]],
424 watches_by_id: dict[int, MonitoredPlaylist],
425) -> Optional[dict[str, Any]]:
426 event = next(
427 (e for e in events if e['outcome'] == OUTCOME_DOWNLOADED), None
428 )
429 estimated = event is None
430 if event is None:
431 event = next(
432 (e for e in events if e['outcome'] == OUTCOME_UNKNOWN), None
433 )
434 if event is None:
435 return None
436 watch = watches_by_id.get(int(event['playlist_id']))
437 # Gone, or since pointed at another playlist: only the name is left.
438 current = watch is not None and watch.spotify_id == event['watch_key']
439 return {
440 'watch': watch if current else None,
441 'watch_id': int(event['playlist_id']),
442 'user_id': int(event.get('user_id') or 0),
443 'name': str(event['watch_name']),
444 'at': event['happened_at'],
445 'provider': str(event['provider'] or ''),
446 'initial': bool(event['initial']),
447 'estimated': estimated,
448 }