Coverage for downtify/watch_stats.py: 97%

187 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Playlist Monitor provenance and per-watch statistics. 

2 

3Pure functions over what the monitor database records - the 

4``downloaded_tracks`` each watch holds and its ``watch_events`` (see 

5:mod:`downtify.monitor`) - plus ``{library path: size}`` for the files 

6still on disk. ``GET /api/monitor/stats`` and 

7``GET /api/monitor/provenance`` (``downtify/api.py``) do the I/O. 

8 

9A file several watches hold is still one file: each of them lists it, 

10but the totals count it once, and a watch's *unique* storage is only 

11what no other watch also holds (``share`` splits a shared file evenly). 

12 

13Activity (rates and the trend) counts the tracks a watch actually 

14added: the ones it downloaded, not those it found already in the 

15library (``linked``), and never its first check's backfill 

16(``initial``). Tracks recorded before outcomes were kept are 

17``unknown`` and counted as added, flagged by ``period.unknown``. 

18""" 

19 

20from __future__ import annotations 

21 

22import math 

23from collections import Counter 

24from dataclasses import dataclass, field 

25from datetime import datetime, timedelta, timezone 

26from typing import Any, Collection, Iterable, Optional 

27 

28from .monitor import MonitoredPlaylist 

29from .watch_history import ( 

30 OUTCOME_DOWNLOADED, 

31 OUTCOME_LINKED, 

32 OUTCOME_UNKNOWN, 

33) 

34 

35DEFAULT_DAYS = 30 

36MAX_DAYS = 365 

37#: Periods up to this many days are bucketed per day, longer per week. 

38DAILY_BUCKETS_UP_TO = 14 

39#: A trend moves when the fitted line changes by more than this share of 

40#: the average bucket (and by at least one song) over the period. 

41TREND_THRESHOLD = 0.25 

42MIN_TREND_BUCKETS = 3 

43 

44#: Outcomes that grew the library. 

45ADDED = frozenset({OUTCOME_DOWNLOADED, OUTCOME_UNKNOWN}) 

46 

47 

48def parse_time(value: Any) -> Optional[datetime]: 

49 """An ISO timestamp as an aware datetime (UTC when it has no zone).""" 

50 

51 try: 

52 moment = datetime.fromisoformat(str(value or '')) 

53 except ValueError: 

54 return None 

55 if moment.tzinfo is None: 

56 moment = moment.replace(tzinfo=timezone.utc) 

57 return moment 

58 

59 

60def clamp_days(value: Any) -> int: 

61 """A requested period in days, kept between 1 and :data:`MAX_DAYS`.""" 

62 

63 try: 

64 days = int(value) 

65 except (TypeError, ValueError): 

66 return DEFAULT_DAYS 

67 return max(1, min(MAX_DAYS, days)) 

68 

69 

70def bucket_days_for(days: int) -> int: 

71 return 1 if days <= DAILY_BUCKETS_UP_TO else 7 

72 

73 

74def norm_path(path: Any) -> str: 

75 return str(path or '').strip().replace('\\', '/') 

76 

77 

78def trend(series: list[int]) -> dict[str, Any]: 

79 """Which way *series* (added songs per bucket, oldest first) goes. 

80 

81 ``direction`` is ``new`` (fewer than three buckets to tell from: a 

82 line through two points says little), ``idle`` (nothing added), 

83 ``up``, ``down`` or ``steady``; ``slope`` is the least-squares change 

84 in songs per bucket, per bucket. 

85 """ 

86 

87 count = len(series) 

88 if count < MIN_TREND_BUCKETS: 

89 return {'direction': 'new', 'slope': 0.0} 

90 total = sum(series) 

91 if not total: 

92 return {'direction': 'idle', 'slope': 0.0} 

93 mean_x = (count - 1) / 2 

94 mean_y = total / count 

95 spread = sum((x - mean_x) ** 2 for x in range(count)) 

96 slope = ( 

97 sum((x - mean_x) * (y - mean_y) for x, y in enumerate(series)) / spread 

98 ) 

99 change = slope * (count - 1) 

100 threshold = max(1.0, TREND_THRESHOLD * mean_y) 

101 if change > threshold: 

102 direction = 'up' 

103 elif change < -threshold: 

104 direction = 'down' 

105 else: 

106 direction = 'steady' 

107 return {'direction': direction, 'slope': round(slope, 3)} 

108 

109 

110def activity_series( 

111 times: Iterable[datetime], now: datetime, start: datetime, days: int 

112) -> list[int]: 

113 """Songs per bucket over the whole buckets between *start* (the 

114 later of the period's start and when the watch began) and *now*, 

115 oldest first.""" 

116 

117 span = timedelta(days=bucket_days_for(days)) 

118 first = max(start, now - timedelta(days=days)) 

119 count = max(0, math.floor((now - first) / span)) 

120 series = [0] * count 

121 for moment in times: 

122 if moment > now: 

123 continue 

124 index = count - 1 - math.floor((now - moment) / span) 

125 if 0 <= index < count: 

126 series[index] += 1 

127 return series 

128 

129 

130def holders_by_file( 

131 associations: Iterable[dict[str, Any]], 

132 on_disk: Optional[Collection[str]] = None, 

133) -> dict[str, set[int]]: 

134 """``{file: ids of the watches holding it}`` - only files *on_disk* 

135 when given.""" 

136 

137 holders: dict[str, set[int]] = {} 

138 for row in associations: 

139 name = norm_path(row.get('filename')) 

140 if name and (on_disk is None or name in on_disk): 

141 holders.setdefault(name, set()).add(int(row['playlist_id'])) 

142 return holders 

143 

144 

145def providers_from_events( 

146 events: Iterable[dict[str, Any]], 

147) -> dict[str, str]: 

148 """``{file: provider}`` as the watches' downloads recorded it (the 

149 latest wins) - what fetched a file another watch only linked.""" 

150 

151 found: dict[str, str] = {} 

152 for event in events: 

153 name = norm_path(event.get('filename')) 

154 if ( 

155 name 

156 and event['outcome'] == OUTCOME_DOWNLOADED 

157 and event['provider'] 

158 ): 

159 found[name] = str(event['provider']) 

160 return found 

161 

162 

163def _event_key(event: dict[str, Any]) -> tuple[int, str, str]: 

164 return ( 

165 int(event['playlist_id']), 

166 str(event['watch_key']), 

167 str(event['track_id']), 

168 ) 

169 

170 

171def first_events( 

172 events: Iterable[dict[str, Any]], 

173) -> dict[tuple[int, str, str], dict[str, Any]]: 

174 """The earliest event per ``(watch id, watch key, track id)``: how 

175 the watch got each track. *events* come oldest first.""" 

176 

177 firsts: dict[tuple[int, str, str], dict[str, Any]] = {} 

178 for event in events: 

179 firsts.setdefault(_event_key(event), event) 

180 return firsts 

181 

182 

183@dataclass 

184class _WatchFacts: 

185 """What one watch holds and did, gathered in one pass.""" 

186 

187 watch: MonitoredPlaylist 

188 rows: list[dict[str, Any]] = field(default_factory=list) 

189 events: list[dict[str, Any]] = field(default_factory=list) 

190 

191 

192def _gather( 

193 watches: list[MonitoredPlaylist], 

194 associations: Iterable[dict[str, Any]], 

195 events: Iterable[dict[str, Any]], 

196) -> dict[int, _WatchFacts]: 

197 facts = {watch.id: _WatchFacts(watch) for watch in watches} 

198 for row in associations: 

199 fact = facts.get(int(row['playlist_id'])) 

200 if fact is not None and norm_path(row.get('filename')): 

201 fact.rows.append(row) 

202 for event in events: 

203 fact = facts.get(int(event['playlist_id'])) 

204 # A retargeted watch's events of its old target are history. 

205 if fact is not None and event['watch_key'] == fact.watch.spotify_id: 

206 fact.events.append(event) 

207 return facts 

208 

209 

210def _storage( 

211 fact: _WatchFacts, 

212 sizes: dict[str, int], 

213 holders: dict[str, set[int]], 

214) -> dict[str, Any]: 

215 files = {norm_path(row['filename']) for row in fact.rows} 

216 present = sorted(name for name in files if name in sizes) 

217 unique = [name for name in present if len(holders[name]) == 1] 

218 shared = [name for name in present if len(holders[name]) > 1] 

219 return { 

220 'files': len(present), 

221 'missing': len(files) - len(present), 

222 'unique_files': len(unique), 

223 'shared_files': len(shared), 

224 'bytes': sum(sizes[name] for name in present), 

225 'unique_bytes': sum(sizes[name] for name in unique), 

226 'shared_bytes': sum(sizes[name] for name in shared), 

227 'share_bytes': round( 

228 sum(sizes[name] / len(holders[name]) for name in present) 

229 ), 

230 } 

231 

232 

233def _status( 

234 fact: _WatchFacts, 

235 firsts: dict[tuple[int, str, str], dict[str, Any]], 

236 sizes: dict[str, int], 

237 providers: dict[str, str], 

238) -> tuple[dict[str, int], dict[str, int]]: 

239 """How the tracks on disk got into the watch, and which provider 

240 fetched their files.""" 

241 

242 outcomes: Counter[str] = Counter() 

243 by_provider: Counter[str] = Counter() 

244 seen: set[str] = set() 

245 for row in fact.rows: 

246 name = norm_path(row['filename']) 

247 if name not in sizes or name in seen: 

248 continue 

249 seen.add(name) 

250 first = firsts.get(( 

251 fact.watch.id, 

252 fact.watch.spotify_id, 

253 str(row['track_spotify_id']), 

254 )) 

255 outcomes[first['outcome'] if first else OUTCOME_UNKNOWN] += 1 

256 provider = providers.get(name) or (first or {}).get('provider') 

257 by_provider[provider or 'unknown'] += 1 

258 status = { 

259 outcome: outcomes.get(outcome, 0) 

260 for outcome in (OUTCOME_DOWNLOADED, OUTCOME_LINKED, OUTCOME_UNKNOWN) 

261 } 

262 return status, dict(by_provider.most_common()) 

263 

264 

265def _activity( 

266 fact: _WatchFacts, 

267 sizes: dict[str, int], 

268 now: datetime, 

269 days: int, 

270) -> dict[str, Any]: 

271 period_start = now - timedelta(days=days) 

272 counts: Counter[str] = Counter() 

273 added_times: list[datetime] = [] 

274 added_files: set[str] = set() 

275 last_added: Optional[datetime] = None 

276 for event in fact.events: 

277 moment = parse_time(event['happened_at']) 

278 if moment is None: 

279 continue 

280 added = event['outcome'] in ADDED and not event['initial'] 

281 if added: 

282 added_times.append(moment) 

283 last_added = ( 

284 moment if last_added is None else max(last_added, moment) 

285 ) 

286 if moment < period_start: 

287 continue 

288 if event['initial']: 

289 counts['initial'] += 1 

290 continue 

291 counts[event['outcome']] += 1 

292 if added: 

293 added_files.add(norm_path(event['filename'])) 

294 

295 # The watch began when its target was first checked (a retargeted 

296 # watch, at its new target), else when it was added. 

297 began = parse_time(fact.events[0]['happened_at']) if fact.events else None 

298 began = began or parse_time(fact.watch.created_at) or now 

299 series = activity_series(added_times, now, began, days) 

300 added_count = counts[OUTCOME_DOWNLOADED] + counts[OUTCOME_UNKNOWN] 

301 span_days = max( 

302 1.0, (now - max(began, period_start)).total_seconds() / 86400 

303 ) 

304 per_day = added_count / span_days 

305 return { 

306 'period': { 

307 'added': added_count, 

308 'downloaded': counts[OUTCOME_DOWNLOADED], 

309 'linked': counts[OUTCOME_LINKED], 

310 'unknown': counts[OUTCOME_UNKNOWN], 

311 'initial': counts['initial'], 

312 'bytes': sum(sizes.get(name, 0) for name in added_files), 

313 }, 

314 'rate': { 

315 'per_day': round(per_day, 3), 

316 'per_week': round(per_day * 7, 2), 

317 'per_month': round(per_day * 30, 2), 

318 }, 

319 'series': series, 

320 'trend': trend(series), 

321 'last_added_at': last_added.isoformat() if last_added else None, 

322 } 

323 

324 

325def watch_stats( 

326 watches: list[MonitoredPlaylist], 

327 associations: list[dict[str, Any]], 

328 events: list[dict[str, Any]], 

329 sizes: dict[str, int], 

330 *, 

331 now: datetime, 

332 days: int = DEFAULT_DAYS, 

333 providers: Optional[dict[str, str]] = None, 

334 failures: Optional[dict[int, int]] = None, 

335) -> dict[str, Any]: 

336 """Storage, status and activity of each of *watches*. 

337 

338 *associations* and *events* are every watch's rows (other watches' 

339 tell which files are shared); *sizes* has the library files on disk. 

340 """ 

341 

342 days = clamp_days(days) 

343 # What the library recorded for the file wins over the events. 

344 providers = {**providers_from_events(events), **(providers or {})} 

345 failures = failures or {} 

346 holders = holders_by_file(associations, sizes) 

347 firsts = first_events(events) 

348 facts = _gather(watches, associations, events) 

349 rows = [] 

350 for watch in watches: 

351 fact = facts[watch.id] 

352 status, by_provider = _status(fact, firsts, sizes, providers) 

353 rows.append({ 

354 'id': watch.id, 

355 'storage': _storage(fact, sizes, holders), 

356 'status': {**status, 'failing': int(failures.get(watch.id, 0))}, 

357 'providers': by_provider, 

358 **_activity(fact, sizes, now, days), 

359 }) 

360 files = { 

361 norm_path(row['filename']) 

362 for fact in facts.values() 

363 for row in fact.rows 

364 if norm_path(row['filename']) in sizes 

365 } 

366 return { 

367 'days': days, 

368 'bucket_days': bucket_days_for(days), 

369 'generated_at': now.isoformat(), 

370 'totals': { 

371 'watches': len(watches), 

372 'files': len(files), 

373 'bytes': sum(sizes[name] for name in files), 

374 'shared_files': sum(1 for name in files if len(holders[name]) > 1), 

375 }, 

376 'watches': rows, 

377 } 

378 

379 

380def provenance( 

381 associations: list[dict[str, Any]], 

382 events: list[dict[str, Any]], 

383 watches_by_id: dict[int, MonitoredPlaylist], 

384) -> dict[str, Any]: 

385 """Where one library file came from. 

386 

387 *associations* are the ``downloaded_tracks`` rows holding it and 

388 *events* every event about it, oldest first. ``origin`` is the first 

389 watch that downloaded it (``estimated`` when only an event recorded 

390 before outcomes were kept says so); ``holders`` every watch that 

391 holds it now, with how and when it got there. 

392 """ 

393 

394 firsts = first_events(events) 

395 holders: dict[int, dict[str, Any]] = {} 

396 for row in associations: 

397 watch = watches_by_id.get(int(row['playlist_id'])) 

398 if watch is None: 

399 continue 

400 first = firsts.get(( 

401 watch.id, 

402 watch.spotify_id, 

403 str(row['track_spotify_id']), 

404 )) 

405 entry = { 

406 'watch': watch, 

407 'added_at': first['happened_at'] 

408 if first 

409 else row['downloaded_at'], 

410 'outcome': first['outcome'] if first else OUTCOME_UNKNOWN, 

411 'initial': bool(first and first['initial']), 

412 } 

413 known = holders.get(watch.id) 

414 if known is None or str(entry['added_at']) < str(known['added_at']): 

415 holders[watch.id] = entry 

416 return { 

417 'origin': _origin(events, watches_by_id), 

418 'holders': sorted(holders.values(), key=lambda h: str(h['added_at'])), 

419 } 

420 

421 

422def _origin( 

423 events: list[dict[str, Any]], 

424 watches_by_id: dict[int, MonitoredPlaylist], 

425) -> Optional[dict[str, Any]]: 

426 event = next( 

427 (e for e in events if e['outcome'] == OUTCOME_DOWNLOADED), None 

428 ) 

429 estimated = event is None 

430 if event is None: 

431 event = next( 

432 (e for e in events if e['outcome'] == OUTCOME_UNKNOWN), None 

433 ) 

434 if event is None: 

435 return None 

436 watch = watches_by_id.get(int(event['playlist_id'])) 

437 # Gone, or since pointed at another playlist: only the name is left. 

438 current = watch is not None and watch.spotify_id == event['watch_key'] 

439 return { 

440 'watch': watch if current else None, 

441 'watch_id': int(event['playlist_id']), 

442 'user_id': int(event.get('user_id') or 0), 

443 'name': str(event['watch_name']), 

444 'at': event['happened_at'], 

445 'provider': str(event['provider'] or ''), 

446 'initial': bool(event['initial']), 

447 'estimated': estimated, 

448 }