Coverage for downtify/traffic.py: 94%

117 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Per-user traffic accounting (Settings → Users summary). 

2 

3Two directions, counted separately: 

4 

5* ``in`` — bytes fetched from upstream: finished library downloads and 

6 stream-cache fetches. A dedup hit fetches nothing and counts nothing. 

7* ``out`` — bytes served to clients, counted by an ASGI middleware that 

8 sums response bodies per signed-in user (audio, covers, archives, API 

9 answers alike; WebSocket frames never pass through it). 

10 

11Counters, not events: one row per ``(user_id, direction)``, updated in 

12place. The Users summary reads them; deleting an account drops its 

13rows with everything else. 

14 

15Adding only touches memory - the middleware runs on the event loop for 

16every response, so a database write there would stall every request 

17(and wait on any other writer of the shared library database). 

18:func:`traffic_flush_loop` saves the pending bytes in the background; 

19:meth:`TrafficCounters.totals` includes what isn't saved yet. 

20""" 

21 

22from __future__ import annotations 

23 

24import asyncio 

25import threading 

26from datetime import datetime, timezone 

27from pathlib import Path 

28from typing import Any, Awaitable, Callable, MutableMapping 

29 

30from loguru import logger 

31 

32from .db.config import Dialect 

33from .db.core import Connection, Database, as_database 

34 

35DIRECTION_IN = 'in' 

36DIRECTION_OUT = 'out' 

37DIRECTIONS = (DIRECTION_IN, DIRECTION_OUT) 

38#: Seconds between saves of the pending counters. 

39FLUSH_INTERVAL_SECONDS = 30 

40 

41_UPSERT = """INSERT INTO traffic_counters 

42 (user_id, direction, total_bytes, updated_at) 

43 VALUES (?, ?, ?, ?) 

44 ON CONFLICT(user_id, direction) DO UPDATE SET 

45 total_bytes = traffic_counters.total_bytes + excluded.total_bytes, 

46 updated_at = excluded.updated_at""" 

47 

48 

49def _now_iso() -> str: 

50 return datetime.now(timezone.utc).isoformat() 

51 

52 

53class TrafficCounters: 

54 """``(user_id, direction)`` byte counters in ``downtify_library.db``.""" 

55 

56 def __init__(self, path_or_db: str | Path | Database) -> None: 

57 self.db = as_database(path_or_db, 'library') 

58 if isinstance(path_or_db, Database): 

59 self._path = str(self.db.path) if self.db.path else None 

60 else: 

61 self._path = str(path_or_db) 

62 self._lock = threading.Lock() 

63 self._pending: dict[tuple[int, str], int] = {} 

64 self._pending_lock = threading.Lock() 

65 if self.db.dialect is Dialect.SQLITE: 

66 self._init_db() 

67 

68 def _connect(self) -> Connection: 

69 return self.db.connect() 

70 

71 def _init_db(self) -> None: 

72 with self._connect() as conn: 

73 conn.execute(""" 

74 CREATE TABLE IF NOT EXISTS traffic_counters ( 

75 user_id INTEGER NOT NULL, 

76 direction TEXT NOT NULL, 

77 total_bytes INTEGER NOT NULL DEFAULT 0, 

78 updated_at TEXT NOT NULL, 

79 PRIMARY KEY (user_id, direction) 

80 ) 

81 """) 

82 

83 def add(self, user_id: int, direction: str, nbytes: int) -> None: 

84 """Add *nbytes* to a counter, in memory (see :meth:`flush`). 

85 Ignores junk (nobody, bad direction, nothing).""" 

86 

87 uid = int(user_id or 0) 

88 try: 

89 count = int(nbytes) 

90 except (TypeError, ValueError): 

91 return 

92 if not uid or direction not in DIRECTIONS or count <= 0: 

93 return 

94 key = (uid, direction) 

95 with self._pending_lock: 

96 self._pending[key] = self._pending.get(key, 0) + count 

97 

98 def flush(self) -> None: 

99 """Save the pending bytes in one transaction. On failure they 

100 go back to the pending counters for the next try.""" 

101 

102 with self._pending_lock: 

103 batch, self._pending = self._pending, {} 

104 if not batch: 

105 return 

106 now = _now_iso() 

107 rows = [(uid, d, n, now) for (uid, d), n in batch.items()] 

108 try: 

109 with self._lock, self._connect() as conn: 

110 conn.executemany(_UPSERT, rows) 

111 except Exception: 

112 # Covers DatabaseError (the Postgres and SQLite error wrappers 

113 # both derive from Exception); the bytes go back either way. 

114 with self._pending_lock: 

115 for key, count in batch.items(): 

116 self._pending[key] = self._pending.get(key, 0) + count 

117 raise 

118 

119 def totals(self) -> dict[int, dict[str, int]]: 

120 """``{user_id: {'in': bytes, 'out': bytes}}``, pending bytes 

121 included.""" 

122 

123 with self._connect() as conn: 

124 rows = conn.execute( 

125 'SELECT user_id, direction, total_bytes FROM traffic_counters' 

126 ).fetchall() 

127 result: dict[int, dict[str, int]] = {} 

128 for row in rows: 

129 entry = result.setdefault(int(row['user_id']), {'in': 0, 'out': 0}) 

130 if str(row['direction']) in DIRECTIONS: 

131 entry[str(row['direction'])] = int(row['total_bytes'] or 0) 

132 with self._pending_lock: 

133 pending = dict(self._pending) 

134 for (uid, direction), count in pending.items(): 

135 entry = result.setdefault(uid, {'in': 0, 'out': 0}) 

136 entry[direction] += count 

137 return result 

138 

139 def remove_user(self, user_id: int) -> None: 

140 uid = int(user_id) 

141 with self._pending_lock: 

142 for direction in DIRECTIONS: 

143 self._pending.pop((uid, direction), None) 

144 with self._lock, self._connect() as conn: 

145 conn.execute( 

146 'DELETE FROM traffic_counters WHERE user_id = ?', 

147 (uid,), 

148 ) 

149 

150 

151async def traffic_flush_loop( 

152 get_counters: Callable[[], Any], 

153 interval_seconds: float = FLUSH_INTERVAL_SECONDS, 

154) -> None: 

155 """Background task: save the pending counters every 

156 *interval_seconds*, off the event loop. Shutdown flushes once more 

157 (see ``api.shutdown_resources``).""" 

158 

159 while True: 

160 await asyncio.sleep(interval_seconds) 

161 counters = get_counters() 

162 if counters is None: 

163 continue 

164 try: 

165 await asyncio.to_thread(counters.flush) 

166 except Exception: 

167 logger.opt(exception=True).debug('Could not save traffic') 

168 

169 

170class TrafficMiddleware: 

171 """Count served bytes per signed-in user. 

172 

173 Runs inside the auth middleware (which puts the principal in the 

174 scope), summing ``http.response.body`` chunks and filing one counter 

175 update when the response is done. Failures to count never fail the 

176 response itself. 

177 """ 

178 

179 def __init__( 

180 self, 

181 app: Callable[..., Awaitable[None]], 

182 get_counters: Callable[[], Any], 

183 ) -> None: 

184 self._app = app 

185 self._get_counters = get_counters 

186 

187 async def __call__( 

188 self, 

189 scope: MutableMapping[str, Any], 

190 receive: Callable[..., Awaitable[Any]], 

191 send: Callable[..., Awaitable[None]], 

192 ) -> None: 

193 if scope.get('type') != 'http': 

194 await self._app(scope, receive, send) 

195 return 

196 principal = (scope.get('state') or {}).get('principal') 

197 user_id = int(getattr(principal, 'user_id', 0) or 0) 

198 if not user_id: 

199 await self._app(scope, receive, send) 

200 return 

201 total = 0 

202 

203 async def counting_send(message: MutableMapping[str, Any]) -> None: 

204 nonlocal total 

205 if message.get('type') == 'http.response.body': 

206 body = message.get('body', b'') 

207 total += len(body) if isinstance(body, bytes) else 0 

208 await send(message) 

209 

210 try: 

211 await self._app(scope, receive, counting_send) 

212 finally: 

213 if total > 0: 

214 try: 

215 counters = self._get_counters() 

216 if counters is not None: 

217 counters.add(user_id, DIRECTION_OUT, total) 

218 except Exception: 

219 pass