Coverage for downtify/traffic.py: 94%
117 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Per-user traffic accounting (Settings → Users summary).
3Two directions, counted separately:
5* ``in`` — bytes fetched from upstream: finished library downloads and
6 stream-cache fetches. A dedup hit fetches nothing and counts nothing.
7* ``out`` — bytes served to clients, counted by an ASGI middleware that
8 sums response bodies per signed-in user (audio, covers, archives, API
9 answers alike; WebSocket frames never pass through it).
11Counters, not events: one row per ``(user_id, direction)``, updated in
12place. The Users summary reads them; deleting an account drops its
13rows with everything else.
15Adding only touches memory - the middleware runs on the event loop for
16every response, so a database write there would stall every request
17(and wait on any other writer of the shared library database).
18:func:`traffic_flush_loop` saves the pending bytes in the background;
19:meth:`TrafficCounters.totals` includes what isn't saved yet.
20"""
22from __future__ import annotations
24import asyncio
25import threading
26from datetime import datetime, timezone
27from pathlib import Path
28from typing import Any, Awaitable, Callable, MutableMapping
30from loguru import logger
32from .db.config import Dialect
33from .db.core import Connection, Database, as_database
35DIRECTION_IN = 'in'
36DIRECTION_OUT = 'out'
37DIRECTIONS = (DIRECTION_IN, DIRECTION_OUT)
38#: Seconds between saves of the pending counters.
39FLUSH_INTERVAL_SECONDS = 30
41_UPSERT = """INSERT INTO traffic_counters
42 (user_id, direction, total_bytes, updated_at)
43 VALUES (?, ?, ?, ?)
44 ON CONFLICT(user_id, direction) DO UPDATE SET
45 total_bytes = traffic_counters.total_bytes + excluded.total_bytes,
46 updated_at = excluded.updated_at"""
49def _now_iso() -> str:
50 return datetime.now(timezone.utc).isoformat()
53class TrafficCounters:
54 """``(user_id, direction)`` byte counters in ``downtify_library.db``."""
56 def __init__(self, path_or_db: str | Path | Database) -> None:
57 self.db = as_database(path_or_db, 'library')
58 if isinstance(path_or_db, Database):
59 self._path = str(self.db.path) if self.db.path else None
60 else:
61 self._path = str(path_or_db)
62 self._lock = threading.Lock()
63 self._pending: dict[tuple[int, str], int] = {}
64 self._pending_lock = threading.Lock()
65 if self.db.dialect is Dialect.SQLITE:
66 self._init_db()
68 def _connect(self) -> Connection:
69 return self.db.connect()
71 def _init_db(self) -> None:
72 with self._connect() as conn:
73 conn.execute("""
74 CREATE TABLE IF NOT EXISTS traffic_counters (
75 user_id INTEGER NOT NULL,
76 direction TEXT NOT NULL,
77 total_bytes INTEGER NOT NULL DEFAULT 0,
78 updated_at TEXT NOT NULL,
79 PRIMARY KEY (user_id, direction)
80 )
81 """)
83 def add(self, user_id: int, direction: str, nbytes: int) -> None:
84 """Add *nbytes* to a counter, in memory (see :meth:`flush`).
85 Ignores junk (nobody, bad direction, nothing)."""
87 uid = int(user_id or 0)
88 try:
89 count = int(nbytes)
90 except (TypeError, ValueError):
91 return
92 if not uid or direction not in DIRECTIONS or count <= 0:
93 return
94 key = (uid, direction)
95 with self._pending_lock:
96 self._pending[key] = self._pending.get(key, 0) + count
98 def flush(self) -> None:
99 """Save the pending bytes in one transaction. On failure they
100 go back to the pending counters for the next try."""
102 with self._pending_lock:
103 batch, self._pending = self._pending, {}
104 if not batch:
105 return
106 now = _now_iso()
107 rows = [(uid, d, n, now) for (uid, d), n in batch.items()]
108 try:
109 with self._lock, self._connect() as conn:
110 conn.executemany(_UPSERT, rows)
111 except Exception:
112 # Covers DatabaseError (the Postgres and SQLite error wrappers
113 # both derive from Exception); the bytes go back either way.
114 with self._pending_lock:
115 for key, count in batch.items():
116 self._pending[key] = self._pending.get(key, 0) + count
117 raise
119 def totals(self) -> dict[int, dict[str, int]]:
120 """``{user_id: {'in': bytes, 'out': bytes}}``, pending bytes
121 included."""
123 with self._connect() as conn:
124 rows = conn.execute(
125 'SELECT user_id, direction, total_bytes FROM traffic_counters'
126 ).fetchall()
127 result: dict[int, dict[str, int]] = {}
128 for row in rows:
129 entry = result.setdefault(int(row['user_id']), {'in': 0, 'out': 0})
130 if str(row['direction']) in DIRECTIONS:
131 entry[str(row['direction'])] = int(row['total_bytes'] or 0)
132 with self._pending_lock:
133 pending = dict(self._pending)
134 for (uid, direction), count in pending.items():
135 entry = result.setdefault(uid, {'in': 0, 'out': 0})
136 entry[direction] += count
137 return result
139 def remove_user(self, user_id: int) -> None:
140 uid = int(user_id)
141 with self._pending_lock:
142 for direction in DIRECTIONS:
143 self._pending.pop((uid, direction), None)
144 with self._lock, self._connect() as conn:
145 conn.execute(
146 'DELETE FROM traffic_counters WHERE user_id = ?',
147 (uid,),
148 )
151async def traffic_flush_loop(
152 get_counters: Callable[[], Any],
153 interval_seconds: float = FLUSH_INTERVAL_SECONDS,
154) -> None:
155 """Background task: save the pending counters every
156 *interval_seconds*, off the event loop. Shutdown flushes once more
157 (see ``api.shutdown_resources``)."""
159 while True:
160 await asyncio.sleep(interval_seconds)
161 counters = get_counters()
162 if counters is None:
163 continue
164 try:
165 await asyncio.to_thread(counters.flush)
166 except Exception:
167 logger.opt(exception=True).debug('Could not save traffic')
170class TrafficMiddleware:
171 """Count served bytes per signed-in user.
173 Runs inside the auth middleware (which puts the principal in the
174 scope), summing ``http.response.body`` chunks and filing one counter
175 update when the response is done. Failures to count never fail the
176 response itself.
177 """
179 def __init__(
180 self,
181 app: Callable[..., Awaitable[None]],
182 get_counters: Callable[[], Any],
183 ) -> None:
184 self._app = app
185 self._get_counters = get_counters
187 async def __call__(
188 self,
189 scope: MutableMapping[str, Any],
190 receive: Callable[..., Awaitable[Any]],
191 send: Callable[..., Awaitable[None]],
192 ) -> None:
193 if scope.get('type') != 'http':
194 await self._app(scope, receive, send)
195 return
196 principal = (scope.get('state') or {}).get('principal')
197 user_id = int(getattr(principal, 'user_id', 0) or 0)
198 if not user_id:
199 await self._app(scope, receive, send)
200 return
201 total = 0
203 async def counting_send(message: MutableMapping[str, Any]) -> None:
204 nonlocal total
205 if message.get('type') == 'http.response.body':
206 body = message.get('body', b'')
207 total += len(body) if isinstance(body, bytes) else 0
208 await send(message)
210 try:
211 await self._app(scope, receive, counting_send)
212 finally:
213 if total > 0:
214 try:
215 counters = self._get_counters()
216 if counters is not None:
217 counters.add(user_id, DIRECTION_OUT, total)
218 except Exception:
219 pass