Coverage for downtify/db/batching.py: 100%
22 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Many small writes, grouped into a few short transactions.
3Each ``with store._connect()`` block on its own opens a connection and
4commits; on SQLite that also means opening the file and its ``-wal`` and
5``-shm`` again. A loop that writes one row at a time pays that per row:
6about 50 ms a row on a plain disk, so a start-up pass over tens of
7thousands of watch-history rows took most of an hour.
9``Database.connect()`` is re-entrant on a thread, so holding one open
10around a run of rows makes the store methods inside it share that one
11connection and commit once. Nothing else changes: the same statements
12run, in the same order, on the same connection a lone call would use.
13"""
15from __future__ import annotations
17import time
18from collections.abc import Iterable, Iterator
19from typing import TypeVar
21from downtify.db.core import Database
23T = TypeVar('T')
25_END = object()
27#: A transaction ends after this many rows ...
28BATCH_ROWS = 500
29#: ... or after this long, whichever comes first. On SQLite a write
30#: transaction keeps other writers waiting (up to the 30 s busy timeout),
31#: so a slow disk must not stretch it.
32BATCH_SECONDS = 0.5
35def in_short_transactions(
36 db: Database,
37 items: Iterable[T],
38 *,
39 max_rows: int = BATCH_ROWS,
40 max_seconds: float = BATCH_SECONDS,
41) -> Iterator[T]:
42 """Yield *items* with a connection of *db* held open around them.
44 Every *max_rows* items or *max_seconds* the transaction commits and a
45 new one starts. On SQLite the write lock is taken at the first write,
46 as with a lone store call, and other writers wait for the commit
47 (up to the busy timeout).
49 An exception raised by the caller's loop body rolls back only the
50 rows of the current transaction (earlier ones are already committed)
51 and propagates. Stopping the loop early (``break``) also rolls back
52 the current transaction, so callers should not.
53 """
55 iterator = iter(items)
56 # Looked up before a transaction opens, so none is ever left empty.
57 pending: object = next(iterator, _END)
58 while pending is not _END:
59 started = time.monotonic()
60 count = 0
61 with db.connect():
62 while pending is not _END:
63 yield pending # type: ignore[misc]
64 count += 1
65 pending = next(iterator, _END)
66 if (
67 count >= max_rows
68 or time.monotonic() - started >= max_seconds
69 ):
70 break