Coverage for downtify/catalog_worker.py: 100%
26 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""CPU-only catalog folding, isolated from the web process's Python GIL.
3Only plain listing rows cross the process boundary. Database stores and
4filesystem mutations remain in the application process. One worker and
5one submission at a time bound memory use during download bursts.
6"""
8from __future__ import annotations
10import multiprocessing
11import threading
12from concurrent.futures import ProcessPoolExecutor
13from typing import Any, Optional
15_LOCK = threading.Lock()
16_WORK = threading.Lock()
17_pool: Optional[ProcessPoolExecutor] = None
20def start_catalog_worker() -> None:
21 """Enable the process worker for a running web application's lifespan."""
23 global _pool
24 with _LOCK:
25 if _pool is None:
26 _pool = ProcessPoolExecutor(
27 max_workers=1,
28 mp_context=multiprocessing.get_context('spawn'),
29 )
32def stop_catalog_worker() -> None:
33 """Release the web pool and restore inline library use; idempotent."""
35 global _pool
36 with _LOCK:
37 pool, _pool = _pool, None
38 if pool is not None:
39 pool.shutdown(wait=True, cancel_futures=True)
42def fold_library_entries(
43 entries: list[dict[str, Any]],
44 primary_extra_id: str = '',
45) -> list[dict[str, Any]]:
46 """Fold in the worker, or inline outside a web application's lifespan.
48 A failed calculation never publishes a partial snapshot. Let callers
49 report the failure and retain their last successfully stored listing.
50 """
52 from .external_library import finish_library_entries # noqa: PLC0415 # import cycle
54 with _WORK:
55 with _LOCK:
56 pool = _pool
57 future = (
58 pool.submit(finish_library_entries, entries, primary_extra_id)
59 if primary_extra_id and pool is not None
60 else pool.submit(finish_library_entries, entries)
61 if pool is not None
62 else None
63 )
64 if future is not None:
65 return future.result()
66 return (
67 finish_library_entries(entries, primary_extra_id)
68 if primary_extra_id
69 else finish_library_entries(entries)
70 )