Coverage for downtify/catalog_worker.py: 100%

26 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""CPU-only catalog folding, isolated from the web process's Python GIL. 

2 

3Only plain listing rows cross the process boundary. Database stores and 

4filesystem mutations remain in the application process. One worker and 

5one submission at a time bound memory use during download bursts. 

6""" 

7 

8from __future__ import annotations 

9 

10import multiprocessing 

11import threading 

12from concurrent.futures import ProcessPoolExecutor 

13from typing import Any, Optional 

14 

15_LOCK = threading.Lock() 

16_WORK = threading.Lock() 

17_pool: Optional[ProcessPoolExecutor] = None 

18 

19 

20def start_catalog_worker() -> None: 

21 """Enable the process worker for a running web application's lifespan.""" 

22 

23 global _pool 

24 with _LOCK: 

25 if _pool is None: 

26 _pool = ProcessPoolExecutor( 

27 max_workers=1, 

28 mp_context=multiprocessing.get_context('spawn'), 

29 ) 

30 

31 

32def stop_catalog_worker() -> None: 

33 """Release the web pool and restore inline library use; idempotent.""" 

34 

35 global _pool 

36 with _LOCK: 

37 pool, _pool = _pool, None 

38 if pool is not None: 

39 pool.shutdown(wait=True, cancel_futures=True) 

40 

41 

42def fold_library_entries( 

43 entries: list[dict[str, Any]], 

44 primary_extra_id: str = '', 

45) -> list[dict[str, Any]]: 

46 """Fold in the worker, or inline outside a web application's lifespan. 

47 

48 A failed calculation never publishes a partial snapshot. Let callers 

49 report the failure and retain their last successfully stored listing. 

50 """ 

51 

52 from .external_library import finish_library_entries # noqa: PLC0415 # import cycle 

53 

54 with _WORK: 

55 with _LOCK: 

56 pool = _pool 

57 future = ( 

58 pool.submit(finish_library_entries, entries, primary_extra_id) 

59 if primary_extra_id and pool is not None 

60 else pool.submit(finish_library_entries, entries) 

61 if pool is not None 

62 else None 

63 ) 

64 if future is not None: 

65 return future.result() 

66 return ( 

67 finish_library_entries(entries, primary_extra_id) 

68 if primary_extra_id 

69 else finish_library_entries(entries) 

70 )