Coverage for src/copytrading_engine/backup/service.py: 86%
219 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Backup and restore for one installation: create, preview, stage, and activate candidates."""
3from __future__ import annotations
5import asyncio
6import json
7import os
8import shutil
9import tempfile
10import threading
11import uuid
12from pathlib import Path, PurePosixPath
14from copytrading_engine.backup.archive import inspect_backup_archive
15from copytrading_engine.backup.configuration import (
16 archive_credential_references,
17 candidate_credential_references,
18 read_installation_id,
19)
20from copytrading_engine.backup.files import (
21 copy_private_file,
22 fsync_directory,
23 fsync_tree,
24 require_directory,
25)
26from copytrading_engine.backup.manifest import (
27 ACCOUNT_DATABASE,
28 RESTORE_STAGE,
29 BackupManifest,
30 BackupManifestError,
31 PendingRestoreCandidate,
32 RestorePreview,
33 json_object,
34 manifest_digest,
35)
36from copytrading_engine.backup.ports import (
37 AccountSnapshotSchemaProvider,
38 BackupFence,
39 RestoredAccountPreparer,
40 SchemaCatalogProvider,
41)
42from copytrading_engine.backup.restore.candidates import (
43 candidate_intent_payload,
44 clear_restore_staging,
45 persist_candidate_intent,
46 read_active_generation,
47 read_candidate_intent,
48 reap_interrupted_restore_candidates,
49 remove_candidate_intent,
50 remove_generation_entry,
51 stage_restore,
52 validate_candidate_generation,
53 validate_published_restore_candidate,
54 verify_staged_tree,
55 write_candidate_manifest,
56)
57from copytrading_engine.backup.restore.gate import (
58 clear_restore_manual_disabled,
59 persist_restore_manual_disabled,
60 restore_manual_disabled,
61 restore_manual_disabled_path,
62)
63from copytrading_engine.backup.snapshot import create_backup
66class BackupRestoreService:
67 """Coordinate a consistent snapshot and isolate an explicitly selected restore."""
69 def __init__(
70 self,
71 data_dir: Path,
72 fence: BackupFence,
73 *,
74 schema_catalog: SchemaCatalogProvider,
75 snapshot_schema: AccountSnapshotSchemaProvider,
76 owner_support_directory: Path,
77 account_preparer: RestoredAccountPreparer | None = None,
78 ) -> None:
79 self._data_dir = data_dir
80 self._fence = fence
81 self._schema_catalog = schema_catalog
82 self._snapshot_schema = snapshot_schema
83 self._owner_support_directory = owner_support_directory
84 self._account_preparer = account_preparer
85 self._staging_lock = threading.RLock()
86 self._staged_manifests: dict[str, BackupManifest] = {}
87 self._prepared_candidates: dict[str, str] = {}
89 @property
90 def staging_root(self) -> Path:
91 root = self._owner_support_directory / ".restore-staging"
92 try:
93 root.mkdir(mode=0o700)
94 except FileExistsError:
95 pass
96 require_directory(root, "restore staging directory is unavailable")
97 os.chmod(root, 0o700)
98 return root
100 async def create_backup(self, destination: Path) -> BackupManifest:
101 async with self._fence():
102 worker = asyncio.create_task(
103 asyncio.to_thread(
104 create_backup,
105 self._data_dir,
106 self._owner_support_directory,
107 destination,
108 self._schema_catalog,
109 self._snapshot_schema,
110 )
111 )
112 cancelled = False
113 while True:
114 try:
115 result = await asyncio.shield(worker)
116 break
117 except asyncio.CancelledError:
118 cancelled = True
119 if worker.done(): 119 ↛ 120line 119 didn't jump to line 120 because the condition on line 119 was never true
120 break
121 if cancelled:
122 worker.result()
123 raise asyncio.CancelledError
124 return result
126 def preview_restore(self, archive_path: Path) -> RestorePreview:
127 manifest = inspect_backup_archive(
128 archive_path,
129 self._schema_catalog,
130 self._snapshot_schema,
131 )
132 current_identity = read_installation_id(self._owner_support_directory)
133 account_ids = tuple(
134 sorted(
135 member.path.split("/")[1]
136 for member in manifest.members
137 if ACCOUNT_DATABASE.fullmatch(member.path)
138 )
139 )
140 credential_references = archive_credential_references(archive_path, manifest)
141 return RestorePreview(
142 manifest=manifest,
143 matches_installation=manifest.installation_id == current_identity,
144 account_ids=account_ids,
145 credential_references=credential_references,
146 )
148 def stage_restore(self, archive_path: Path, staging_root: Path) -> tuple[Path, RestorePreview]:
149 with self._staging_lock:
150 service_root = self.staging_root
151 try:
152 supplied_root = staging_root.resolve(strict=True)
153 expected_root = service_root.resolve(strict=True)
154 except OSError as exc:
155 raise BackupManifestError("restore staging directory is unavailable") from exc
156 require_directory(staging_root, "restore staging directory is unsafe")
157 if supplied_root != expected_root:
158 raise BackupManifestError("restore staging must use the service-owned directory")
159 clear_restore_staging(service_root)
160 self._staged_manifests.clear()
161 self._prepared_candidates.clear()
162 preview = self.preview_restore(archive_path)
163 staged = stage_restore(
164 archive_path,
165 service_root,
166 preview.manifest,
167 self._schema_catalog,
168 self._snapshot_schema,
169 )
170 self._staged_manifests[staged.name] = preview.manifest
171 return staged, preview
173 def prepare_restore_candidate(self, staging_identifier: str) -> str:
174 """Copy one validated preview into an inactive generation and gate startup."""
175 with self._staging_lock:
176 if not isinstance(staging_identifier, str) or not RESTORE_STAGE.fullmatch(
177 staging_identifier
178 ):
179 raise BackupManifestError("restore staging identifier is invalid")
180 prior_candidate = self._prepared_candidates.get(staging_identifier)
181 if prior_candidate is not None:
182 validate_published_restore_candidate(
183 self._owner_support_directory,
184 prior_candidate,
185 self._schema_catalog,
186 self._snapshot_schema,
187 )
188 return prior_candidate
189 manifest = self._staged_manifests.get(staging_identifier)
190 if manifest is None: 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true
191 raise BackupManifestError("restore staging identifier is unknown")
193 stage_root = self.staging_root
194 staged = stage_root / staging_identifier
195 require_directory(staged, "restore staging directory is unavailable")
196 owner = self._owner_support_directory
197 require_directory(owner, "restore owner directory is unavailable")
198 if read_installation_id(owner) != manifest.installation_id: 198 ↛ 199line 198 didn't jump to line 199 because the condition on line 198 was never true
199 raise BackupManifestError("backup belongs to a different installation")
200 generations = owner / "generations"
201 require_directory(generations, "operational generations directory is unavailable")
202 active_generation = read_active_generation(owner)
203 reap_interrupted_restore_candidates(owner, generations, active_generation)
204 if restore_manual_disabled(restore_manual_disabled_path(owner)): 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true
205 raise BackupManifestError("another restore activation is already in progress")
207 verify_staged_tree(
208 staged,
209 manifest,
210 self._schema_catalog,
211 self._snapshot_schema,
212 )
213 candidate_id = str(uuid.uuid4())
214 candidate = generations / candidate_id
215 if candidate.exists() or candidate.is_symlink(): 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 raise BackupManifestError("restore candidate identifier already exists")
217 temporary = Path(tempfile.mkdtemp(prefix=".restore-candidate-", dir=generations))
218 os.chmod(temporary, 0o700)
219 published = False
220 try:
221 for member in manifest.members:
222 source = staged.joinpath(*PurePosixPath(member.path).parts)
223 target = temporary.joinpath(*PurePosixPath(member.path).parts)
224 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
225 digest, size = copy_private_file(source, target, member.size)
226 if digest != member.sha256 or size != member.size: 226 ↛ 227line 226 didn't jump to line 227 because the condition on line 226 was never true
227 raise BackupManifestError("restore member changed after validation")
229 account_members = tuple(
230 member for member in manifest.members if ACCOUNT_DATABASE.fullmatch(member.path)
231 )
232 if account_members and self._account_preparer is None: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true
233 raise BackupManifestError(
234 "restored account manual-state preparation is unavailable"
235 )
236 for member in account_members:
237 account_path = temporary.joinpath(*PurePosixPath(member.path).parts)
238 assert self._account_preparer is not None
239 try:
240 self._account_preparer(account_path)
241 except Exception as exc:
242 raise BackupManifestError(
243 "restored account could not be made manual-disabled"
244 ) from exc
246 candidate_manifest_sha256 = write_candidate_manifest(temporary, manifest)
247 validate_candidate_generation(
248 temporary,
249 manifest,
250 self._schema_catalog,
251 self._snapshot_schema,
252 )
253 intent_payload = candidate_intent_payload(
254 manifest,
255 active_generation,
256 candidate_id,
257 candidate_manifest_sha256,
258 )
259 persist_candidate_intent(owner, intent_payload)
260 fsync_tree(temporary)
261 os.rename(temporary, candidate)
262 published = True
263 fsync_directory(generations)
265 marker_payload = json.dumps(
266 {
267 "version": 2,
268 "installation_id": manifest.installation_id,
269 "previous_generation": active_generation,
270 "candidate_generation": candidate_id,
271 "backup_created_at": manifest.created_at.isoformat(),
272 "manifest_sha256": manifest_digest(manifest),
273 "candidate_manifest_sha256": candidate_manifest_sha256,
274 },
275 sort_keys=True,
276 separators=(",", ":"),
277 ).encode("utf-8")
278 persist_restore_manual_disabled(restore_manual_disabled_path(owner), marker_payload)
279 self._prepared_candidates[staging_identifier] = candidate_id
280 return candidate_id
281 except BaseException:
282 if published:
283 if not restore_manual_disabled(restore_manual_disabled_path(owner)):
284 remove_generation_entry(candidate)
285 fsync_directory(generations)
286 remove_candidate_intent(owner, candidate_id)
287 else:
288 shutil.rmtree(temporary, ignore_errors=True)
289 remove_candidate_intent(owner, candidate_id)
290 raise
292 def validate_restore_candidate(self, candidate_identifier: str) -> BackupManifest:
293 """Reload and validate a prepared candidate using only durable evidence."""
294 with self._staging_lock:
295 return validate_published_restore_candidate(
296 self._owner_support_directory,
297 candidate_identifier,
298 self._schema_catalog,
299 self._snapshot_schema,
300 )
302 def pending_restore_candidate(self) -> PendingRestoreCandidate | None:
303 """Read the durable restore intent and gate after an uncertain IPC result or restart."""
304 with self._staging_lock:
305 owner = self._owner_support_directory
306 marker_path = restore_manual_disabled_path(owner)
307 if not restore_manual_disabled(marker_path):
308 return None
309 marker = json_object(marker_path.read_bytes())
310 candidate_id = marker.get("candidate_generation")
311 intent = read_candidate_intent(owner)
312 if intent is None or not isinstance(candidate_id, str): 312 ↛ 313line 312 didn't jump to line 313 because the condition on line 312 was never true
313 raise BackupManifestError("restore candidate status is unavailable")
314 if ( 314 ↛ 327line 314 didn't jump to line 327 because the condition on line 314 was never true
315 set(marker)
316 != {
317 "version",
318 "installation_id",
319 "previous_generation",
320 "candidate_generation",
321 "backup_created_at",
322 "manifest_sha256",
323 "candidate_manifest_sha256",
324 }
325 or marker.get("version") != 2
326 ):
327 raise BackupManifestError("restore gate is invalid")
328 active_generation = read_active_generation(owner)
329 previous_generation = intent["previous_generation"]
330 expected = {
331 "installation_id": intent["installation_id"],
332 "previous_generation": previous_generation,
333 "candidate_generation": candidate_id,
334 "manifest_sha256": intent["manifest_sha256"],
335 "candidate_manifest_sha256": intent["candidate_manifest_sha256"],
336 }
337 if any(marker.get(key) != value for key, value in expected.items()): 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true
338 raise BackupManifestError("restore gate does not match candidate intent")
339 if active_generation not in {previous_generation, candidate_id}: 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true
340 raise BackupManifestError("active generation does not match restore candidate")
342 environment_ids: tuple[str, ...] = ()
343 account_ids: tuple[str, ...] = ()
344 credential_references: tuple[str, ...] = ()
345 candidate_valid = False
346 try:
347 manifest = validate_published_restore_candidate(
348 owner,
349 candidate_id,
350 self._schema_catalog,
351 self._snapshot_schema,
352 )
353 except BackupManifestError:
354 pass
355 else:
356 candidate_valid = True
357 environment_ids = manifest.environment_ids
358 account_ids = tuple(
359 sorted(
360 member.path.split("/")[1]
361 for member in manifest.members
362 if ACCOUNT_DATABASE.fullmatch(member.path)
363 )
364 )
365 credential_references = candidate_credential_references(
366 owner / "generations" / candidate_id, manifest
367 )
368 return PendingRestoreCandidate(
369 candidate_id=candidate_id,
370 previous_generation=str(previous_generation),
371 active_generation=active_generation,
372 installation_id=str(intent["installation_id"]),
373 environment_ids=environment_ids,
374 account_ids=account_ids,
375 credential_references=credential_references,
376 candidate_valid=candidate_valid,
377 )
379 def abort_restore_candidate(self, candidate_identifier: str) -> None:
380 """Abort only after the previous generation is selected and candidate absence is durable."""
381 with self._staging_lock:
382 status = self.pending_restore_candidate()
383 if status is None or status.candidate_id != candidate_identifier: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true
384 raise BackupManifestError("restore candidate is unavailable for rollback")
385 if status.active_generation != status.previous_generation:
386 raise BackupManifestError("restore candidate must be switched back before rollback")
387 owner = self._owner_support_directory
388 candidate = owner / "generations" / candidate_identifier
389 if candidate.exists() or candidate.is_symlink(): 389 ↛ 393line 389 didn't jump to line 393 because the condition on line 389 was always true
390 if not remove_generation_entry(candidate): 390 ↛ 391line 390 didn't jump to line 391 because the condition on line 390 was never true
391 raise BackupManifestError("restore candidate could not be removed")
392 fsync_directory(owner / "generations")
393 clear_restore_manual_disabled(restore_manual_disabled_path(owner))
394 remove_candidate_intent(owner, candidate_identifier)
395 self._prepared_candidates = {
396 staging_id: candidate_id
397 for staging_id, candidate_id in self._prepared_candidates.items()
398 if candidate_id != candidate_identifier
399 }
401 def resolve_active_restore_candidate(
402 self, candidate_identifier: str
403 ) -> tuple[BackupManifest, Path]:
404 """Validate and resolve a candidate only after its generation is selected."""
405 with self._staging_lock:
406 manifest = validate_published_restore_candidate(
407 self._owner_support_directory,
408 candidate_identifier,
409 self._schema_catalog,
410 self._snapshot_schema,
411 )
412 if read_active_generation(self._owner_support_directory) != candidate_identifier: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true
413 raise BackupManifestError("restore candidate generation is not active")
414 candidate = self._owner_support_directory / "generations" / candidate_identifier
415 require_directory(candidate, "restore candidate generation is unavailable")
416 return manifest, candidate
418 def complete_restore_candidate(self, candidate_identifier: str) -> None:
419 """Clear the durable startup gate after a successful read-only preflight."""
420 with self._staging_lock:
421 owner = self._owner_support_directory
422 manifest = validate_published_restore_candidate(
423 owner,
424 candidate_identifier,
425 self._schema_catalog,
426 self._snapshot_schema,
427 )
428 if read_active_generation(owner) != candidate_identifier: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true
429 raise BackupManifestError("restore candidate generation is not active")
430 intent = read_candidate_intent(owner)
431 if ( 431 ↛ 437line 431 didn't jump to line 437 because the condition on line 431 was never true
432 intent is None
433 or intent["installation_id"] != manifest.installation_id
434 or intent["candidate_generation"] != candidate_identifier
435 or intent["manifest_sha256"] != manifest_digest(manifest)
436 ):
437 raise BackupManifestError("restore candidate intent does not match activation")
438 clear_restore_manual_disabled(restore_manual_disabled_path(owner))