Coverage for src/copytrading_engine/backup/service.py: 86%

219 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Backup and restore for one installation: create, preview, stage, and activate candidates.""" 

2 

3from __future__ import annotations 

4 

5import asyncio 

6import json 

7import os 

8import shutil 

9import tempfile 

10import threading 

11import uuid 

12from pathlib import Path, PurePosixPath 

13 

14from copytrading_engine.backup.archive import inspect_backup_archive 

15from copytrading_engine.backup.configuration import ( 

16 archive_credential_references, 

17 candidate_credential_references, 

18 read_installation_id, 

19) 

20from copytrading_engine.backup.files import ( 

21 copy_private_file, 

22 fsync_directory, 

23 fsync_tree, 

24 require_directory, 

25) 

26from copytrading_engine.backup.manifest import ( 

27 ACCOUNT_DATABASE, 

28 RESTORE_STAGE, 

29 BackupManifest, 

30 BackupManifestError, 

31 PendingRestoreCandidate, 

32 RestorePreview, 

33 json_object, 

34 manifest_digest, 

35) 

36from copytrading_engine.backup.ports import ( 

37 AccountSnapshotSchemaProvider, 

38 BackupFence, 

39 RestoredAccountPreparer, 

40 SchemaCatalogProvider, 

41) 

42from copytrading_engine.backup.restore.candidates import ( 

43 candidate_intent_payload, 

44 clear_restore_staging, 

45 persist_candidate_intent, 

46 read_active_generation, 

47 read_candidate_intent, 

48 reap_interrupted_restore_candidates, 

49 remove_candidate_intent, 

50 remove_generation_entry, 

51 stage_restore, 

52 validate_candidate_generation, 

53 validate_published_restore_candidate, 

54 verify_staged_tree, 

55 write_candidate_manifest, 

56) 

57from copytrading_engine.backup.restore.gate import ( 

58 clear_restore_manual_disabled, 

59 persist_restore_manual_disabled, 

60 restore_manual_disabled, 

61 restore_manual_disabled_path, 

62) 

63from copytrading_engine.backup.snapshot import create_backup 

64 

65 

66class BackupRestoreService: 

67 """Coordinate a consistent snapshot and isolate an explicitly selected restore.""" 

68 

69 def __init__( 

70 self, 

71 data_dir: Path, 

72 fence: BackupFence, 

73 *, 

74 schema_catalog: SchemaCatalogProvider, 

75 snapshot_schema: AccountSnapshotSchemaProvider, 

76 owner_support_directory: Path, 

77 account_preparer: RestoredAccountPreparer | None = None, 

78 ) -> None: 

79 self._data_dir = data_dir 

80 self._fence = fence 

81 self._schema_catalog = schema_catalog 

82 self._snapshot_schema = snapshot_schema 

83 self._owner_support_directory = owner_support_directory 

84 self._account_preparer = account_preparer 

85 self._staging_lock = threading.RLock() 

86 self._staged_manifests: dict[str, BackupManifest] = {} 

87 self._prepared_candidates: dict[str, str] = {} 

88 

89 @property 

90 def staging_root(self) -> Path: 

91 root = self._owner_support_directory / ".restore-staging" 

92 try: 

93 root.mkdir(mode=0o700) 

94 except FileExistsError: 

95 pass 

96 require_directory(root, "restore staging directory is unavailable") 

97 os.chmod(root, 0o700) 

98 return root 

99 

100 async def create_backup(self, destination: Path) -> BackupManifest: 

101 async with self._fence(): 

102 worker = asyncio.create_task( 

103 asyncio.to_thread( 

104 create_backup, 

105 self._data_dir, 

106 self._owner_support_directory, 

107 destination, 

108 self._schema_catalog, 

109 self._snapshot_schema, 

110 ) 

111 ) 

112 cancelled = False 

113 while True: 

114 try: 

115 result = await asyncio.shield(worker) 

116 break 

117 except asyncio.CancelledError: 

118 cancelled = True 

119 if worker.done(): 119 ↛ 120line 119 didn't jump to line 120 because the condition on line 119 was never true

120 break 

121 if cancelled: 

122 worker.result() 

123 raise asyncio.CancelledError 

124 return result 

125 

126 def preview_restore(self, archive_path: Path) -> RestorePreview: 

127 manifest = inspect_backup_archive( 

128 archive_path, 

129 self._schema_catalog, 

130 self._snapshot_schema, 

131 ) 

132 current_identity = read_installation_id(self._owner_support_directory) 

133 account_ids = tuple( 

134 sorted( 

135 member.path.split("/")[1] 

136 for member in manifest.members 

137 if ACCOUNT_DATABASE.fullmatch(member.path) 

138 ) 

139 ) 

140 credential_references = archive_credential_references(archive_path, manifest) 

141 return RestorePreview( 

142 manifest=manifest, 

143 matches_installation=manifest.installation_id == current_identity, 

144 account_ids=account_ids, 

145 credential_references=credential_references, 

146 ) 

147 

148 def stage_restore(self, archive_path: Path, staging_root: Path) -> tuple[Path, RestorePreview]: 

149 with self._staging_lock: 

150 service_root = self.staging_root 

151 try: 

152 supplied_root = staging_root.resolve(strict=True) 

153 expected_root = service_root.resolve(strict=True) 

154 except OSError as exc: 

155 raise BackupManifestError("restore staging directory is unavailable") from exc 

156 require_directory(staging_root, "restore staging directory is unsafe") 

157 if supplied_root != expected_root: 

158 raise BackupManifestError("restore staging must use the service-owned directory") 

159 clear_restore_staging(service_root) 

160 self._staged_manifests.clear() 

161 self._prepared_candidates.clear() 

162 preview = self.preview_restore(archive_path) 

163 staged = stage_restore( 

164 archive_path, 

165 service_root, 

166 preview.manifest, 

167 self._schema_catalog, 

168 self._snapshot_schema, 

169 ) 

170 self._staged_manifests[staged.name] = preview.manifest 

171 return staged, preview 

172 

173 def prepare_restore_candidate(self, staging_identifier: str) -> str: 

174 """Copy one validated preview into an inactive generation and gate startup.""" 

175 with self._staging_lock: 

176 if not isinstance(staging_identifier, str) or not RESTORE_STAGE.fullmatch( 

177 staging_identifier 

178 ): 

179 raise BackupManifestError("restore staging identifier is invalid") 

180 prior_candidate = self._prepared_candidates.get(staging_identifier) 

181 if prior_candidate is not None: 

182 validate_published_restore_candidate( 

183 self._owner_support_directory, 

184 prior_candidate, 

185 self._schema_catalog, 

186 self._snapshot_schema, 

187 ) 

188 return prior_candidate 

189 manifest = self._staged_manifests.get(staging_identifier) 

190 if manifest is None: 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true

191 raise BackupManifestError("restore staging identifier is unknown") 

192 

193 stage_root = self.staging_root 

194 staged = stage_root / staging_identifier 

195 require_directory(staged, "restore staging directory is unavailable") 

196 owner = self._owner_support_directory 

197 require_directory(owner, "restore owner directory is unavailable") 

198 if read_installation_id(owner) != manifest.installation_id: 198 ↛ 199line 198 didn't jump to line 199 because the condition on line 198 was never true

199 raise BackupManifestError("backup belongs to a different installation") 

200 generations = owner / "generations" 

201 require_directory(generations, "operational generations directory is unavailable") 

202 active_generation = read_active_generation(owner) 

203 reap_interrupted_restore_candidates(owner, generations, active_generation) 

204 if restore_manual_disabled(restore_manual_disabled_path(owner)): 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true

205 raise BackupManifestError("another restore activation is already in progress") 

206 

207 verify_staged_tree( 

208 staged, 

209 manifest, 

210 self._schema_catalog, 

211 self._snapshot_schema, 

212 ) 

213 candidate_id = str(uuid.uuid4()) 

214 candidate = generations / candidate_id 

215 if candidate.exists() or candidate.is_symlink(): 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true

216 raise BackupManifestError("restore candidate identifier already exists") 

217 temporary = Path(tempfile.mkdtemp(prefix=".restore-candidate-", dir=generations)) 

218 os.chmod(temporary, 0o700) 

219 published = False 

220 try: 

221 for member in manifest.members: 

222 source = staged.joinpath(*PurePosixPath(member.path).parts) 

223 target = temporary.joinpath(*PurePosixPath(member.path).parts) 

224 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) 

225 digest, size = copy_private_file(source, target, member.size) 

226 if digest != member.sha256 or size != member.size: 226 ↛ 227line 226 didn't jump to line 227 because the condition on line 226 was never true

227 raise BackupManifestError("restore member changed after validation") 

228 

229 account_members = tuple( 

230 member for member in manifest.members if ACCOUNT_DATABASE.fullmatch(member.path) 

231 ) 

232 if account_members and self._account_preparer is None: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 raise BackupManifestError( 

234 "restored account manual-state preparation is unavailable" 

235 ) 

236 for member in account_members: 

237 account_path = temporary.joinpath(*PurePosixPath(member.path).parts) 

238 assert self._account_preparer is not None 

239 try: 

240 self._account_preparer(account_path) 

241 except Exception as exc: 

242 raise BackupManifestError( 

243 "restored account could not be made manual-disabled" 

244 ) from exc 

245 

246 candidate_manifest_sha256 = write_candidate_manifest(temporary, manifest) 

247 validate_candidate_generation( 

248 temporary, 

249 manifest, 

250 self._schema_catalog, 

251 self._snapshot_schema, 

252 ) 

253 intent_payload = candidate_intent_payload( 

254 manifest, 

255 active_generation, 

256 candidate_id, 

257 candidate_manifest_sha256, 

258 ) 

259 persist_candidate_intent(owner, intent_payload) 

260 fsync_tree(temporary) 

261 os.rename(temporary, candidate) 

262 published = True 

263 fsync_directory(generations) 

264 

265 marker_payload = json.dumps( 

266 { 

267 "version": 2, 

268 "installation_id": manifest.installation_id, 

269 "previous_generation": active_generation, 

270 "candidate_generation": candidate_id, 

271 "backup_created_at": manifest.created_at.isoformat(), 

272 "manifest_sha256": manifest_digest(manifest), 

273 "candidate_manifest_sha256": candidate_manifest_sha256, 

274 }, 

275 sort_keys=True, 

276 separators=(",", ":"), 

277 ).encode("utf-8") 

278 persist_restore_manual_disabled(restore_manual_disabled_path(owner), marker_payload) 

279 self._prepared_candidates[staging_identifier] = candidate_id 

280 return candidate_id 

281 except BaseException: 

282 if published: 

283 if not restore_manual_disabled(restore_manual_disabled_path(owner)): 

284 remove_generation_entry(candidate) 

285 fsync_directory(generations) 

286 remove_candidate_intent(owner, candidate_id) 

287 else: 

288 shutil.rmtree(temporary, ignore_errors=True) 

289 remove_candidate_intent(owner, candidate_id) 

290 raise 

291 

292 def validate_restore_candidate(self, candidate_identifier: str) -> BackupManifest: 

293 """Reload and validate a prepared candidate using only durable evidence.""" 

294 with self._staging_lock: 

295 return validate_published_restore_candidate( 

296 self._owner_support_directory, 

297 candidate_identifier, 

298 self._schema_catalog, 

299 self._snapshot_schema, 

300 ) 

301 

302 def pending_restore_candidate(self) -> PendingRestoreCandidate | None: 

303 """Read the durable restore intent and gate after an uncertain IPC result or restart.""" 

304 with self._staging_lock: 

305 owner = self._owner_support_directory 

306 marker_path = restore_manual_disabled_path(owner) 

307 if not restore_manual_disabled(marker_path): 

308 return None 

309 marker = json_object(marker_path.read_bytes()) 

310 candidate_id = marker.get("candidate_generation") 

311 intent = read_candidate_intent(owner) 

312 if intent is None or not isinstance(candidate_id, str): 312 ↛ 313line 312 didn't jump to line 313 because the condition on line 312 was never true

313 raise BackupManifestError("restore candidate status is unavailable") 

314 if ( 314 ↛ 327line 314 didn't jump to line 327 because the condition on line 314 was never true

315 set(marker) 

316 != { 

317 "version", 

318 "installation_id", 

319 "previous_generation", 

320 "candidate_generation", 

321 "backup_created_at", 

322 "manifest_sha256", 

323 "candidate_manifest_sha256", 

324 } 

325 or marker.get("version") != 2 

326 ): 

327 raise BackupManifestError("restore gate is invalid") 

328 active_generation = read_active_generation(owner) 

329 previous_generation = intent["previous_generation"] 

330 expected = { 

331 "installation_id": intent["installation_id"], 

332 "previous_generation": previous_generation, 

333 "candidate_generation": candidate_id, 

334 "manifest_sha256": intent["manifest_sha256"], 

335 "candidate_manifest_sha256": intent["candidate_manifest_sha256"], 

336 } 

337 if any(marker.get(key) != value for key, value in expected.items()): 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true

338 raise BackupManifestError("restore gate does not match candidate intent") 

339 if active_generation not in {previous_generation, candidate_id}: 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true

340 raise BackupManifestError("active generation does not match restore candidate") 

341 

342 environment_ids: tuple[str, ...] = () 

343 account_ids: tuple[str, ...] = () 

344 credential_references: tuple[str, ...] = () 

345 candidate_valid = False 

346 try: 

347 manifest = validate_published_restore_candidate( 

348 owner, 

349 candidate_id, 

350 self._schema_catalog, 

351 self._snapshot_schema, 

352 ) 

353 except BackupManifestError: 

354 pass 

355 else: 

356 candidate_valid = True 

357 environment_ids = manifest.environment_ids 

358 account_ids = tuple( 

359 sorted( 

360 member.path.split("/")[1] 

361 for member in manifest.members 

362 if ACCOUNT_DATABASE.fullmatch(member.path) 

363 ) 

364 ) 

365 credential_references = candidate_credential_references( 

366 owner / "generations" / candidate_id, manifest 

367 ) 

368 return PendingRestoreCandidate( 

369 candidate_id=candidate_id, 

370 previous_generation=str(previous_generation), 

371 active_generation=active_generation, 

372 installation_id=str(intent["installation_id"]), 

373 environment_ids=environment_ids, 

374 account_ids=account_ids, 

375 credential_references=credential_references, 

376 candidate_valid=candidate_valid, 

377 ) 

378 

379 def abort_restore_candidate(self, candidate_identifier: str) -> None: 

380 """Abort only after the previous generation is selected and candidate absence is durable.""" 

381 with self._staging_lock: 

382 status = self.pending_restore_candidate() 

383 if status is None or status.candidate_id != candidate_identifier: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true

384 raise BackupManifestError("restore candidate is unavailable for rollback") 

385 if status.active_generation != status.previous_generation: 

386 raise BackupManifestError("restore candidate must be switched back before rollback") 

387 owner = self._owner_support_directory 

388 candidate = owner / "generations" / candidate_identifier 

389 if candidate.exists() or candidate.is_symlink(): 389 ↛ 393line 389 didn't jump to line 393 because the condition on line 389 was always true

390 if not remove_generation_entry(candidate): 390 ↛ 391line 390 didn't jump to line 391 because the condition on line 390 was never true

391 raise BackupManifestError("restore candidate could not be removed") 

392 fsync_directory(owner / "generations") 

393 clear_restore_manual_disabled(restore_manual_disabled_path(owner)) 

394 remove_candidate_intent(owner, candidate_identifier) 

395 self._prepared_candidates = { 

396 staging_id: candidate_id 

397 for staging_id, candidate_id in self._prepared_candidates.items() 

398 if candidate_id != candidate_identifier 

399 } 

400 

401 def resolve_active_restore_candidate( 

402 self, candidate_identifier: str 

403 ) -> tuple[BackupManifest, Path]: 

404 """Validate and resolve a candidate only after its generation is selected.""" 

405 with self._staging_lock: 

406 manifest = validate_published_restore_candidate( 

407 self._owner_support_directory, 

408 candidate_identifier, 

409 self._schema_catalog, 

410 self._snapshot_schema, 

411 ) 

412 if read_active_generation(self._owner_support_directory) != candidate_identifier: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true

413 raise BackupManifestError("restore candidate generation is not active") 

414 candidate = self._owner_support_directory / "generations" / candidate_identifier 

415 require_directory(candidate, "restore candidate generation is unavailable") 

416 return manifest, candidate 

417 

418 def complete_restore_candidate(self, candidate_identifier: str) -> None: 

419 """Clear the durable startup gate after a successful read-only preflight.""" 

420 with self._staging_lock: 

421 owner = self._owner_support_directory 

422 manifest = validate_published_restore_candidate( 

423 owner, 

424 candidate_identifier, 

425 self._schema_catalog, 

426 self._snapshot_schema, 

427 ) 

428 if read_active_generation(owner) != candidate_identifier: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true

429 raise BackupManifestError("restore candidate generation is not active") 

430 intent = read_candidate_intent(owner) 

431 if ( 431 ↛ 437line 431 didn't jump to line 437 because the condition on line 431 was never true

432 intent is None 

433 or intent["installation_id"] != manifest.installation_id 

434 or intent["candidate_generation"] != candidate_identifier 

435 or intent["manifest_sha256"] != manifest_digest(manifest) 

436 ): 

437 raise BackupManifestError("restore candidate intent does not match activation") 

438 clear_restore_manual_disabled(restore_manual_disabled_path(owner))