Coverage for src/copytrading_engine/backup/snapshot.py: 86%
141 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-16 16:55 +0000
1"""Writing a backup archive from a fenced, consistent snapshot of the installation."""
3from __future__ import annotations
5import hashlib
6import json
7import os
8import re
9import shutil
10import sqlite3
11import tempfile
12import zipfile
13from datetime import UTC, datetime
14from pathlib import Path, PurePosixPath
15from zipfile import ZIP_DEFLATED
17from copytrading_engine.backup.archive import inspect_backup_archive
18from copytrading_engine.backup.configuration import (
19 read_configuration_identities,
20 read_installation_id,
21 validate_activation_json,
22)
23from copytrading_engine.backup.databases import (
24 account_environment,
25 snapshot_database,
26 sqlite_schema_version,
27 validate_sqlite,
28)
29from copytrading_engine.backup.files import (
30 copy_private_bytes,
31 copy_private_file,
32 require_directory,
33 require_regular_file,
34)
35from copytrading_engine.backup.manifest import (
36 FORMAT_VERSION,
37 MANIFEST_NAME,
38 MAX_MEMBER_BYTES,
39 BackupManifest,
40 BackupManifestError,
41 BackupMember,
42 is_sqlite_member,
43 manifest_dict,
44)
45from copytrading_engine.backup.ports import AccountSnapshotSchemaProvider, SchemaCatalogProvider
48def create_backup(
49 data_dir: Path,
50 owner_support_directory: Path,
51 destination: Path,
52 schema_catalog: SchemaCatalogProvider,
53 snapshot_schema: AccountSnapshotSchemaProvider,
54) -> BackupManifest:
55 installation_id = read_installation_id(owner_support_directory)
56 parent = destination.parent.resolve(strict=True)
57 if destination.name in {"", ".", ".."}: 57 ↛ 58line 57 didn't jump to line 58 because the condition on line 57 was never true
58 raise BackupManifestError("backup destination is invalid")
59 destination = parent / destination.name
60 try:
61 destination.lstat()
62 except FileNotFoundError:
63 pass
64 else:
65 raise BackupManifestError("backup destination already exists")
67 work: Path | None = None
68 try:
69 work = Path(tempfile.mkdtemp(prefix=".copytrading-backup-work-", dir=parent))
70 os.chmod(work, 0o700)
71 staging = work / "files"
72 staging.mkdir(mode=0o700)
73 # Mark the start of the fenced multi-store cut. Restore reconciliation
74 # checks broker activity from this point, including writes during copying.
75 snapshot_started_at = datetime.now(UTC)
76 members, environment_ids = _snapshot_members(
77 data_dir,
78 staging,
79 installation_id,
80 schema_catalog,
81 snapshot_schema,
82 )
83 manifest = BackupManifest(
84 format_version=FORMAT_VERSION,
85 created_at=snapshot_started_at,
86 installation_id=installation_id,
87 environment_ids=environment_ids,
88 members=members,
89 )
90 archive_temp = work / "backup.zip"
91 with zipfile.ZipFile(
92 archive_temp, "w", compression=ZIP_DEFLATED, compresslevel=6
93 ) as archive:
94 for member in members:
95 archive.write(staging.joinpath(*PurePosixPath(member.path).parts), member.path)
96 # The manifest is appended only after every payload hash and schema was captured.
97 archive.writestr(
98 MANIFEST_NAME,
99 json.dumps(manifest_dict(manifest), sort_keys=True, separators=(",", ":")),
100 )
101 os.chmod(archive_temp, 0o600)
102 with archive_temp.open("rb") as stream:
103 os.fsync(stream.fileno())
104 verified = inspect_backup_archive(archive_temp, schema_catalog, snapshot_schema)
105 if verified != manifest: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true
106 raise BackupManifestError("backup manifest changed during publication")
107 # A same-volume hard link publishes atomically and fails if the chosen name exists.
108 os.link(archive_temp, destination, follow_symlinks=False)
109 archive_temp.unlink()
110 directory_fd = os.open(parent, os.O_RDONLY)
111 try:
112 os.fsync(directory_fd)
113 finally:
114 os.close(directory_fd)
115 return manifest
116 except BackupManifestError:
117 raise
118 except (OSError, sqlite3.Error, ValueError, zipfile.BadZipFile) as exc:
119 raise BackupManifestError("backup could not be completed") from exc
120 finally:
121 if work is not None:
122 shutil.rmtree(work, ignore_errors=True)
125def _snapshot_members(
126 data_dir: Path,
127 staging: Path,
128 installation_id: str,
129 schema_catalog: SchemaCatalogProvider,
130 snapshot_schema: AccountSnapshotSchemaProvider,
131) -> tuple[tuple[BackupMember, ...], tuple[str, ...]]:
132 members: list[BackupMember] = []
133 environment_ids: set[str] = set()
134 configured_environments: dict[str, str] = {}
135 database = data_dir / "application.db"
136 require_regular_file(database, "operational database is unavailable")
137 snapshot_database(database, staging / "application.db")
138 validate_sqlite(
139 staging / "application.db",
140 "application.db",
141 installation_id,
142 schema_catalog,
143 snapshot_schema,
144 )
145 members.append(_member_for(staging, "application.db", schema_catalog, snapshot_schema))
147 for name in ("trading-configuration.json", "trading-activation.json"):
148 source = data_dir / name
149 if source.exists() or source.is_symlink():
150 require_regular_file(source, "operational configuration is invalid")
151 content = source.read_bytes()
152 if len(content) > MAX_MEMBER_BYTES: 152 ↛ 153line 152 didn't jump to line 153 because the condition on line 152 was never true
153 raise BackupManifestError("operational configuration exceeds the supported size")
154 copy_private_bytes(staging / name, content)
155 if name == "trading-configuration.json": 155 ↛ 162line 155 didn't jump to line 162 because the condition on line 155 was always true
156 identities = read_configuration_identities(content)
157 environment_ids.update(identities)
158 configured_environments = {
159 identity.split(":", 1)[1]: identity.split(":", 1)[0] for identity in identities
160 }
161 else:
162 validate_activation_json(content)
163 members.append(_member_for(staging, name, schema_catalog, snapshot_schema))
165 accounts_root = data_dir / "accounts"
166 if accounts_root.exists() or accounts_root.is_symlink():
167 require_directory(accounts_root, "account evidence path is invalid")
168 for account_dir in sorted(accounts_root.iterdir()):
169 if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", account_dir.name): 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 raise BackupManifestError("account evidence identity is invalid")
171 require_directory(account_dir, "account evidence path is invalid")
172 source = account_dir / "execution.sqlite3"
173 if not source.exists() and not source.is_symlink(): 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 raise BackupManifestError("account evidence database is missing")
175 require_regular_file(source, "account evidence database is invalid")
176 member_path = f"accounts/{account_dir.name}/execution.sqlite3"
177 target = staging.joinpath(*PurePosixPath(member_path).parts)
178 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
179 snapshot_database(source, target)
180 validate_sqlite(
181 target,
182 member_path,
183 installation_id,
184 schema_catalog,
185 snapshot_schema,
186 )
187 account_id, environment = account_dir.name, account_environment(target)
188 if configured_environments.get(account_id, environment) != environment: 188 ↛ 189line 188 didn't jump to line 189 because the condition on line 188 was never true
189 raise BackupManifestError(
190 "account database environment conflicts with configuration"
191 )
192 environment_ids.add(f"{environment}:{account_id}")
193 members.append(_member_for(staging, member_path, schema_catalog, snapshot_schema))
195 attachments_root = data_dir / "attachments"
196 if attachments_root.exists() or attachments_root.is_symlink():
197 require_directory(attachments_root, "source attachments path is invalid")
198 for prefix_dir in sorted(attachments_root.iterdir()):
199 if not re.fullmatch(r"[0-9a-f]{2}", prefix_dir.name): 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true
200 raise BackupManifestError("source attachment path is invalid")
201 require_directory(prefix_dir, "source attachment path is invalid")
202 for source in sorted(prefix_dir.iterdir()):
203 if ( 203 ↛ 207line 203 didn't jump to line 207 because the condition on line 203 was never true
204 not re.fullmatch(r"[0-9a-f]{64}", source.name)
205 or source.name[:2] != prefix_dir.name
206 ):
207 raise BackupManifestError("source attachment path is invalid")
208 require_regular_file(source, "source attachment is invalid")
209 member_path = f"attachments/{prefix_dir.name}/{source.name}"
210 target = staging.joinpath(*PurePosixPath(member_path).parts)
211 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
212 digest, _ = copy_private_file(source, target, MAX_MEMBER_BYTES)
213 if digest != source.name: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true
214 raise BackupManifestError("source attachment content hash is invalid")
215 members.append(_member_for(staging, member_path, schema_catalog, snapshot_schema))
216 return tuple(sorted(members, key=lambda member: member.path)), tuple(sorted(environment_ids))
219def _member_for(
220 staging: Path,
221 member_path: str,
222 schema_catalog: SchemaCatalogProvider,
223 snapshot_schema: AccountSnapshotSchemaProvider,
224) -> BackupMember:
225 path = staging.joinpath(*PurePosixPath(member_path).parts)
226 digest = hashlib.sha256()
227 size = 0
228 with path.open("rb") as stream:
229 while chunk := stream.read(1024 * 1024):
230 size += len(chunk)
231 if size > MAX_MEMBER_BYTES: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true
232 raise BackupManifestError("backup member exceeds the supported size")
233 digest.update(chunk)
234 if is_sqlite_member(member_path):
235 schema_version = sqlite_schema_version(path, member_path, schema_catalog, snapshot_schema)
236 elif member_path.endswith(".json"):
237 schema_version = "json:1"
238 else:
239 schema_version = "blob:1"
240 return BackupMember(member_path, size, digest.hexdigest(), schema_version)