Coverage for src/copytrading_engine/backup/snapshot.py: 86%

141 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Writing a backup archive from a fenced, consistent snapshot of the installation.""" 

2 

3from __future__ import annotations 

4 

5import hashlib 

6import json 

7import os 

8import re 

9import shutil 

10import sqlite3 

11import tempfile 

12import zipfile 

13from datetime import UTC, datetime 

14from pathlib import Path, PurePosixPath 

15from zipfile import ZIP_DEFLATED 

16 

17from copytrading_engine.backup.archive import inspect_backup_archive 

18from copytrading_engine.backup.configuration import ( 

19 read_configuration_identities, 

20 read_installation_id, 

21 validate_activation_json, 

22) 

23from copytrading_engine.backup.databases import ( 

24 account_environment, 

25 snapshot_database, 

26 sqlite_schema_version, 

27 validate_sqlite, 

28) 

29from copytrading_engine.backup.files import ( 

30 copy_private_bytes, 

31 copy_private_file, 

32 require_directory, 

33 require_regular_file, 

34) 

35from copytrading_engine.backup.manifest import ( 

36 FORMAT_VERSION, 

37 MANIFEST_NAME, 

38 MAX_MEMBER_BYTES, 

39 BackupManifest, 

40 BackupManifestError, 

41 BackupMember, 

42 is_sqlite_member, 

43 manifest_dict, 

44) 

45from copytrading_engine.backup.ports import AccountSnapshotSchemaProvider, SchemaCatalogProvider 

46 

47 

48def create_backup( 

49 data_dir: Path, 

50 owner_support_directory: Path, 

51 destination: Path, 

52 schema_catalog: SchemaCatalogProvider, 

53 snapshot_schema: AccountSnapshotSchemaProvider, 

54) -> BackupManifest: 

55 installation_id = read_installation_id(owner_support_directory) 

56 parent = destination.parent.resolve(strict=True) 

57 if destination.name in {"", ".", ".."}: 57 ↛ 58line 57 didn't jump to line 58 because the condition on line 57 was never true

58 raise BackupManifestError("backup destination is invalid") 

59 destination = parent / destination.name 

60 try: 

61 destination.lstat() 

62 except FileNotFoundError: 

63 pass 

64 else: 

65 raise BackupManifestError("backup destination already exists") 

66 

67 work: Path | None = None 

68 try: 

69 work = Path(tempfile.mkdtemp(prefix=".copytrading-backup-work-", dir=parent)) 

70 os.chmod(work, 0o700) 

71 staging = work / "files" 

72 staging.mkdir(mode=0o700) 

73 # Mark the start of the fenced multi-store cut. Restore reconciliation 

74 # checks broker activity from this point, including writes during copying. 

75 snapshot_started_at = datetime.now(UTC) 

76 members, environment_ids = _snapshot_members( 

77 data_dir, 

78 staging, 

79 installation_id, 

80 schema_catalog, 

81 snapshot_schema, 

82 ) 

83 manifest = BackupManifest( 

84 format_version=FORMAT_VERSION, 

85 created_at=snapshot_started_at, 

86 installation_id=installation_id, 

87 environment_ids=environment_ids, 

88 members=members, 

89 ) 

90 archive_temp = work / "backup.zip" 

91 with zipfile.ZipFile( 

92 archive_temp, "w", compression=ZIP_DEFLATED, compresslevel=6 

93 ) as archive: 

94 for member in members: 

95 archive.write(staging.joinpath(*PurePosixPath(member.path).parts), member.path) 

96 # The manifest is appended only after every payload hash and schema was captured. 

97 archive.writestr( 

98 MANIFEST_NAME, 

99 json.dumps(manifest_dict(manifest), sort_keys=True, separators=(",", ":")), 

100 ) 

101 os.chmod(archive_temp, 0o600) 

102 with archive_temp.open("rb") as stream: 

103 os.fsync(stream.fileno()) 

104 verified = inspect_backup_archive(archive_temp, schema_catalog, snapshot_schema) 

105 if verified != manifest: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 raise BackupManifestError("backup manifest changed during publication") 

107 # A same-volume hard link publishes atomically and fails if the chosen name exists. 

108 os.link(archive_temp, destination, follow_symlinks=False) 

109 archive_temp.unlink() 

110 directory_fd = os.open(parent, os.O_RDONLY) 

111 try: 

112 os.fsync(directory_fd) 

113 finally: 

114 os.close(directory_fd) 

115 return manifest 

116 except BackupManifestError: 

117 raise 

118 except (OSError, sqlite3.Error, ValueError, zipfile.BadZipFile) as exc: 

119 raise BackupManifestError("backup could not be completed") from exc 

120 finally: 

121 if work is not None: 

122 shutil.rmtree(work, ignore_errors=True) 

123 

124 

125def _snapshot_members( 

126 data_dir: Path, 

127 staging: Path, 

128 installation_id: str, 

129 schema_catalog: SchemaCatalogProvider, 

130 snapshot_schema: AccountSnapshotSchemaProvider, 

131) -> tuple[tuple[BackupMember, ...], tuple[str, ...]]: 

132 members: list[BackupMember] = [] 

133 environment_ids: set[str] = set() 

134 configured_environments: dict[str, str] = {} 

135 database = data_dir / "application.db" 

136 require_regular_file(database, "operational database is unavailable") 

137 snapshot_database(database, staging / "application.db") 

138 validate_sqlite( 

139 staging / "application.db", 

140 "application.db", 

141 installation_id, 

142 schema_catalog, 

143 snapshot_schema, 

144 ) 

145 members.append(_member_for(staging, "application.db", schema_catalog, snapshot_schema)) 

146 

147 for name in ("trading-configuration.json", "trading-activation.json"): 

148 source = data_dir / name 

149 if source.exists() or source.is_symlink(): 

150 require_regular_file(source, "operational configuration is invalid") 

151 content = source.read_bytes() 

152 if len(content) > MAX_MEMBER_BYTES: 152 ↛ 153line 152 didn't jump to line 153 because the condition on line 152 was never true

153 raise BackupManifestError("operational configuration exceeds the supported size") 

154 copy_private_bytes(staging / name, content) 

155 if name == "trading-configuration.json": 155 ↛ 162line 155 didn't jump to line 162 because the condition on line 155 was always true

156 identities = read_configuration_identities(content) 

157 environment_ids.update(identities) 

158 configured_environments = { 

159 identity.split(":", 1)[1]: identity.split(":", 1)[0] for identity in identities 

160 } 

161 else: 

162 validate_activation_json(content) 

163 members.append(_member_for(staging, name, schema_catalog, snapshot_schema)) 

164 

165 accounts_root = data_dir / "accounts" 

166 if accounts_root.exists() or accounts_root.is_symlink(): 

167 require_directory(accounts_root, "account evidence path is invalid") 

168 for account_dir in sorted(accounts_root.iterdir()): 

169 if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", account_dir.name): 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 raise BackupManifestError("account evidence identity is invalid") 

171 require_directory(account_dir, "account evidence path is invalid") 

172 source = account_dir / "execution.sqlite3" 

173 if not source.exists() and not source.is_symlink(): 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true

174 raise BackupManifestError("account evidence database is missing") 

175 require_regular_file(source, "account evidence database is invalid") 

176 member_path = f"accounts/{account_dir.name}/execution.sqlite3" 

177 target = staging.joinpath(*PurePosixPath(member_path).parts) 

178 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) 

179 snapshot_database(source, target) 

180 validate_sqlite( 

181 target, 

182 member_path, 

183 installation_id, 

184 schema_catalog, 

185 snapshot_schema, 

186 ) 

187 account_id, environment = account_dir.name, account_environment(target) 

188 if configured_environments.get(account_id, environment) != environment: 188 ↛ 189line 188 didn't jump to line 189 because the condition on line 188 was never true

189 raise BackupManifestError( 

190 "account database environment conflicts with configuration" 

191 ) 

192 environment_ids.add(f"{environment}:{account_id}") 

193 members.append(_member_for(staging, member_path, schema_catalog, snapshot_schema)) 

194 

195 attachments_root = data_dir / "attachments" 

196 if attachments_root.exists() or attachments_root.is_symlink(): 

197 require_directory(attachments_root, "source attachments path is invalid") 

198 for prefix_dir in sorted(attachments_root.iterdir()): 

199 if not re.fullmatch(r"[0-9a-f]{2}", prefix_dir.name): 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true

200 raise BackupManifestError("source attachment path is invalid") 

201 require_directory(prefix_dir, "source attachment path is invalid") 

202 for source in sorted(prefix_dir.iterdir()): 

203 if ( 203 ↛ 207line 203 didn't jump to line 207 because the condition on line 203 was never true

204 not re.fullmatch(r"[0-9a-f]{64}", source.name) 

205 or source.name[:2] != prefix_dir.name 

206 ): 

207 raise BackupManifestError("source attachment path is invalid") 

208 require_regular_file(source, "source attachment is invalid") 

209 member_path = f"attachments/{prefix_dir.name}/{source.name}" 

210 target = staging.joinpath(*PurePosixPath(member_path).parts) 

211 target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) 

212 digest, _ = copy_private_file(source, target, MAX_MEMBER_BYTES) 

213 if digest != source.name: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true

214 raise BackupManifestError("source attachment content hash is invalid") 

215 members.append(_member_for(staging, member_path, schema_catalog, snapshot_schema)) 

216 return tuple(sorted(members, key=lambda member: member.path)), tuple(sorted(environment_ids)) 

217 

218 

219def _member_for( 

220 staging: Path, 

221 member_path: str, 

222 schema_catalog: SchemaCatalogProvider, 

223 snapshot_schema: AccountSnapshotSchemaProvider, 

224) -> BackupMember: 

225 path = staging.joinpath(*PurePosixPath(member_path).parts) 

226 digest = hashlib.sha256() 

227 size = 0 

228 with path.open("rb") as stream: 

229 while chunk := stream.read(1024 * 1024): 

230 size += len(chunk) 

231 if size > MAX_MEMBER_BYTES: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true

232 raise BackupManifestError("backup member exceeds the supported size") 

233 digest.update(chunk) 

234 if is_sqlite_member(member_path): 

235 schema_version = sqlite_schema_version(path, member_path, schema_catalog, snapshot_schema) 

236 elif member_path.endswith(".json"): 

237 schema_version = "json:1" 

238 else: 

239 schema_version = "blob:1" 

240 return BackupMember(member_path, size, digest.hexdigest(), schema_version)