Coverage for src/copytrading_engine/backup/archive.py: 74%

211 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Reading a backup archive: bounded extraction, member hashing, and manifest validation.""" 

2 

3from __future__ import annotations 

4 

5import hashlib 

6import json 

7import os 

8import re 

9import stat 

10import tempfile 

11import uuid 

12import zipfile 

13from collections.abc import Generator 

14from contextlib import contextmanager 

15from datetime import UTC, datetime 

16from pathlib import Path, PurePosixPath 

17from typing import Any 

18 

19from copytrading_engine.backup.databases import validate_sqlite 

20from copytrading_engine.backup.manifest import ( 

21 ACCOUNT_DATABASE, 

22 APPLICATION_SCHEMA_PREFIX, 

23 ARCHIVE_FORMAT, 

24 DIGEST, 

25 FORMAT_VERSION, 

26 MANIFEST_NAME, 

27 MAX_MEMBER_BYTES, 

28 BackupManifest, 

29 BackupManifestError, 

30 BackupMember, 

31 account_schema_version, 

32 is_sqlite_member, 

33) 

34from copytrading_engine.backup.ports import AccountSnapshotSchemaProvider, SchemaCatalogProvider 

35 

36_MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 * 1024 

37 

38 

39_MAX_TOTAL_MEMBER_BYTES = 8 * 1024 * 1024 * 1024 

40 

41 

42_MAX_MEMBER_COUNT = 50_000 

43 

44 

45_MAX_MANIFEST_BYTES = 1024 * 1024 

46 

47 

48_ATTACHMENT = re.compile(r"^attachments/[0-9a-f]{2}/[0-9a-f]{64}$") 

49 

50 

51_TOP_LEVEL_MEMBERS = { 

52 "application.db", 

53 "trading-configuration.json", 

54 "trading-activation.json", 

55} 

56 

57 

58def inspect_backup_archive( 

59 archive_path: Path, 

60 schema_catalog: SchemaCatalogProvider, 

61 snapshot_schema: AccountSnapshotSchemaProvider, 

62) -> BackupManifest: 

63 """Read and verify every archive member before returning its manifest.""" 

64 try: 

65 with open_archive(archive_path) as archive: 

66 entries = archive.infolist() 

67 if len(entries) > _MAX_MEMBER_COUNT + 1: 67 ↛ 68line 67 didn't jump to line 68 because the condition on line 67 was never true

68 raise BackupManifestError("backup contains too many members") 

69 names: dict[str, zipfile.ZipInfo] = {} 

70 for entry in entries: 

71 _validate_member_name(entry.filename) 

72 key = entry.filename.casefold() 

73 if key in names: 

74 raise BackupManifestError("backup contains a duplicate member") 

75 names[key] = entry 

76 mode = entry.external_attr >> 16 

77 file_kind = stat.S_IFMT(mode) 

78 if ( 78 ↛ 83line 78 didn't jump to line 83 because the condition on line 78 was never true

79 file_kind == stat.S_IFLNK 

80 or entry.is_dir() 

81 or file_kind not in {0, stat.S_IFREG} 

82 ): 

83 raise BackupManifestError("backup contains a non-regular member") 

84 if entry.file_size > MAX_MEMBER_BYTES: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true

85 raise BackupManifestError("backup member exceeds the supported size limit") 

86 manifest_info = names.get(MANIFEST_NAME) 

87 if manifest_info is None or manifest_info.file_size > _MAX_MANIFEST_BYTES: 87 ↛ 88line 87 didn't jump to line 88 because the condition on line 87 was never true

88 raise BackupManifestError("backup manifest is missing or too large") 

89 declared = parse_manifest(read_limited(archive, manifest_info, _MAX_MANIFEST_BYTES)) 

90 

91 expected = {member.path: member for member in declared.members} 

92 payload_entries = { 

93 entry.filename: entry for entry in entries if entry.filename != MANIFEST_NAME 

94 } 

95 if len(payload_entries) != len(declared.members) or set(payload_entries) != set( 95 ↛ 98line 95 didn't jump to line 98 because the condition on line 95 was never true

96 expected 

97 ): 

98 raise BackupManifestError("backup manifest does not match its archive members") 

99 

100 total_size = 0 

101 for member_path, member in expected.items(): 

102 entry = payload_entries[member_path] 

103 if entry.file_size != member.size: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true

104 raise BackupManifestError(f"backup member size mismatch: {member_path}") 

105 total_size += member.size 

106 if total_size > _MAX_TOTAL_MEMBER_BYTES: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true

107 raise BackupManifestError("backup exceeds the supported total size") 

108 digest, header = _hash_member(archive, entry, member.size) 

109 if digest != member.sha256: 

110 raise BackupManifestError(f"backup member hash mismatch: {member_path}") 

111 _validate_member_schema(member, header, snapshot_schema) 

112 if is_sqlite_member(member_path): 

113 _validate_archived_sqlite( 

114 archive, 

115 entry, 

116 member, 

117 declared.installation_id, 

118 schema_catalog, 

119 snapshot_schema, 

120 ) 

121 return declared 

122 except BackupManifestError: 

123 raise 

124 except (OSError, RuntimeError, ValueError, zipfile.BadZipFile, EOFError) as exc: 

125 raise BackupManifestError("backup archive cannot be read") from exc 

126 

127 

128@contextmanager 

129def open_archive(archive_path: Path) -> Generator[zipfile.ZipFile]: 

130 """Open one regular archive without following a path replaced by a symlink.""" 

131 descriptor = os.open( 

132 archive_path, 

133 os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), 

134 ) 

135 try: 

136 info = os.fstat(descriptor) 

137 if not stat.S_ISREG(info.st_mode): 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true

138 raise BackupManifestError("backup archive must be a regular file") 

139 if info.st_size <= 0 or info.st_size > _MAX_ARCHIVE_BYTES: 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true

140 raise BackupManifestError("backup archive size is outside the supported limit") 

141 stream = os.fdopen(descriptor, "rb") 

142 descriptor = -1 

143 with stream, zipfile.ZipFile(stream, "r") as archive: 

144 yield archive 

145 finally: 

146 if descriptor >= 0: 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true

147 os.close(descriptor) 

148 

149 

150def read_limited(archive: zipfile.ZipFile, entry: zipfile.ZipInfo, limit: int) -> bytes: 

151 if entry.file_size > limit: 151 ↛ 152line 151 didn't jump to line 152 because the condition on line 151 was never true

152 raise BackupManifestError("backup member exceeds its declared size") 

153 with archive.open(entry, "r") as source: 

154 content = source.read(limit + 1) 

155 if len(content) > limit: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true

156 raise BackupManifestError("backup member exceeds its declared size") 

157 return content 

158 

159 

160def _hash_member( 

161 archive: zipfile.ZipFile, entry: zipfile.ZipInfo, expected_size: int 

162) -> tuple[str, bytes]: 

163 digest = hashlib.sha256() 

164 size = 0 

165 header = bytearray() 

166 with archive.open(entry, "r") as source: 

167 while chunk := source.read(1024 * 1024): 

168 size += len(chunk) 

169 if size > expected_size: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 raise BackupManifestError("backup member exceeds its declared size") 

171 digest.update(chunk) 

172 if len(header) < 32: 172 ↛ 167line 172 didn't jump to line 167 because the condition on line 172 was always true

173 header.extend(chunk[: 32 - len(header)]) 

174 if size != expected_size: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true

175 raise BackupManifestError("backup member size mismatch") 

176 return digest.hexdigest(), bytes(header) 

177 

178 

179def _validate_member_name(name: str) -> None: 

180 if not name or "\\" in name or "\x00" in name or name.startswith("/") or ":" in name: 

181 raise BackupManifestError("backup contains an unsafe member path") 

182 path = PurePosixPath(name) 

183 if path.is_absolute() or any(part in {"", ".", ".."} for part in name.split("/")): 

184 raise BackupManifestError("backup contains an unsafe member path") 

185 if name == MANIFEST_NAME: 

186 return 

187 if ( 187 ↛ 193line 187 didn't jump to line 193 because the condition on line 187 was always true

188 name in _TOP_LEVEL_MEMBERS 

189 or ACCOUNT_DATABASE.fullmatch(name) 

190 or _ATTACHMENT.fullmatch(name) 

191 ): 

192 return 

193 raise BackupManifestError("backup contains an unsupported member path") 

194 

195 

196def parse_manifest(content: bytes) -> BackupManifest: 

197 def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: 

198 result: dict[str, Any] = {} 

199 for key, value in pairs: 

200 if key in result: 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true

201 raise BackupManifestError("backup manifest contains a duplicate field") 

202 result[key] = value 

203 return result 

204 

205 try: 

206 raw = json.loads(content, object_pairs_hook=unique_object) 

207 except (UnicodeDecodeError, json.JSONDecodeError) as exc: 

208 raise BackupManifestError("backup manifest is invalid JSON") from exc 

209 if not isinstance(raw, dict): 209 ↛ 210line 209 didn't jump to line 210 because the condition on line 209 was never true

210 raise BackupManifestError("backup manifest must be an object") 

211 expected_fields = { 

212 "format", 

213 "format_version", 

214 "created_at", 

215 "installation_id", 

216 "environment_ids", 

217 "members", 

218 } 

219 if set(raw) != expected_fields or raw.get("format") != ARCHIVE_FORMAT: 219 ↛ 220line 219 didn't jump to line 220 because the condition on line 219 was never true

220 raise BackupManifestError("backup manifest format is unsupported") 

221 version = raw.get("format_version") 

222 if type(version) is not int or version != FORMAT_VERSION: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true

223 raise BackupManifestError("backup manifest version is unsupported") 

224 installation_id = raw.get("installation_id") 

225 try: 

226 canonical_id = str(uuid.UUID(installation_id)) 

227 except (ValueError, TypeError, AttributeError) as exc: 

228 raise BackupManifestError("backup installation identity is invalid") from exc 

229 if installation_id.lower() != canonical_id: 229 ↛ 230line 229 didn't jump to line 230 because the condition on line 229 was never true

230 raise BackupManifestError("backup installation identity is invalid") 

231 created_value = raw.get("created_at") 

232 if not isinstance(created_value, str): 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 raise BackupManifestError("backup timestamp is invalid") 

234 try: 

235 created_at = datetime.fromisoformat(created_value) 

236 except ValueError as exc: 

237 raise BackupManifestError("backup timestamp is invalid") from exc 

238 if created_at.tzinfo is None or created_at.utcoffset() is None: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true

239 raise BackupManifestError("backup timestamp must include a timezone") 

240 environment_ids_raw = raw.get("environment_ids") 

241 if ( 241 ↛ 248line 241 didn't jump to line 248 because the condition on line 241 was never true

242 not isinstance(environment_ids_raw, list) 

243 or any( 

244 not isinstance(item, str) or not item or len(item) > 128 for item in environment_ids_raw 

245 ) 

246 or len(environment_ids_raw) != len(set(environment_ids_raw)) 

247 ): 

248 raise BackupManifestError("backup account identities are invalid") 

249 members_raw = raw.get("members") 

250 if not isinstance(members_raw, list) or not members_raw or len(members_raw) > _MAX_MEMBER_COUNT: 250 ↛ 251line 250 didn't jump to line 251 because the condition on line 250 was never true

251 raise BackupManifestError("backup member list is invalid") 

252 members: list[BackupMember] = [] 

253 seen_paths: set[str] = set() 

254 for entry in members_raw: 

255 if not isinstance(entry, dict) or set(entry) != { 255 ↛ 261line 255 didn't jump to line 261 because the condition on line 255 was never true

256 "path", 

257 "size", 

258 "sha256", 

259 "schema_version", 

260 }: 

261 raise BackupManifestError("backup member metadata is invalid") 

262 member_path = entry["path"] 

263 _validate_member_name(member_path) 

264 if member_path == MANIFEST_NAME or member_path in seen_paths: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true

265 raise BackupManifestError("backup manifest contains a duplicate member") 

266 seen_paths.add(member_path) 

267 size = entry["size"] 

268 digest = entry["sha256"] 

269 schema_version = entry["schema_version"] 

270 if type(size) is not int or size < 0 or size > MAX_MEMBER_BYTES: 270 ↛ 271line 270 didn't jump to line 271 because the condition on line 270 was never true

271 raise BackupManifestError("backup member size is invalid") 

272 if not isinstance(digest, str) or not DIGEST.fullmatch(digest): 272 ↛ 273line 272 didn't jump to line 273 because the condition on line 272 was never true

273 raise BackupManifestError("backup member hash is invalid") 

274 if not isinstance(schema_version, str) or not schema_version or len(schema_version) > 256: 274 ↛ 275line 274 didn't jump to line 275 because the condition on line 274 was never true

275 raise BackupManifestError("backup member schema version is invalid") 

276 members.append(BackupMember(member_path, size, digest, schema_version)) 

277 if "application.db" not in seen_paths: 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true

278 raise BackupManifestError("backup is missing the operational database") 

279 return BackupManifest( 

280 format_version=version, 

281 created_at=created_at.astimezone(UTC), 

282 installation_id=canonical_id, 

283 environment_ids=tuple(environment_ids_raw), 

284 members=tuple(members), 

285 ) 

286 

287 

288def _validate_member_schema( 

289 member: BackupMember, 

290 header: bytes, 

291 snapshot_schema: AccountSnapshotSchemaProvider, 

292) -> None: 

293 if is_sqlite_member(member.path): 

294 if not member.schema_version.startswith("sqlite:") or not header.startswith( 

295 b"SQLite format 3\x00" 

296 ): 

297 raise BackupManifestError(f"backup SQLite schema is missing: {member.path}") 

298 if member.path == "application.db" and not member.schema_version.startswith( 

299 APPLICATION_SCHEMA_PREFIX 

300 ): 

301 raise BackupManifestError("backup application database schema is unsupported") 

302 if ACCOUNT_DATABASE.fullmatch( 302 ↛ 305line 302 didn't jump to line 305 because the condition on line 302 was never true

303 member.path 

304 ) and member.schema_version != account_schema_version(snapshot_schema): 

305 raise BackupManifestError("backup account database schema is unsupported") 

306 elif member.path.endswith(".json") and member.schema_version != "json:1": 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true

307 raise BackupManifestError(f"backup JSON schema is unsupported: {member.path}") 

308 elif member.path.startswith("attachments/") and member.schema_version != "blob:1": 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true

309 raise BackupManifestError(f"backup attachment schema is unsupported: {member.path}") 

310 

311 

312def _validate_archived_sqlite( 

313 archive: zipfile.ZipFile, 

314 entry: zipfile.ZipInfo, 

315 member: BackupMember, 

316 installation_id: str, 

317 schema_catalog: SchemaCatalogProvider, 

318 snapshot_schema: AccountSnapshotSchemaProvider, 

319) -> None: 

320 with tempfile.TemporaryDirectory(prefix="copytrading-backup-check-") as temporary: 

321 target = Path(temporary) / "member.sqlite3" 

322 size = 0 

323 with archive.open(entry, "r") as source, target.open("xb") as destination: 

324 while chunk := source.read(1024 * 1024): 

325 size += len(chunk) 

326 if size > member.size: 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true

327 raise BackupManifestError("backup SQLite member exceeds its declared size") 

328 destination.write(chunk) 

329 if size != member.size: 329 ↛ 330line 329 didn't jump to line 330 because the condition on line 329 was never true

330 raise BackupManifestError("backup SQLite member size mismatch") 

331 os.chmod(target, 0o600) 

332 actual_schema = validate_sqlite( 

333 target, 

334 member.path, 

335 installation_id, 

336 schema_catalog, 

337 snapshot_schema, 

338 ) 

339 if actual_schema != member.schema_version: 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true

340 raise BackupManifestError(f"backup SQLite schema mismatch: {member.path}")