Coverage for src/copytrading_engine/execution/application/recovery.py: 88%

165 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-16 16:55 +0000

1"""Offline, broker-verified operator recovery for the execution ledger.""" 

2 

3import datetime as dt 

4from collections.abc import Callable 

5from decimal import Decimal 

6 

7from copytrading_engine.execution.application.ledger import TradingLedger 

8from copytrading_engine.execution.application.ports import Broker 

9from copytrading_engine.execution.domain.market import Account, Position 

10from copytrading_engine.execution.domain.order_lifecycle import OrderStatus, is_terminal 

11from copytrading_engine.execution.domain.ownership import ( 

12 OwnershipInspection, 

13 OwnershipResolution, 

14 OwnershipResolutionRequest, 

15 account_activity_reason, 

16 still_open, 

17) 

18from copytrading_engine.execution.domain.recovery import ( 

19 IncidentClearanceEvidence, 

20 ManualSale, 

21 ReleaseEvidence, 

22) 

23from copytrading_engine.execution.domain.risk import account_exposure 

24from copytrading_engine.execution.domain.values import BrokerAccountNumber 

25 

26 

27class RecoveryApplication: 

28 """Verify current broker observations before asking the ledger to record recovery.""" 

29 

30 def __init__( 

31 self, 

32 ledger: TradingLedger, 

33 broker: Broker, 

34 *, 

35 clock: Callable[[], dt.datetime] | None = None, 

36 ) -> None: 

37 self._ledger = ledger 

38 self._broker = broker 

39 self._clock = clock or (lambda: dt.datetime.now(dt.UTC)) 

40 

41 def inspect_ownership( 

42 self, 

43 *, 

44 account: Account | None = None, 

45 positions: tuple[Position, ...] | None = None, 

46 ) -> OwnershipInspection: 

47 """Inspect against the broker now, or against an account and positions just read.""" 

48 snapshot = self._ledger.state 

49 if snapshot.account_id is None or snapshot.environment is None: 49 ↛ 50line 49 didn't jump to line 50 because the condition on line 49 was never true

50 raise RuntimeError("Broker account inventory has not been initialized") 

51 if account is None: 

52 account = self._verify_account(snapshot.account_id, snapshot.account_id) 

53 elif account.id != snapshot.account_id: 53 ↛ 54line 53 didn't jump to line 54 because the condition on line 53 was never true

54 raise ValueError("Supplied, broker, and ledger account identities must match") 

55 if positions is None: 

56 positions = self._broker.positions() 

57 self._position_map(positions) 

58 try: 

59 exposure = account_exposure( 

60 positions, 

61 tuple(snapshot.lots.values()), 

62 tuple(order for order in snapshot.orders.values() if order.pending), 

63 account_currency=account.currency, 

64 ) 

65 risk_status = "ready" 

66 risk_reason = None 

67 total_exposure = exposure.total 

68 except ValueError as error: 

69 risk_status = "unavailable" 

70 risk_reason = str(error) 

71 total_exposure = None 

72 activity_reason = account_activity_reason( 

73 still_open(self._broker.open_orders(), tuple(snapshot.orders.values())), 

74 tuple(order for order in snapshot.orders.values() if order.pending), 

75 unresolved_order_incidents=snapshot.entry_halted, 

76 ) 

77 return OwnershipInspection( 

78 account_id=snapshot.account_id, 

79 environment=snapshot.environment, 

80 external_positions=tuple(snapshot.external_positions.values()), 

81 incidents=tuple(snapshot.ownership_incidents.values()), 

82 broker_positions=positions, 

83 expected_quantities=snapshot.expected_holdings, 

84 account_risk_status=risk_status, 

85 account_risk_reason=risk_reason, 

86 total_exposure_usd=total_exposure, 

87 account_activity_status="ready" if activity_reason is None else "unavailable", 

88 account_activity_reason=activity_reason, 

89 ) 

90 

91 def release_uncertain_intent( 

92 self, 

93 client_id: str, 

94 *, 

95 actor: str, 

96 reason: str, 

97 order_history_ref: str, 

98 fill_history_ref: str, 

99 account_id: str, 

100 ) -> ReleaseEvidence: 

101 """Release a never-confirmed intent without replaying its source instruction.""" 

102 self._require_attestation(actor, reason, order_history_ref, fill_history_ref) 

103 snapshot = self._ledger.state 

104 order = snapshot.orders.get(client_id) 

105 if order is None or order.status != OrderStatus.UNCERTAIN: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 raise ValueError("Only a saved uncertain order intent can be released") 

107 if order.filled_qty != 0 or order.broker_id is not None: 107 ↛ 108line 107 didn't jump to line 108 because the condition on line 107 was never true

108 raise ValueError("An uncertain intent with saved broker activity cannot be released") 

109 if client_id in snapshot.release_evidence: 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true

110 raise ValueError("Order intent already has release evidence") 

111 

112 self._verify_account(snapshot.account_id, account_id) 

113 if self._broker.lookup(client_id) is not None: 

114 raise ValueError("Broker lookup found the uncertain order; reconcile it first") 

115 open_orders = self._broker.open_orders() 

116 if any( 

117 item.client_order_id == client_id 

118 or (order.broker_id is not None and item.id == order.broker_id) 

119 for item in open_orders 

120 ): 

121 raise ValueError("A matching broker order is still open") 

122 actual = self._position_map(self._broker.positions()) 

123 expected_qty = snapshot.expected_holdings.get(order.symbol, Decimal(0)) 

124 if actual.get(order.symbol, Decimal(0)) != expected_qty: 

125 raise ValueError("Broker position does not exactly match the saved ledger position") 

126 

127 evidence = ReleaseEvidence( 

128 actor=actor, 

129 reason=reason, 

130 order_history_ref=order_history_ref, 

131 fill_history_ref=fill_history_ref, 

132 checked_at=self._clock(), 

133 account_id=account_id, 

134 submission_state=("started" if order.submit_started_at is not None else "not_started"), 

135 ) 

136 self._ledger.release_uncertain_intent(client_id, evidence) 

137 return evidence 

138 

139 def record_manual_sale(self, sale: ManualSale, *, account_id: str) -> None: 

140 """Import an externally completed sale after verifying broker fill and holdings.""" 

141 self._require_attestation(sale.reason) 

142 snapshot = self._ledger.state 

143 self._verify_account(snapshot.account_id, account_id) 

144 

145 observed = self._broker.lookup(sale.order.client_order_id) 

146 if observed is None or observed != sale.order: 

147 raise ValueError("Provided manual sale does not match the broker order record") 

148 if self._ledger.pending(): 148 ↛ 149line 148 didn't jump to line 149 because the condition on line 148 was never true

149 raise ValueError("Reconcile pending copier orders before recording a manual sale") 

150 if self._broker.open_orders(): 

151 raise ValueError("Broker has open orders; verify the account before recording a sale") 

152 

153 expected = snapshot.expected_holdings 

154 existing = snapshot.manual_sales.get(sale.order.id) 

155 if existing is None: 

156 expected[sale.order.symbol] = ( 

157 expected.get(sale.order.symbol, Decimal(0)) - sale.order.filled_qty 

158 ) 

159 elif existing.lot_id is None: 

160 incident = snapshot.ownership_incidents[sale.order.id] 

161 if not incident.resolved: 161 ↛ 163line 161 didn't jump to line 163 because the condition on line 161 was always true

162 expected[sale.order.symbol] = incident.actual_qty 

163 actual = self._position_map(self._broker.positions()) 

164 if not self._positions_equal(expected, actual): 

165 raise ValueError("Resulting broker positions do not match the projected ledger") 

166 

167 # The ledger remains authoritative for the selected lot and quantity policy. 

168 self._ledger.record_manual_sale(sale, actual.get(sale.order.symbol, Decimal(0))) 

169 

170 def resolve_ownership(self, request: OwnershipResolutionRequest) -> OwnershipResolution: 

171 """Allocate a verified broker quantity without creating a broker order.""" 

172 request = OwnershipResolutionRequest.model_validate(request.model_dump()) 

173 snapshot = self._ledger.state 

174 prior = snapshot.ownership_resolutions.get(request.resolution_id) 

175 if prior is not None: 

176 if prior.request != request: 

177 raise ValueError("Ownership resolution ID conflicts with different content") 

178 return prior 

179 self._verify_account(snapshot.account_id, request.account_id) 

180 if self._ledger.pending(request.symbol): 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true

181 raise ValueError("Pending app order overlaps ownership resolution") 

182 open_orders = self._broker.open_orders() 

183 known_open = tuple(order for order in snapshot.orders.values() if order.pending) 

184 if len(open_orders) >= 500: 

185 raise ValueError("Broker open-order observation is incomplete") 

186 if any(order.symbol == request.symbol for order in open_orders): 

187 raise ValueError("Broker open order overlaps ownership resolution") 

188 if reason := account_activity_reason( 188 ↛ 193line 188 didn't jump to line 193 because the condition on line 188 was never true

189 open_orders, 

190 known_open, 

191 unresolved_order_incidents=snapshot.entry_halted, 

192 ): 

193 raise ValueError(f"Broker account activity is unresolved: {reason}") 

194 actual = self._position_map(self._broker.positions()) 

195 refreshed_orders = self._broker.open_orders() 

196 if len(refreshed_orders) >= 500: 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true

197 raise ValueError("Broker open-order observation is incomplete") 

198 if {order.client_order_id: order for order in open_orders} != { 

199 order.client_order_id: order for order in refreshed_orders 

200 }: 

201 raise ValueError("Broker open-order activity changed during ownership resolution") 

202 if any(order.symbol == request.symbol for order in refreshed_orders): 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 raise ValueError("Broker open order overlaps ownership resolution") 

204 if reason := account_activity_reason( 204 ↛ 209line 204 didn't jump to line 209 because the condition on line 204 was never true

205 refreshed_orders, 

206 known_open, 

207 unresolved_order_incidents=snapshot.entry_halted, 

208 ): 

209 raise ValueError(f"Broker account activity is unresolved: {reason}") 

210 if actual.get(request.symbol, Decimal(0)) != request.broker_qty: 

211 raise ValueError("Fresh broker quantity differs from requested allocation") 

212 self._verify_account(snapshot.account_id, request.account_id) 

213 return self._ledger.resolve_ownership(request, self._clock()) 

214 

215 def clear_late_order_incident( 

216 self, 

217 client_id: str, 

218 *, 

219 actor: str, 

220 reason: str, 

221 matched_audit_ref: str, 

222 account_id: str, 

223 ) -> IncidentClearanceEvidence: 

224 """Refresh a late order, then clear only after an exact account position audit.""" 

225 self._require_attestation(actor, reason, matched_audit_ref) 

226 snapshot = self._ledger.state 

227 if client_id not in snapshot.release_evidence: 227 ↛ 228line 227 didn't jump to line 228 because the condition on line 227 was never true

228 raise ValueError("No released uncertain order exists for this client ID") 

229 self._verify_account(snapshot.account_id, account_id) 

230 

231 observed = self._broker.lookup(client_id) 

232 if observed is None: 

233 raise ValueError("Broker lookup could not confirm the late order") 

234 observed_at = self._clock() 

235 self._ledger.apply_order(client_id, observed, observed_at) 

236 refreshed = self._ledger.state 

237 incident = refreshed.late_order_incidents.get(client_id) 

238 if incident is None: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true

239 raise ValueError("Broker refresh did not create a late-order incident") 

240 order = refreshed.orders[client_id] 

241 if not is_terminal(order.status): 

242 raise ValueError("The refreshed late order is not in a terminal state") 

243 if self._ledger.pending() or self._broker.open_orders(): 

244 raise ValueError("Open orders prevent a reliable incident position audit") 

245 

246 actual = self._position_map(self._broker.positions()) 

247 expected = refreshed.expected_holdings 

248 if not self._positions_equal(expected, actual): 

249 raise ValueError("Fresh broker positions do not exactly match ledger-owned positions") 

250 checked_at = self._clock() 

251 evidence = IncidentClearanceEvidence( 

252 actor=actor, 

253 reason=reason, 

254 checked_at=checked_at, 

255 account_id=account_id, 

256 matched_audit_ref=matched_audit_ref, 

257 symbol=incident.symbol, 

258 expected_qty=expected.get(incident.symbol, Decimal(0)), 

259 broker_qty=actual.get(incident.symbol, Decimal(0)), 

260 ) 

261 self._ledger.clear_late_order_incident(client_id, evidence) 

262 return evidence 

263 

264 def _verify_account( 

265 self, ledger_account_id: BrokerAccountNumber | None, supplied_account_id: str 

266 ) -> Account: 

267 account = self._broker.account() 

268 if ( 

269 ledger_account_id is None 

270 or account.id != ledger_account_id 

271 or supplied_account_id != account.id 

272 ): 

273 raise ValueError("Supplied, broker, and ledger account identities must match") 

274 return account 

275 

276 @staticmethod 

277 def _require_attestation(*values: str) -> None: 

278 if any(not value.strip() for value in values): 

279 raise ValueError("Operator attestation and evidence references must be nonblank") 

280 

281 @staticmethod 

282 def _position_map(positions: tuple[Position, ...]) -> dict[str, Decimal]: 

283 result: dict[str, Decimal] = {} 

284 for position in positions: 

285 if position.symbol in result: 285 ↛ 286line 285 didn't jump to line 286 because the condition on line 285 was never true

286 raise ValueError("Broker returned duplicate position symbols") 

287 result[position.symbol] = position.qty 

288 return result 

289 

290 @staticmethod 

291 def _positions_equal(expected: dict[str, Decimal], actual: dict[str, Decimal]) -> bool: 

292 return all( 

293 expected.get(symbol, Decimal(0)) == actual.get(symbol, Decimal(0)) 

294 for symbol in expected.keys() | actual.keys() 

295 )