Coverage for pydantic_evals/pydantic_evals/reporting/__init__.py: 96.95%

1from __future__ import annotations as _annotations (empty)

2 

3from collections import defaultdict (empty)

4from collections.abc import Mapping (empty)

5from dataclasses import dataclass, field (empty)

6from io import StringIO (empty)

7from typing import Any, Callable, Literal, Protocol, TypeVar (empty)

8 

9from pydantic import BaseModel (empty)

10from rich.console import Console (empty)

11from rich.table import Table (empty)

12from typing_extensions import TypedDict (empty)

13 

14from pydantic_evals._utils import UNSET, Unset (empty)

15 

16from ..evaluators import EvaluationResult (empty)

17from .render_numbers import ( (empty)

18 default_render_duration, 

19 default_render_duration_diff, 

20 default_render_number, 

21 default_render_number_diff, 

22 default_render_percentage, 

23) 

24 

25__all__ = ( (empty)

26 'EvaluationReport', 

27 'ReportCase', 

28 'EvaluationRenderer', 

29 'RenderValueConfig', 

30 'RenderNumberConfig', 

31 'ReportCaseAggregate', 

32) 

33 

34MISSING_VALUE_STR = '[i]<missing>[/i]' (empty)

35EMPTY_CELL_STR = '-' (empty)

36EMPTY_AGGREGATE_CELL_STR = '' (empty)

37 

38 

39class ReportCase(BaseModel): (empty)

40 """A single case in an evaluation report.""" 

41 

42 name: str (empty)

43 """The name of the [case][pydantic_evals.Case].""" (empty)

44 inputs: Any (empty)

45 """The inputs to the task, from [`Case.inputs`][pydantic_evals.Case.inputs].""" (empty)

46 metadata: Any (empty)

47 """Any metadata associated with the case, from [`Case.metadata`][pydantic_evals.Case.metadata].""" (empty)

48 expected_output: Any (empty)

49 """The expected output of the task, from [`Case.expected_output`][pydantic_evals.Case.expected_output].""" (empty)

50 output: Any (empty)

51 """The output of the task execution.""" (empty)

52 

53 metrics: dict[str, float | int] (empty)

54 attributes: dict[str, Any] (empty)

55 

56 scores: dict[str, EvaluationResult[int | float]] = field(init=False) (empty)

57 labels: dict[str, EvaluationResult[str]] = field(init=False) (empty)

58 assertions: dict[str, EvaluationResult[bool]] = field(init=False) (empty)

59 

60 task_duration: float (empty)

61 total_duration: float # includes evaluator execution time (empty)

62 

63 # TODO(DavidM): Drop these once we can reference child spans in details panel: 

64 trace_id: str (empty)

65 span_id: str (empty)

66 

67 

68class ReportCaseAggregate(BaseModel): (empty)

69 """A synthetic case that summarizes a set of cases.""" 

70 

71 name: str (empty)

72 

73 scores: dict[str, float | int] (empty)

74 labels: dict[str, dict[str, float]] (empty)

75 metrics: dict[str, float | int] (empty)

76 assertions: float | None (empty)

77 task_duration: float (empty)

78 total_duration: float (empty)

79 

80 @staticmethod (empty)

81 def average(cases: list[ReportCase]) -> ReportCaseAggregate: (empty)

82 """Produce a synthetic "summary" case by averaging quantitative attributes.""" 

83 num_cases = len(cases) (empty)

84 if num_cases == 0: (empty)

85 return ReportCaseAggregate( (empty)

86 name='Averages', 

87 scores={}, 

88 labels={}, 

89 metrics={}, 

90 assertions=None, 

91 task_duration=0.0, 

92 total_duration=0.0, 

93 ) 

94 

95 def _scores_averages(scores_by_name: list[dict[str, int | float | bool]]) -> dict[str, float]: (empty)

96 counts_by_name: dict[str, int] = defaultdict(int) (empty)

97 sums_by_name: dict[str, float] = defaultdict(float) (empty)

98 for sbn in scores_by_name: (empty)

99 for name, score in sbn.items(): (empty)

100 counts_by_name[name] += 1 (empty)

101 sums_by_name[name] += score (empty)

102 return {name: sums_by_name[name] / counts_by_name[name] for name in sums_by_name} (empty)

103 

104 def _labels_averages(labels_by_name: list[dict[str, str]]) -> dict[str, dict[str, float]]: (empty)

105 counts_by_name: dict[str, int] = defaultdict(int) (empty)

106 sums_by_name: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float)) (empty)

107 for lbn in labels_by_name: (empty)

108 for name, label in lbn.items(): (empty)

109 counts_by_name[name] += 1 (empty)

110 sums_by_name[name][label] += 1 (empty)

111 return { (empty)

112 name: {value: count / counts_by_name[name] for value, count in sums_by_name[name].items()} 

113 for name in sums_by_name 

114 } 

115 

116 average_task_duration = sum(case.task_duration for case in cases) / num_cases (empty)

117 average_total_duration = sum(case.total_duration for case in cases) / num_cases (empty)

118 

119 # average_assertions: dict[str, float] = _scores_averages([{k: v.value for k, v in case.scores.items()} for case in cases]) 

120 average_scores: dict[str, float] = _scores_averages( (empty)

121 [{k: v.value for k, v in case.scores.items()} for case in cases] 

122 ) 

123 average_labels: dict[str, dict[str, float]] = _labels_averages( (empty)

124 [{k: v.value for k, v in case.labels.items()} for case in cases] 

125 ) 

126 average_metrics: dict[str, float] = _scores_averages([case.metrics for case in cases]) (empty)

127 

128 average_assertions: float | None = None (empty)

129 n_assertions = sum(len(case.assertions) for case in cases) (empty)

130 if n_assertions > 0: (empty)

131 n_passing = sum(1 for case in cases for assertion in case.assertions.values() if assertion.value) (empty)

132 average_assertions = n_passing / n_assertions (empty)

133 

134 return ReportCaseAggregate( (empty)

135 name='Averages', 

136 scores=average_scores, 

137 labels=average_labels, 

138 metrics=average_metrics, 

139 assertions=average_assertions, 

140 task_duration=average_task_duration, 

141 total_duration=average_total_duration, 

142 ) 

143 

144 

145class EvaluationReport(BaseModel): (empty)

146 """A report of the results of evaluating a model on a set of cases.""" 

147 

148 name: str (empty)

149 """The name of the report.""" (empty)

150 cases: list[ReportCase] (empty)

151 """The cases in the report.""" (empty)

152 

153 def print( (empty)

154 self, 

155 width: int | None = None, (empty)

156 baseline: EvaluationReport | None = None, (empty)

157 include_input: bool = False, (empty)

158 include_output: bool = False, (empty)

159 include_total_duration: bool = False, (empty)

160 include_removed_cases: bool = False, (empty)

161 include_averages: bool = True, (empty)

162 input_config: RenderValueConfig | None = None, (empty)

163 output_config: RenderValueConfig | None = None, (empty)

164 score_configs: dict[str, RenderNumberConfig] | None = None, (empty)

165 label_configs: dict[str, RenderValueConfig] | None = None, (empty)

166 metric_configs: dict[str, RenderNumberConfig] | None = None, (empty)

167 duration_config: RenderNumberConfig | None = None, (empty)

168 ): # pragma: no cover 

169 """Print this report to the console, optionally comparing it to a baseline report. 

170 

171 If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`. 

172 """ 

173 table = self.console_table( 

174 baseline=baseline, 

175 include_input=include_input, 

176 include_output=include_output, 

177 include_total_duration=include_total_duration, 

178 include_removed_cases=include_removed_cases, 

179 include_averages=include_averages, 

180 input_config=input_config, 

181 output_config=output_config, 

182 score_configs=score_configs, 

183 label_configs=label_configs, 

184 metric_configs=metric_configs, 

185 duration_config=duration_config, 

186 ) 

187 Console(width=width).print(table) 

188 

189 def console_table( (empty)

190 self, 

191 baseline: EvaluationReport | None = None, 

192 include_input: bool = False, 

193 include_output: bool = False, 

194 include_total_duration: bool = False, 

195 include_removed_cases: bool = False, 

196 include_averages: bool = True, 

197 input_config: RenderValueConfig | None = None, 

198 output_config: RenderValueConfig | None = None, 

199 score_configs: dict[str, RenderNumberConfig] | None = None, 

200 label_configs: dict[str, RenderValueConfig] | None = None, 

201 metric_configs: dict[str, RenderNumberConfig] | None = None, 

202 duration_config: RenderNumberConfig | None = None, 

203 ) -> Table: 

204 """Return a table containing the data from this report, or the diff between this report and a baseline report. 

205 

206 Optionally include input and output details. 

207 """ 

208 renderer = EvaluationRenderer( (empty)

209 include_input=include_input, 

210 include_output=include_output, 

211 include_total_duration=include_total_duration, 

212 include_removed_cases=include_removed_cases, 

213 include_averages=include_averages, 

214 input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})}, 

215 output_config=output_config or _DEFAULT_VALUE_CONFIG, 

216 score_configs=score_configs or {}, 

217 label_configs=label_configs or {}, 

218 metric_configs=metric_configs or {}, 

219 duration_config=duration_config or _DEFAULT_DURATION_CONFIG, 

220 ) 

221 if baseline is None: (empty)

222 return renderer.build_table(self) (empty)

223 else: # pragma: no cover 

224 return renderer.build_diff_table(self, baseline) 

225 

226 def __str__(self) -> str: (empty)

227 """Return a string representation of the report.""" 

228 table = self.console_table() (empty)

229 io_file = StringIO() (empty)

230 Console(file=io_file).print(table) (empty)

231 return io_file.getvalue() (empty)

232 

233 

234class RenderValueConfig(TypedDict, total=False): (empty)

235 """A configuration for rendering a values in an Evaluation report.""" 

236 

237 value_formatter: str | Callable[[Any], str] (empty)

238 diff_checker: Callable[[Any, Any], bool] | None (empty)

239 diff_formatter: Callable[[Any, Any], str | None] | None (empty)

240 diff_style: str (empty)

241 

242 

243@dataclass (empty)

244class _ValueRenderer: (empty)

245 value_formatter: str | Callable[[Any], str] = '{}' (empty)

246 diff_checker: Callable[[Any, Any], bool] | None = lambda x, y: x != y (empty)

247 diff_formatter: Callable[[Any, Any], str | None] | None = None (empty)

248 diff_style: str = 'magenta' (empty)

249 

250 @staticmethod (empty)

251 def from_config(config: RenderValueConfig) -> _ValueRenderer: (empty)

252 return _ValueRenderer( (empty)

253 value_formatter=config.get('value_formatter', '{}'), 

254 diff_checker=config.get('diff_checker', lambda x, y: x != y), 

255 diff_formatter=config.get('diff_formatter'), 

256 diff_style=config.get('diff_style', 'magenta'), 

257 ) 

258 

259 def render_value(self, name: str | None, v: Any) -> str: (empty)

260 result = self._get_value_str(v) (empty)

261 if name: (empty)

262 result = f'{name}: {result}' (empty)

263 return result (empty)

264 

265 def render_diff(self, name: str | None, old: Any | None, new: Any | None) -> str: (empty)

266 old_str = self._get_value_str(old) or MISSING_VALUE_STR (empty)

267 new_str = self._get_value_str(new) or MISSING_VALUE_STR (empty)

268 if old_str == new_str: (empty)

269 result = old_str (empty)

270 else: 

271 result = f'{old_str} → {new_str}' (empty)

272 

273 has_diff = self.diff_checker and self.diff_checker(old, new) (empty)

274 if has_diff: 274 ↛ 283line 274 didn't jump to line 283 because the condition on line 274 was always true(empty)

275 # If there is a diff, make the name bold and compute the diff_str 

276 name = name and f'[bold]{name}[/]' (empty)

277 diff_str = self.diff_formatter and self.diff_formatter(old, new) (empty)

278 if diff_str: # pragma: no cover (empty)

279 result += f' ({diff_str})' 

280 result = f'[{self.diff_style}]{result}[/]' (empty)

281 

282 # Add the name 

283 if name: (empty)

284 result = f'{name}: {result}' (empty)

285 

286 return result (empty)

287 

288 def _get_value_str(self, value: Any) -> str: (empty)

289 if value is None: (empty)

290 return MISSING_VALUE_STR (empty)

291 if isinstance(self.value_formatter, str): (empty)

292 return self.value_formatter.format(value) (empty)

293 else: 

294 return self.value_formatter(value) (empty)

295 

296 

297class RenderNumberConfig(TypedDict, total=False): (empty)

298 """A configuration for rendering a particular score or metric in an Evaluation report. 

299 

300 See the implementation of `_RenderNumber` for more clarity on how these parameters affect the rendering. 

301 """ 

302 

303 value_formatter: str | Callable[[float | int], str] (empty)

304 """The logic to use for formatting values. (empty)

305 

306 * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures. 

307 * You can also use a custom string format spec, e.g. '{:.3f}' 

308 * You can also use a custom function, e.g. lambda x: f'{x:.3f}' 

309 """ 

310 diff_formatter: str | Callable[[float | int, float | int], str | None] | None (empty)

311 """The logic to use for formatting details about the diff. (empty)

312 

313 The strings produced by the value_formatter will always be included in the reports, but the diff_formatter is 

314 used to produce additional text about the difference between the old and new values, such as the absolute or 

315 relative difference. 

316 

317 * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four 

318 significant figures, and will include the percentage change. 

319 * You can also use a custom string format spec, e.g. '{:+.3f}' 

320 * You can also use a custom function, e.g. lambda x: f'{x:+.3f}'. 

321 If this function returns None, no extra diff text will be added. 

322 * You can also use None to never generate extra diff text. 

323 """ 

324 diff_atol: float (empty)

325 """The absolute tolerance for considering a difference "significant". (empty)

326 

327 A difference is "significant" if `abs(new - old) < self.diff_atol + self.diff_rtol * abs(old)`. 

328 

329 If a difference is not significant, it will not have the diff styles applied. Note that we still show 

330 both the rendered before and after values in the diff any time they differ, even if the difference is not 

331 significant. (If the rendered values are exactly the same, we only show the value once.) 

332 

333 If not provided, use 1e-6. 

334 """ 

335 diff_rtol: float (empty)

336 """The relative tolerance for considering a difference "significant". (empty)

337 

338 See the description of `diff_atol` for more details about what makes a difference "significant". 

339 

340 If not provided, use 0.001 if all values are ints, otherwise 0.05. 

341 """ 

342 diff_increase_style: str (empty)

343 """The style to apply to diffed values that have a significant increase. (empty)

344 

345 See the description of `diff_atol` for more details about what makes a difference "significant". 

346 

347 If not provided, use green for scores and red for metrics. You can also use arbitrary `rich` styles, such as "bold red". 

348 """ 

349 diff_decrease_style: str (empty)

350 """The style to apply to diffed values that have significant decrease. (empty)

351 

352 See the description of `diff_atol` for more details about what makes a difference "significant". 

353 

354 If not provided, use red for scores and green for metrics. You can also use arbitrary `rich` styles, such as "bold red". 

355 """ 

356 

357 

358@dataclass (empty)

359class _NumberRenderer: (empty)

360 """See documentation of `RenderNumberConfig` for more details about the parameters here.""" 

361 

362 value_formatter: str | Callable[[float | int], str] (empty)

363 diff_formatter: str | Callable[[float | int, float | int], str | None] | None (empty)

364 diff_atol: float (empty)

365 diff_rtol: float (empty)

366 diff_increase_style: str (empty)

367 diff_decrease_style: str (empty)

368 

369 def render_value(self, name: str | None, v: float | int) -> str: (empty)

370 result = self._get_value_str(v) (empty)

371 if name: (empty)

372 result = f'{name}: {result}' (empty)

373 return result (empty)

374 

375 def render_diff(self, name: str | None, old: float | int | None, new: float | int | None) -> str: (empty)

376 old_str = self._get_value_str(old) (empty)

377 new_str = self._get_value_str(new) (empty)

378 if old_str == new_str: (empty)

379 result = old_str (empty)

380 else: 

381 result = f'{old_str} → {new_str}' (empty)

382 

383 diff_style = self._get_diff_style(old, new) (empty)

384 if diff_style: (empty)

385 # If there is a diff, make the name bold and compute the diff_str 

386 name = name and f'[bold]{name}[/]' (empty)

387 diff_str = self._get_diff_str(old, new) (empty)

388 if diff_str: 388 ↛ 390line 388 didn't jump to line 390 because the condition on line 388 was always true(empty)

389 result += f' ({diff_str})' (empty)

390 result = f'[{diff_style}]{result}[/]' (empty)

391 

392 # Add the name 

393 if name: 393 ↛ 396line 393 didn't jump to line 396 because the condition on line 393 was always true(empty)

394 result = f'{name}: {result}' (empty)

395 

396 return result (empty)

397 

398 @staticmethod (empty)

399 def infer_from_config( (empty)

400 config: RenderNumberConfig, kind: Literal['score', 'metric', 'duration'], values: list[float | int] 

401 ) -> _NumberRenderer: 

402 value_formatter = config.get('value_formatter', UNSET) (empty)

403 if isinstance(value_formatter, Unset): (empty)

404 value_formatter = default_render_number (empty)

405 

406 diff_formatter = config.get('diff_formatter', UNSET) (empty)

407 if isinstance(diff_formatter, Unset): (empty)

408 diff_formatter = default_render_number_diff (empty)

409 

410 diff_atol = config.get('diff_atol', UNSET) (empty)

411 if isinstance(diff_atol, Unset): (empty)

412 diff_atol = 1e-6 (empty)

413 

414 diff_rtol = config.get('diff_rtol', UNSET) (empty)

415 if isinstance(diff_rtol, Unset): (empty)

416 values_are_ints = all(isinstance(v, int) for v in values) (empty)

417 diff_rtol = 0.001 if values_are_ints else 0.05 (empty)

418 

419 diff_increase_style = config.get('diff_increase_style', UNSET) (empty)

420 if isinstance(diff_increase_style, Unset): (empty)

421 diff_increase_style = 'green' if kind == 'score' else 'red' (empty)

422 

423 diff_decrease_style = config.get('diff_decrease_style', UNSET) (empty)

424 if isinstance(diff_decrease_style, Unset): (empty)

425 diff_decrease_style = 'red' if kind == 'score' else 'green' (empty)

426 

427 return _NumberRenderer( (empty)

428 value_formatter=value_formatter, 

429 diff_formatter=diff_formatter, 

430 diff_rtol=diff_rtol, 

431 diff_atol=diff_atol, 

432 diff_increase_style=diff_increase_style, 

433 diff_decrease_style=diff_decrease_style, 

434 ) 

435 

436 def _get_value_str(self, value: float | int | None) -> str: (empty)

437 if value is None: (empty)

438 return MISSING_VALUE_STR (empty)

439 if isinstance(self.value_formatter, str): (empty)

440 return self.value_formatter.format(value) (empty)

441 else: 

442 return self.value_formatter(value) (empty)

443 

444 def _get_diff_str(self, old: float | int | None, new: float | int | None) -> str | None: (empty)

445 if old is None or new is None: # pragma: no cover (empty)

446 return None 

447 if isinstance(self.diff_formatter, str): # pragma: no cover (empty)

448 return self.diff_formatter.format(new - old) 

449 elif self.diff_formatter is None: # pragma: no cover (empty)

450 return None 

451 else: 

452 return self.diff_formatter(old, new) (empty)

453 

454 def _get_diff_style(self, old: float | int | None, new: float | int | None) -> str | None: (empty)

455 # 1 means new is higher, -1 means new is lower, 0 means no change 

456 if old is None or new is None: (empty)

457 return None (empty)

458 

459 diff = new - old (empty)

460 if abs(diff) < self.diff_atol + self.diff_rtol * abs(old): # pragma: no cover (empty)

461 return None 

462 return self.diff_increase_style if diff > 0 else self.diff_decrease_style (empty)

463 

464 

465T_contra = TypeVar('T_contra', contravariant=True) (empty)

466 

467 

468class _AbstractRenderer(Protocol[T_contra]): (empty)

469 def render_value(self, name: str | None, v: T_contra) -> str: ... 469 ↛ exitline 469 didn't return from function 'render_value' because (empty)

470 

471 def render_diff(self, name: str | None, old: T_contra | None, new: T_contra | None) -> str: ... 471 ↛ exitline 471 didn't return from function 'render_diff' because (empty)

472 

473 

474_DEFAULT_NUMBER_CONFIG = RenderNumberConfig() (empty)

475_DEFAULT_VALUE_CONFIG = RenderValueConfig() (empty)

476_DEFAULT_DURATION_CONFIG = RenderNumberConfig( (empty)

477 value_formatter=default_render_duration, 

478 diff_formatter=default_render_duration_diff, 

479 diff_atol=1e-6, # one microsecond 

480 diff_rtol=0.1, 

481 diff_increase_style='red', 

482 diff_decrease_style='green', 

483) 

484 

485 

486T = TypeVar('T') (empty)

487 

488 

489@dataclass (empty)

490class ReportCaseRenderer: (empty)

491 include_input: bool (empty)

492 include_output: bool (empty)

493 include_scores: bool (empty)

494 include_labels: bool (empty)

495 include_metrics: bool (empty)

496 include_assertions: bool (empty)

497 include_total_duration: bool (empty)

498 

499 input_renderer: _ValueRenderer (empty)

500 output_renderer: _ValueRenderer (empty)

501 score_renderers: dict[str, _NumberRenderer] (empty)

502 label_renderers: dict[str, _ValueRenderer] (empty)

503 metric_renderers: dict[str, _NumberRenderer] (empty)

504 duration_renderer: _NumberRenderer (empty)

505 

506 def build_base_table(self, title: str) -> Table: (empty)

507 """Build and return a Rich Table for the diff output.""" 

508 table = Table(title=title, show_lines=True) (empty)

509 table.add_column('Case ID', style='bold') (empty)

510 if self.include_input: (empty)

511 table.add_column('Inputs', overflow='fold') (empty)

512 if self.include_output: (empty)

513 table.add_column('Outputs', overflow='fold') (empty)

514 if self.include_scores: (empty)

515 table.add_column('Scores', overflow='fold') (empty)

516 if self.include_labels: (empty)

517 table.add_column('Labels', overflow='fold') (empty)

518 if self.include_metrics: (empty)

519 table.add_column('Metrics', overflow='fold') (empty)

520 if self.include_assertions: (empty)

521 table.add_column('Assertions', overflow='fold') (empty)

522 table.add_column('Durations' if self.include_total_duration else 'Duration', justify='right') (empty)

523 return table (empty)

524 

525 def build_row(self, case: ReportCase) -> list[str]: (empty)

526 """Build a table row for a single case.""" 

527 row = [case.name] (empty)

528 

529 if self.include_input: (empty)

530 row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR) (empty)

531 

532 if self.include_output: (empty)

533 row.append(self.output_renderer.render_value(None, case.output) or EMPTY_CELL_STR) (empty)

534 

535 if self.include_scores: (empty)

536 row.append(self._render_dict({k: v.value for k, v in case.scores.items()}, self.score_renderers)) (empty)

537 

538 if self.include_labels: (empty)

539 row.append(self._render_dict({k: v.value for k, v in case.labels.items()}, self.label_renderers)) (empty)

540 

541 if self.include_metrics: (empty)

542 row.append(self._render_dict(case.metrics, self.metric_renderers)) (empty)

543 

544 if self.include_assertions: (empty)

545 row.append(self._render_assertions(list(case.assertions.values()))) (empty)

546 

547 row.append(self._render_durations(case)) (empty)

548 return row (empty)

549 

550 def build_aggregate_row(self, aggregate: ReportCaseAggregate) -> list[str]: (empty)

551 """Build a table row for an aggregated case.""" 

552 row = [f'[b i]{aggregate.name}[/]'] (empty)

553 

554 if self.include_input: (empty)

555 row.append(EMPTY_AGGREGATE_CELL_STR) (empty)

556 

557 if self.include_output: (empty)

558 row.append(EMPTY_AGGREGATE_CELL_STR) (empty)

559 

560 if self.include_scores: (empty)

561 row.append(self._render_dict(aggregate.scores, self.score_renderers)) (empty)

562 

563 if self.include_labels: (empty)

564 row.append(self._render_dict(aggregate.labels, self.label_renderers)) (empty)

565 

566 if self.include_metrics: (empty)

567 row.append(self._render_dict(aggregate.metrics, self.metric_renderers)) (empty)

568 

569 if self.include_assertions: (empty)

570 row.append(self._render_aggregate_assertions(aggregate.assertions)) (empty)

571 

572 row.append(self._render_durations(aggregate)) (empty)

573 return row (empty)

574 

575 def build_diff_row( (empty)

576 self, 

577 new_case: ReportCase, 

578 baseline: ReportCase, 

579 ) -> list[str]: 

580 """Build a table row for a given case ID.""" 

581 assert baseline.name == new_case.name, 'This should only be called for matching case IDs' (empty)

582 row = [baseline.name] (empty)

583 

584 if self.include_input: 584 ↛ 588line 584 didn't jump to line 588 because the condition on line 584 was always true(empty)

585 input_diff = self.input_renderer.render_diff(None, baseline.inputs, new_case.inputs) or EMPTY_CELL_STR (empty)

586 row.append(input_diff) (empty)

587 

588 if self.include_output: 588 ↛ 592line 588 didn't jump to line 592 because the condition on line 588 was always true(empty)

589 output_diff = self.output_renderer.render_diff(None, baseline.output, new_case.output) or EMPTY_CELL_STR (empty)

590 row.append(output_diff) (empty)

591 

592 if self.include_scores: 592 ↛ 600line 592 didn't jump to line 600 because the condition on line 592 was always true(empty)

593 scores_diff = self._render_dicts_diff( (empty)

594 {k: v.value for k, v in baseline.scores.items()}, 

595 {k: v.value for k, v in new_case.scores.items()}, 

596 self.score_renderers, 

597 ) 

598 row.append(scores_diff) (empty)

599 

600 if self.include_labels: 600 ↛ 604line 600 didn't jump to line 604 because the condition on line 600 was always true(empty)

601 labels_diff = self._render_dicts_diff(baseline.labels, new_case.labels, self.label_renderers) (empty)

602 row.append(labels_diff) (empty)

603 

604 if self.include_metrics: 604 ↛ 608line 604 didn't jump to line 608 because the condition on line 604 was always true(empty)

605 metrics_diff = self._render_dicts_diff(baseline.metrics, new_case.metrics, self.metric_renderers) (empty)

606 row.append(metrics_diff) (empty)

607 

608 if self.include_assertions: 608 ↛ 614line 608 didn't jump to line 614 because the condition on line 608 was always true(empty)

609 assertions_diff = self._render_assertions_diff( (empty)

610 list(baseline.assertions.values()), list(new_case.assertions.values()) 

611 ) 

612 row.append(assertions_diff) (empty)

613 

614 row.append(self._render_durations_diff(baseline, new_case)) (empty)

615 

616 return row (empty)

617 

618 def build_diff_aggregate_row( (empty)

619 self, 

620 new: ReportCaseAggregate, 

621 baseline: ReportCaseAggregate, 

622 ) -> list[str]: 

623 """Build a table row for a given case ID.""" 

624 assert baseline.name == new.name, 'This should only be called for aggregates with matching names' (empty)

625 row = [f'[b i]{baseline.name}[/]'] (empty)

626 

627 if self.include_input: 627 ↛ 630line 627 didn't jump to line 630 because the condition on line 627 was always true(empty)

628 row.append(EMPTY_AGGREGATE_CELL_STR) (empty)

629 

630 if self.include_output: 630 ↛ 633line 630 didn't jump to line 633 because the condition on line 630 was always true(empty)

631 row.append(EMPTY_AGGREGATE_CELL_STR) (empty)

632 

633 if self.include_scores: 633 ↛ 637line 633 didn't jump to line 637 because the condition on line 633 was always true(empty)

634 scores_diff = self._render_dicts_diff(baseline.scores, new.scores, self.score_renderers) (empty)

635 row.append(scores_diff) (empty)

636 

637 if self.include_labels: 637 ↛ 641line 637 didn't jump to line 641 because the condition on line 637 was always true(empty)

638 labels_diff = self._render_dicts_diff(baseline.labels, new.labels, self.label_renderers) (empty)

639 row.append(labels_diff) (empty)

640 

641 if self.include_metrics: 641 ↛ 645line 641 didn't jump to line 645 because the condition on line 641 was always true(empty)

642 metrics_diff = self._render_dicts_diff(baseline.metrics, new.metrics, self.metric_renderers) (empty)

643 row.append(metrics_diff) (empty)

644 

645 if self.include_assertions: 645 ↛ 649line 645 didn't jump to line 649 because the condition on line 645 was always true(empty)

646 assertions_diff = self._render_aggregate_assertions_diff(baseline.assertions, new.assertions) (empty)

647 row.append(assertions_diff) (empty)

648 

649 row.append(self._render_durations_diff(baseline, new)) (empty)

650 

651 return row (empty)

652 

653 def _render_durations(self, case: ReportCase | ReportCaseAggregate) -> str: (empty)

654 """Build the diff string for a duration value.""" 

655 case_durations: dict[str, float] = {'task': case.task_duration} (empty)

656 if self.include_total_duration: (empty)

657 case_durations['total'] = case.total_duration (empty)

658 return self._render_dict( (empty)

659 case_durations, 

660 {'task': self.duration_renderer, 'total': self.duration_renderer}, 

661 include_names=self.include_total_duration, 

662 ) 

663 

664 def _render_durations_diff( (empty)

665 self, 

666 base_case: ReportCase | ReportCaseAggregate, 

667 new_case: ReportCase | ReportCaseAggregate, 

668 ) -> str: 

669 """Build the diff string for a duration value.""" 

670 base_case_durations: dict[str, float] = {'task': base_case.task_duration} (empty)

671 new_case_durations: dict[str, float] = {'task': new_case.task_duration} (empty)

672 if self.include_total_duration: 672 ↛ 675line 672 didn't jump to line 675 because the condition on line 672 was always true(empty)

673 base_case_durations['total'] = base_case.total_duration (empty)

674 new_case_durations['total'] = new_case.total_duration (empty)

675 return self._render_dicts_diff( (empty)

676 base_case_durations, 

677 new_case_durations, 

678 {'task': self.duration_renderer, 'total': self.duration_renderer}, 

679 include_names=self.include_total_duration, 

680 ) 

681 

682 @staticmethod (empty)

683 def _render_dicts_diff( (empty)

684 baseline_dict: dict[str, T], 

685 new_dict: dict[str, T], 

686 renderers: Mapping[str, _AbstractRenderer[T]], 

687 *, 

688 include_names: bool = True, 

689 ) -> str: 

690 keys: set[str] = set() (empty)

691 keys.update(baseline_dict.keys()) (empty)

692 keys.update(new_dict.keys()) (empty)

693 diff_lines: list[str] = [] (empty)

694 for key in sorted(keys): (empty)

695 old_val = baseline_dict.get(key) (empty)

696 new_val = new_dict.get(key) (empty)

697 rendered = renderers[key].render_diff(key if include_names else None, old_val, new_val) (empty)

698 diff_lines.append(rendered) (empty)

699 return '\n'.join(diff_lines) if diff_lines else EMPTY_CELL_STR (empty)

700 

701 @staticmethod (empty)

702 def _render_dict( (empty)

703 case_dict: dict[str, T], 

704 renderers: Mapping[str, _AbstractRenderer[T]], 

705 *, 

706 include_names: bool = True, 

707 ) -> str: 

708 diff_lines: list[str] = [] (empty)

709 for key, val in case_dict.items(): (empty)

710 rendered = renderers[key].render_value(key if include_names else None, val) (empty)

711 diff_lines.append(rendered) (empty)

712 return '\n'.join(diff_lines) if diff_lines else EMPTY_CELL_STR (empty)

713 

714 @staticmethod (empty)

715 def _render_assertions( (empty)

716 assertions: list[EvaluationResult[bool]], 

717 ) -> str: 

718 if not assertions: (empty)