Coverage for tests/test_build_docs.py: 99.21%

1"""Tests for the docs generator (prices.build_docs). 

2 

3Covers the price-shape normalizer, modality detection (including the curated overrides), MDX 

4rendering, the generated navigation, and a full build against the real prices/data.json so the 

5committed pages can never silently drift from the catalog they claim to describe. 

6""" 

7 

8from __future__ import annotations (empty)

9 

10import json (empty)

11import re (empty)

12import shutil (empty)

13from pathlib import Path (empty)

14from typing import Any, cast (empty)

15 

16import pytest (empty)

17 

18from prices.build_docs import ( (empty)

19 CATEGORIES, 

20 DATA_JSON, 

21 DOCS_DIR, 

22 INDEX_PAGE, 

23 MODALITY_OVERRIDES, 

24 Column, 

25 Comparison, 

26 ModelRow, 

27 ProviderEntry, 

28 _resolve_direct, 

29 active_columns, 

30 base_prices, 

31 build_catalog, 

32 build_comparison, 

33 build_docs, 

34 build_navigation, 

35 category_tab, 

36 cell, 

37 detect_modality, 

38 fmt_pct, 

39 fmt_usd, 

40 inject_blocks, 

41 mdx_table, 

42 missing_alias_targets, 

43 provenance_source, 

44 render_index_page, 

45 render_provider_page, 

46 slug, 

47 unknown_prefix_providers, 

48) 

49 

50 

51def _real_data() -> list[dict[str, Any]]: (empty)

52 return json.loads(DATA_JSON.read_text()) (empty)

53 

54 

55# ---- price shape normalizer -------------------------------------------------- 

56 

57 

58def test_base_prices_flat(): (empty)

59 flat, tiered, daily = base_prices({'input_mtok': 2.0, 'output_mtok': 8.0}) (empty)

60 assert flat == {'input_mtok': 2.0, 'output_mtok': 8.0} (empty)

61 assert tiered is False (empty)

62 assert daily is False (empty)

63 

64 

65def test_base_prices_tiered_reads_base(): (empty)

66 flat, tiered, daily = base_prices({'input_mtok': {'base': 3, 'tiers': [{'start': 200000, 'price': 6}]}}) (empty)

67 assert flat == {'input_mtok': 3.0} (empty)

68 assert tiered is True (empty)

69 assert daily is False (empty)

70 

71 

72def test_base_prices_conditional_uses_base_block_and_flags_daily(): (empty)

73 prices = [ (empty)

74 {'prices': {'input_mtok': 0.135, 'output_mtok': 0.55}}, 

75 { 

76 'constraint': {'start_time': '00:30:00Z', 'end_time': '16:30:00Z'}, 

77 'prices': {'input_mtok': 0.27, 'output_mtok': 1.1}, 

78 }, 

79 ] 

80 flat, tiered, daily = base_prices(prices) (empty)

81 assert flat == {'input_mtok': 0.135, 'output_mtok': 0.55} (empty)

82 assert tiered is False (empty)

83 assert daily is True (empty)

84 

85 

86def test_base_prices_skips_voice_multipliers(): (empty)

87 flat, _, _ = base_prices({'input_kchars': 0.04, 'voice_multipliers': {'default': 1.0, 'premium': 1.5}}) (empty)

88 assert flat == {'input_kchars': 0.04} (empty)

89 

90 

91def test_base_prices_empty(): (empty)

92 flat, tiered, daily = base_prices({}) (empty)

93 assert flat == {} (empty)

94 assert tiered is False (empty)

95 assert daily is False (empty)

96 

97 

98# ---- modality detection ------------------------------------------------------ 

99 

100 

101def test_detect_modality_by_priced_field(): (empty)

102 assert detect_modality({'input_kchars': 0.04}) == 'tts' (empty)

103 assert detect_modality({'input_audio_kseconds': 0.08}) == 'stt' (empty)

104 assert detect_modality({'input_mtok': 2.0, 'output_mtok': 8.0}) == 'llm' (empty)

105 assert detect_modality({}) is None (empty)

106 

107 

108def test_detect_modality_s2s_needs_both_audio_directions(): (empty)

109 # Bidirectional audio pricing is what makes a model speech-to-speech. 

110 assert detect_modality({'input_audio_mtok': 32.0, 'output_audio_mtok': 64.0}) == 's2s' (empty)

111 # Audio in only is a multimodal chat model, not a realtime voice model. 

112 assert detect_modality({'input_mtok': 5.0, 'input_audio_mtok': 40.0}) == 'llm' (empty)

113 # Audio out only, with no override, is likewise not s2s. 

114 assert detect_modality({'input_mtok': 0.6, 'output_audio_mtok': 12.0}) == 'llm' (empty)

115 

116 

117def test_detect_modality_overrides_win_over_the_field_rule(): (empty)

118 # gpt-4o-transcribe bills audio per token, so the field rule alone would file it under llm and 

119 # it would disappear from the STT tab. 

120 audio_tokens = {'input_mtok': 2.5, 'output_mtok': 10.0, 'input_audio_mtok': 6.0} (empty)

121 assert detect_modality(audio_tokens) == 'llm' (empty)

122 assert detect_modality(audio_tokens, ('openai', 'gpt-4o-transcribe')) == 'stt' (empty)

123 assert detect_modality({'output_audio_mtok': 12.0}, ('openai', 'gpt-4o-mini-tts')) == 'tts' (empty)

124 # A VAD model is priced exactly like STT and is only distinguishable by the override. 

125 assert detect_modality({'input_audio_kseconds': 0.025}) == 'stt' (empty)

126 assert detect_modality({'input_audio_kseconds': 0.025}, ('ai_coustics', 'quail-vad-2.0-xxs-16khz')) == 'vad' (empty)

127 

128 

129def test_modality_overrides_are_exactly_these(): (empty)

130 # The override map is the one place a model's category is decided by hand rather than by its 

131 # priced fields. Pinning it means a new entry has to be a deliberate, reviewed change. 

132 assert MODALITY_OVERRIDES == { (empty)

133 ('openai', 'gpt-4o-transcribe'): 'stt', 

134 ('openai', 'gpt-4o-mini-transcribe'): 'stt', 

135 ('openai', 'gpt-4o-mini-tts'): 'tts', 

136 ('soniox', 'stt-async'): 'stt', 

137 ('soniox', 'stt-rt'): 'stt', 

138 ('soniox', 'tts-rt'): 'tts', 

139 ('ai_coustics', 'quail-vad-2.0-xxs-16khz'): 'vad', 

140 ('ai_coustics', 'quail-vf-vad-2.0-s-16khz'): 'vad', 

141 } 

142 

143 

144def test_every_override_target_exists_in_the_real_catalog(): (empty)

145 # An override naming a renamed or removed model would silently stop applying, dropping the model 

146 # back onto the wrong tab with no error anywhere. 

147 present: set[tuple[str, str]] = set() (empty)

148 for provider in _real_data(): (empty)

149 models: list[dict[str, Any]] = provider.get('models') or [] (empty)

150 for model in models: (empty)

151 present.add((str(provider.get('id')), str(model.get('id')))) (empty)

152 assert set(MODALITY_OVERRIDES) <= present (empty)

153 

154 

155# ---- catalog ----------------------------------------------------------------- 

156 

157 

158def test_build_catalog_excludes_unpriced_and_deprecated(): (empty)

159 data: list[dict[str, Any]] = [ (empty)

160 { 

161 'id': 'acme', 

162 'name': 'Acme', 

163 'models': [ 

164 {'id': 'acme-1', 'prices': {'input_mtok': 2.0, 'output_mtok': 8.0}, 'context_window': 1000}, 

165 {'id': 'acme-free', 'prices': {}}, # unpriced: excluded 

166 {'id': 'acme-old', 'prices': {'input_mtok': 1.0}, 'deprecated': True}, # deprecated: excluded 

167 ], 

168 } 

169 ] 

170 catalog = build_catalog(data) (empty)

171 assert [p['id'] for p in catalog['llm']] == ['acme'] (empty)

172 assert [m['id'] for m in catalog['llm'][0]['models']] == ['acme-1'] (empty)

173 

174 

175def test_build_catalog_sorts_models_by_id(): (empty)

176 data: list[dict[str, Any]] = [ (empty)

177 { 

178 'id': 'acme', 

179 'name': 'Acme', 

180 'models': [ 

181 {'id': 'z-model', 'prices': {'input_mtok': 1.0}}, 

182 {'id': 'a-model', 'prices': {'input_mtok': 2.0}}, 

183 ], 

184 } 

185 ] 

186 assert [m['id'] for m in build_catalog(data)['llm'][0]['models']] == ['a-model', 'z-model'] (empty)

187 

188 

189def test_build_catalog_provider_can_span_categories(): (empty)

190 data: list[dict[str, Any]] = [ (empty)

191 { 

192 'id': 'multi', 

193 'name': 'Multi', 

194 'models': [ 

195 {'id': 'multi-llm', 'prices': {'input_mtok': 1.0, 'output_mtok': 2.0}}, 

196 {'id': 'multi-tts', 'prices': {'input_kchars': 0.02}}, 

197 ], 

198 } 

199 ] 

200 catalog = build_catalog(data) (empty)

201 assert [p['id'] for p in catalog['llm']] == ['multi'] (empty)

202 assert [p['id'] for p in catalog['tts']] == ['multi'] (empty)

203 assert catalog['stt'] == [] (empty)

204 

205 

206def test_stt_derives_per_minute(): (empty)

207 data: list[dict[str, Any]] = [ (empty)

208 {'id': 'd', 'name': 'D', 'models': [{'id': 'nova', 'prices': {'input_audio_kseconds': 0.08}}]} 

209 ] 

210 row = build_catalog(data)['stt'][0]['models'][0] (empty)

211 assert row['values']['per_min'] == 0.08 * 60 / 1000 # 0.0048 (empty)

212 

213 

214def test_tts_derives_per_million_chars(): (empty)

215 data: list[dict[str, Any]] = [ (empty)

216 {'id': 'e', 'name': 'E', 'models': [{'id': 'flash', 'prices': {'input_kchars': 0.05}}]} 

217 ] 

218 row = build_catalog(data)['tts'][0]['models'][0] (empty)

219 assert row['values']['per_mchars'] == 50.0 (empty)

220 

221 

222def test_voice_rows_carry_source_and_verified_date(): (empty)

223 data: list[dict[str, Any]] = [ (empty)

224 { 

225 'id': 'd', 

226 'name': 'D', 

227 'models': [ 

228 { 

229 'id': 'a-verified', 

230 'prices': {'input_audio_kseconds': 0.08}, 

231 'provenance': {'last_verified': '2026-07-01'}, 

232 }, 

233 {'id': 'b-imported', 'prices': {'input_audio_kseconds': 0.09}, 'provenance': {'source': 'imported'}}, 

234 {'id': 'c-seed', 'prices': {'input_audio_kseconds': 0.10}}, 

235 ], 

236 } 

237 ] 

238 rows = {r['id']: r for r in build_catalog(data)['stt'][0]['models']} (empty)

239 assert (rows['a-verified']['status'], rows['a-verified']['verified']) == ('verified', '2026-07-01') (empty)

240 assert (rows['b-imported']['status'], rows['b-imported']['verified']) == ('imported', None) (empty)

241 assert (rows['c-seed']['status'], rows['c-seed']['verified']) == ('seed', None) (empty)

242 

243 

244def test_llm_rows_carry_no_source(): (empty)

245 data: list[dict[str, Any]] = [ (empty)

246 {'id': 'o', 'name': 'O', 'models': [{'id': 'gpt', 'prices': {'input_mtok': 2.5, 'output_mtok': 10.0}}]} 

247 ] 

248 row = build_catalog(data)['llm'][0]['models'][0] (empty)

249 assert row['status'] is None (empty)

250 assert row['verified'] is None (empty)

251 

252 

253def test_provenance_source_never_computes_staleness(): (empty)

254 # The committed pages must be a pure function of data.json. If this ever returned a 

255 # wall-clock-dependent value, every page would rewrite itself the day a model went stale and 

256 # turn an unrelated pull request red. 

257 assert provenance_source({'last_verified': '2020-01-01'}) == ('verified', '2020-01-01') (empty)

258 assert provenance_source(None) == ('seed', None) (empty)

259 assert provenance_source({'source': 'imported'}) == ('imported', None) (empty)

260 

261 

262def test_api_backed_models_carry_an_api_marker(): (empty)

263 # The split the docs expose: a rate a machine can re-read versus one a person read off a page. 

264 data: list[dict[str, Any]] = [ (empty)

265 { 

266 'id': 'acme', 

267 'name': 'Acme', 

268 'models': [ 

269 {'id': 'from-api', 'prices': {'input_mtok': 1.0}, 'provenance': {'api_backed': True}}, 

270 {'id': 'from-page', 'prices': {'input_mtok': 2.0}, 'provenance': {'last_verified': '2026-07-01'}}, 

271 {'id': 'no-provenance', 'prices': {'input_mtok': 3.0}}, 

272 ], 

273 } 

274 ] 

275 rows = {row['id']: row for row in build_catalog(data)['llm'][0]['models']} (empty)

276 assert rows['from-api']['markers'] == ['api'] (empty)

277 assert rows['from-page']['markers'] == [] (empty)

278 assert rows['no-provenance']['markers'] == [] (empty)

279 

280 

281def test_the_api_marker_is_explained_only_when_it_appears(): (empty)

282 data: list[dict[str, Any]] = [ (empty)

283 { 

284 'id': 'a', 

285 'name': 'A', 

286 'models': [{'id': 'm', 'prices': {'input_mtok': 1.0}, 'provenance': {'api_backed': True}}], 

287 } 

288 ] 

289 catalog = build_catalog(data) (empty)

290 page = render_provider_page('llm', catalog['llm'][0]) (empty)

291 assert '^api' in page (empty)

292 assert 'machine-readable endpoint' in page (empty)

293 

294 

295def test_real_catalog_has_at_least_one_api_backed_model(): (empty)

296 # Telnyx and Cerebras both publish a machine-readable pricing endpoint, so the coverage line on 

297 # the overview is not vacuous. Pinned as a set so adding a provider here is a deliberate change: 

298 # the marker asserts a rate is re-read automatically, which is only true if an importer reads it. 

299 catalog = build_catalog(_real_data()) (empty)

300 backed = [ (empty)

301 (entry['id'], row['id']) 

302 for category in CATEGORIES 

303 for entry in catalog[category] 

304 for row in entry['models'] 

305 if 'api' in row['markers'] 

306 ] 

307 assert backed (empty)

308 assert {provider for provider, _ in backed} == {'telnyx', 'cerebras', 'livekit', 'livekit-scale'} (empty)

309 # The count is pinned too, not just the provider set: without it this passes even if every 

310 # marker but one per provider disappeared, and the coverage line on the overview would quietly 

311 # collapse from 15 to 2 with nothing failing. 

312 assert len(backed) == 158, f'expected 158 api-backed models, found {len(backed)}: {sorted(backed)}' (empty)

313 

314 

315def test_markers_flag_tiered_daily_and_voices(): (empty)

316 data: list[dict[str, Any]] = [ (empty)

317 { 

318 'id': 'm', 

319 'name': 'M', 

320 'models': [ 

321 {'id': 'tiered-one', 'prices': {'input_mtok': {'base': 1.0, 'tiers': [{'start': 1, 'price': 2}]}}}, 

322 {'id': 'voiced', 'prices': {'input_kchars': 0.04, 'voice_multipliers': {'default': 1.0, 'p': 1.5}}}, 

323 ], 

324 } 

325 ] 

326 catalog = build_catalog(data) (empty)

327 assert catalog['llm'][0]['models'][0]['markers'] == ['tiered'] (empty)

328 assert catalog['tts'][0]['models'][0]['markers'] == ['voices'] (empty)

329 

330 

331# ---- real catalog ------------------------------------------------------------ 

332 

333 

334def test_real_catalog_classifies_every_priced_model(): (empty)

335 # Nothing priced may fall through the classifier: a model with no category is a model that 

336 # silently vanishes from the docs site. 

337 data = _real_data() (empty)

338 unclassified: list[str] = [] (empty)

339 for provider in data: (empty)

340 models: list[dict[str, Any]] = provider.get('models') or [] (empty)

341 for model in models: (empty)

342 if model.get('deprecated') is True: (empty)

343 continue (empty)

344 flat, _, _ = base_prices(model.get('prices')) (empty)

345 if flat and detect_modality(flat, (str(provider['id']), str(model['id']))) is None: 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true(empty)

346 unclassified.append(f'{provider["id"]}:{model["id"]}') 

347 assert unclassified == [] (empty)

348 

349 

350def test_real_catalog_has_every_category_populated(): (empty)

351 catalog = build_catalog(_real_data()) (empty)

352 for category in CATEGORIES: (empty)

353 assert catalog[category], f'{category} has no providers' (empty)

354 assert 'anthropic' in {p['id'] for p in catalog['llm']} (empty)

355 assert 'deepgram' in {p['id'] for p in catalog['stt']} (empty)

356 assert 'cartesia' in {p['id'] for p in catalog['tts']} (empty)

357 

358 

359def test_s2s_is_exactly_the_bidirectional_realtime_models(): (empty)

360 # Pinned on purpose: a new realtime model appearing here should be a visible, reviewed change, 

361 # and a model dropping out means the S2S tab quietly lost a row. 

362 catalog = build_catalog(_real_data()) (empty)

363 found = {(entry['id'], model['id']) for entry in catalog['s2s'] for model in entry['models']} (empty)

364 assert found == { (empty)

365 ('aws', 'amazon.nova-sonic-v1:0'), 

366 ('boson', 'higgs-realtime'), 

367 ('openrouter', 'openai/gpt-audio'), 

368 ('google', 'gemini-live-2.5-flash-preview'), 

369 ('openai', 'gpt-4o-mini-realtime-preview'), 

370 ('openai', 'gpt-4o-realtime-preview'), 

371 ('openai', 'gpt-realtime'), 

372 ('openai', 'gpt-realtime-mini'), 

373 } 

374 

375 

376def test_vad_is_the_ai_coustics_models(): (empty)

377 catalog = build_catalog(_real_data()) (empty)

378 assert [entry['id'] for entry in catalog['vad']] == ['ai_coustics'] (empty)

379 assert [m['id'] for m in catalog['vad'][0]['models']] == [ (empty)

380 'quail-vad-2.0-xxs-16khz', 

381 'quail-vf-vad-2.0-s-16khz', 

382 ] 

383 

384 

385def test_ai_coustics_vad_rate_matches_the_published_per_minute_rate(): (empty)

386 # $0.0015/minute at the Startup tier, stored as $ per 1,000 seconds. 

387 catalog = build_catalog(_real_data()) (empty)

388 for model in catalog['vad'][0]['models']: (empty)

389 per_min = model['values']['per_min'] (empty)

390 assert per_min is not None (empty)

391 assert round(per_min, 8) == 0.0015 (empty)

392 

393 

394def test_no_deprecated_model_leaks_into_the_catalog(): (empty)

395 # Model ids are not globally unique (OpenRouter mirrors other providers' ids), so this is 

396 # checked per (provider id, model id), not globally. 

397 data = _real_data() (empty)

398 deprecated: set[tuple[str, str]] = set() (empty)

399 for provider in data: (empty)

400 models: list[dict[str, Any]] = provider.get('models') or [] (empty)

401 for model_data in models: (empty)

402 if model_data.get('deprecated') is True and provider['id'] not in ('livekit', 'livekit-scale'): (empty)

403 deprecated.add((str(provider['id']), str(model_data['id']))) (empty)

404 assert deprecated # data.json has at least one deprecated model today (empty)

405 catalog = build_catalog(data) (empty)

406 for entries in catalog.values(): (empty)

407 for entry in entries: (empty)

408 for model in entry['models']: (empty)

409 assert (entry['id'], model['id']) not in deprecated (empty)

410 

411 

412# ---- MDX rendering ----------------------------------------------------------- 

413 

414 

415def test_cell_escapes_mdx_and_table_syntax(): (empty)

416 assert cell('a|b') == r'a\|b' (empty)

417 assert cell('<script>') == '&lt;script>' (empty)

418 assert cell('{value}') == '&#123;value}' (empty)

419 

420 

421def test_fmt_usd_strips_trailing_zeros_and_keeps_small_rates(): (empty)

422 assert fmt_usd(50.0) == '$50' (empty)

423 assert fmt_usd(1.25) == '$1.25' (empty)

424 assert fmt_usd(0.0015) == '$0.0015' (empty)

425 assert fmt_usd(0) == '$0' (empty)

426 assert fmt_usd(None) == '-' (empty)

427 

428 

429def test_fmt_pct_never_shows_a_misleading_plus_zero(): (empty)

430 assert fmt_pct(200.0) == '+200.0%' (empty)

431 assert fmt_pct(-2.0) == '-2.0%' (empty)

432 assert fmt_pct(0.0) == 'same' (empty)

433 assert fmt_pct(None) == 'LiveKit only' (empty)

434 

435 

436def test_mdx_table_aligns_text_then_numbers(): (empty)

437 out = mdx_table(['Model', 'Name', 'Rate'], [['`a`', 'A', '$1']]) (empty)

438 assert out.splitlines()[1] == '| --- | --- | ---: |' (empty)

439 wide = mdx_table(['Provider', 'Model', 'Name', 'Rate'], [['P', '`a`', 'A', '$1']], text_cols=3) (empty)

440 assert wide.splitlines()[1] == '| --- | --- | --- | ---: |' (empty)

441 

442 

443def test_mdx_table_empty_says_so_rather_than_rendering_a_headless_table(): (empty)

444 assert mdx_table(['Model'], []) == '_No models priced yet._' (empty)

445 

446 

447def _row(**values: float | int | None) -> ModelRow: (empty)

448 return {'id': 'm', 'name': 'M', 'values': dict(values), 'status': None, 'verified': None, 'markers': []} (empty)

449 

450 

451def test_active_columns_drops_columns_that_are_none_for_every_row(): (empty)

452 columns = (Column('A', 'a', 'usd'), Column('B', 'b', 'usd')) (empty)

453 rows = [_row(a=1.0, b=None), _row(a=2.0, b=None)] (empty)

454 assert [c.header for c in active_columns(rows, columns)] == ['A'] (empty)

455 rows = [_row(a=1.0, b=None), _row(a=2.0, b=3.0)] (empty)

456 assert [c.header for c in active_columns(rows, columns)] == ['A', 'B'] (empty)

457 

458 

459def test_slug_makes_provider_ids_url_safe(): (empty)

460 assert slug('huggingface_novita') == 'huggingface-novita' (empty)

461 assert slug('x-ai') == 'x-ai' (empty)

462 

463 

464def test_render_provider_page_has_frontmatter_and_the_do_not_edit_notice(): (empty)

465 catalog = build_catalog(_real_data()) (empty)

466 entry = next(e for e in catalog['stt'] if e['id'] == 'deepgram') (empty)

467 page = render_provider_page('stt', entry) (empty)

468 assert page.startswith('---\n') (empty)

469 assert 'sidebarTitle: "Deepgram"' in page (empty)

470 assert 'Do not edit by hand' in page (empty)

471 assert 'prices/providers/deepgram.yml' in page (empty)

472 assert '$ / min' in page (empty)

473 

474 

475def test_frontmatter_escapes_quotes_in_contributor_supplied_values(): (empty)

476 # A provider `name` comes straight from YAML someone else wrote. One unescaped quote would make 

477 # the frontmatter invalid and fail the whole docs build, not just that page. 

478 entry: ProviderEntry = { (empty)

479 'id': 'q', 

480 'name': 'A "quoted" vendor', 

481 'models': [_row(per_min=0.01)], 

482 'pricing_tier': None, 

483 } 

484 page = render_provider_page('stt', entry) (empty)

485 assert 'sidebarTitle: "A \\"quoted\\" vendor"' in page (empty)

486 

487 

488def test_page_descriptions_only_claim_what_the_page_renders(): (empty)

489 # An LLM page shows no per-model source, so its description must not promise one. 

490 catalog = build_catalog(_real_data()) (empty)

491 llm = render_provider_page('llm', next(e for e in catalog['llm'] if e['id'] == 'anthropic')) (empty)

492 assert 'where it came from' not in llm (empty)

493 stt = render_provider_page('stt', next(e for e in catalog['stt'] if e['id'] == 'deepgram')) (empty)

494 assert 'where it came from' in stt (empty)

495 # The LLM index summarises by vendor, so it is not "cheapest first" either. 

496 index = render_index_page('llm', catalog['llm'], build_comparison(_real_data())['llm']) (empty)

497 assert 'cheapest first' not in index (empty)

498 

499 

500def test_llm_pages_carry_no_source_column(): (empty)

501 # LLM rates are cross-checked against aggregators rather than per-model vendor pages, so a 

502 # per-model source column there would read as low confidence when it means nothing of the sort. 

503 catalog = build_catalog(_real_data()) (empty)

504 entry = next(e for e in catalog['llm'] if e['id'] == 'anthropic') (empty)

505 page = render_provider_page('llm', entry) (empty)

506 assert 'Last verified' not in page (empty)

507 assert '<Badge' not in page (empty)

508 

509 

510def test_voice_pages_carry_a_source_badge(): (empty)

511 catalog = build_catalog(_real_data()) (empty)

512 entry = next(e for e in catalog['tts'] if e['id'] == 'elevenlabs') (empty)

513 page = render_provider_page('tts', entry) (empty)

514 assert 'Last verified' in page (empty)

515 assert '<Badge color="green" size="sm">verified</Badge>' in page (empty)

516 

517 

518def test_token_priced_voice_models_render_in_their_own_table(): (empty)

519 # TTS rather than STT: every STT model in the catalog now carries a per-second rate, so 

520 # gpt-4o-mini-tts is the last remaining voice model priced only in audio tokens. OpenAI 

521 # publishes no per-character figure for it, not even an estimate, so it cannot be given one. 

522 catalog = build_catalog(_real_data()) (empty)

523 entry = next(e for e in catalog['tts'] if e['id'] == 'openai') (empty)

524 page = render_provider_page('tts', entry) (empty)

525 assert '## Billed per audio token' in page (empty)

526 assert 'gpt-4o-mini-tts' in page (empty)

527 

528 

529def test_llm_index_summarises_by_vendor_rather_than_listing_every_model(): (empty)

530 # Over a thousand rows on one page is not readable; the vendor pages carry the detail. 

531 data = _real_data() (empty)

532 catalog = build_catalog(data) (empty)

533 page = render_index_page('llm', catalog['llm'], build_comparison(data)['llm']) (empty)

534 assert 'Cheapest input $ / Mtok' in page (empty)

535 assert page.count('\n|') < 100 # one row per provider, not per model (empty)

536 assert '/llm/openrouter' in page (empty)

537 

538 

539def test_index_page_lists_every_model_for_small_categories(): (empty)

540 data = _real_data() (empty)

541 catalog = build_catalog(data) (empty)

542 page = render_index_page('stt', catalog['stt'], build_comparison(data)['stt']) (empty)

543 total = sum(len(entry['models']) for entry in catalog['stt']) (empty)

544 # every model gets a row, minus the token-priced ones which are called out separately 

545 token_priced = sum(1 for e in catalog['stt'] for m in e['models'] if m['values']['per_min'] is None) (empty)

546 assert page.count('\n| [') == total - token_priced (empty)

547 

548 # STT has token-priced rows again: Soniox bills transcription by token rather than by 

549 # audio-second, so they cannot share a column with the per-minute rates and are called out 

550 # separately. (Between #107 and Soniox landing this was briefly zero, and this assertion 

551 # said so; the callout is data-driven, not a permanent fixture either way.) 

552 assert token_priced > 0 (empty)

553 assert 'bill audio as tokens' in page (empty)

554 

555 # TTS likewise: gpt-4o-mini-tts and Soniox's tts-rt. 

556 tts_page = render_index_page('tts', catalog['tts'], build_comparison(data)['tts']) (empty)

557 assert 'bill audio as tokens' in tts_page (empty)

558 

559 

560def test_estimated_rates_are_marked_and_footnoted(): (empty)

561 """A vendor's own estimate must not print like a billed rate. 

562 

563 gpt-4o-transcribe bills per token; its $0.006/minute comes from a column OpenAI heads 

564 "Estimated cost". whisper-1 shows the same $0.006/minute as its actual billed meter. The 

565 two sit adjacent in the same column, so the marker is the only thing separating them. 

566 """ 

567 catalog = build_catalog(_real_data()) (empty)

568 entry = next(e for e in catalog['stt'] if e['id'] == 'openai') (empty)

569 page = render_provider_page('stt', entry) (empty)

570 

571 rows = {row['id']: row for row in entry['models']} (empty)

572 assert 'estimated' in rows['gpt-4o-transcribe']['markers'] (empty)

573 assert 'estimated' in rows['gpt-4o-mini-transcribe']['markers'] (empty)

574 # Billed meters, including one at the identical rate, stay unmarked. 

575 assert 'estimated' not in rows['whisper-1']['markers'] (empty)

576 assert 'estimated' not in rows['gpt-transcribe']['markers'] (empty)

577 

578 assert '`gpt-4o-transcribe` ^estimated' in page (empty)

579 assert 'not the meter the vendor bills on' in page (empty)

580 

581 

582def test_comparison_section_renders_on_voice_index_pages(): (empty)

583 data = _real_data() (empty)

584 catalog = build_catalog(data) (empty)

585 page = render_index_page('tts', catalog['tts'], build_comparison(data)['tts']) (empty)

586 assert '## Via LiveKit Inference' in page (empty)

587 assert 'vs direct' in page (empty)

588 

589 

590# ---- navigation -------------------------------------------------------------- 

591 

592 

593def test_category_tab_groups_a_large_sidebar_and_leaves_a_small_one_flat(): (empty)

594 catalog = build_catalog(_real_data()) (empty)

595 llm = category_tab('llm', catalog['llm']) (empty)

596 assert llm['tab'] == 'LLM' (empty)

597 pages = cast('list[Any]', llm['pages']) (empty)

598 assert pages[0] == f'llm/{INDEX_PAGE}' (empty)

599 groups = [str(cast('dict[str, Any]', page)['group']) for page in pages if isinstance(page, dict)] (empty)

600 assert groups == ['Direct vendors', 'Gateways', 'HuggingFace Inference'] (empty)

601 

602 # S2S crossed MIN_PROVIDERS_FOR_GROUPS once Boson and OpenRouter landed, so it groups now too. 

603 s2s = category_tab('s2s', catalog['s2s']) (empty)

604 s2s_groups = [ (empty)

605 str(cast('dict[str, Any]', page)['group']) for page in cast('list[Any]', s2s['pages']) if isinstance(page, dict) 

606 ] 

607 assert s2s_groups == ['Direct vendors', 'Gateways'] (empty)

608 

609 # VAD is the small one now: one provider, so no group headings. 

610 vad = category_tab('vad', catalog['vad']) (empty)

611 assert all(isinstance(page, str) for page in cast('list[Any]', vad['pages'])) (empty)

612 

613 

614def test_build_navigation_has_the_eight_tabs_in_order(): (empty)

615 nav = build_navigation(build_catalog(_real_data())) (empty)

616 assert [tab['tab'] for tab in nav['tabs']] == [ (empty)

617 'Overview', 

618 'STT', 

619 'LLM', 

620 'TTS', 

621 'S2S', 

622 'VAD', 

623 'Agents', 

624 'Telephony', 

625 'Add Yours', 

626 ] 

627 

628 

629# Every icon name used anywhere in docs/, verified to exist in the Lucide set. Mintlify defaults to 

630# Font Awesome, where most of these names do not exist, so the library declaration below and this 

631# list have to stay in step: a mismatch silently renders a tab with no icon at all. 

632LUCIDE_ICONS = { (empty)

633 'activity', 

634 'audio-waveform', 

635 'book-open', 

636 'bot', 

637 'brain', 

638 'clock', 

639 'git-pull-request', 

640 'mic', 

641 'phone', 

642 'rss', 

643 'volume-2', 

644} 

645 

646 

647def test_docs_json_declares_the_lucide_icon_library(): (empty)

648 config = json.loads((DOCS_DIR / 'docs.json').read_text()) (empty)

649 assert config.get('icons', {}).get('library') == 'lucide' (empty)

650 

651 

652def test_every_navigation_icon_is_a_known_lucide_name(): (empty)

653 nav = build_navigation(build_catalog(_real_data())) (empty)

654 icons = {str(tab['icon']) for tab in nav['tabs']} (empty)

655 assert icons <= LUCIDE_ICONS, f'unverified icon name(s): {sorted(icons - LUCIDE_ICONS)}' (empty)

656 assert len(icons) == len(nav['tabs']) # every tab has its own icon, none dropped (empty)

657 

658 

659def test_every_page_frontmatter_icon_is_a_known_lucide_name(): (empty)

660 used: set[str] = set() (empty)

661 for page in DOCS_DIR.rglob('*.mdx'): (empty)

662 match = re.search(r'^icon: "([^"]+)"$', page.read_text(), re.MULTILINE) (empty)

663 if match: (empty)

664 used.add(match.group(1)) (empty)

665 assert used <= LUCIDE_ICONS, f'unverified icon name(s): {sorted(used - LUCIDE_ICONS)}' (empty)

666 

667 

668def _nav_pages(pages: list[Any]) -> list[str]: (empty)

669 flat: list[str] = [] (empty)

670 for page in pages: (empty)

671 if isinstance(page, str): (empty)

672 flat.append(page) (empty)

673 else: 

674 flat += _nav_pages(page['pages']) (empty)

675 return flat (empty)

676 

677 

678def test_every_navigation_entry_resolves_to_a_committed_page(): (empty)

679 # Mintlify fails the build on a nav entry with no file, so this catches it before a deploy does. 

680 config = json.loads((DOCS_DIR / 'docs.json').read_text()) (empty)

681 for tab in config['navigation']['tabs']: (empty)

682 for page in _nav_pages(tab['pages']): (empty)

683 assert (DOCS_DIR / f'{page}.mdx').is_file(), f'{page}.mdx missing' (empty)

684 

685 

686def test_no_internal_link_points_at_a_missing_page(): (empty)

687 # A broken internal link fails the Mintlify build. This also covers the hand-written pages, 

688 # which the generator never touches and so nothing else would catch. 

689 pages = {str(path.relative_to(DOCS_DIR).with_suffix('')) for path in DOCS_DIR.rglob('*.mdx')} (empty)