Coverage for tests/test_release.py: 100%
220 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-13 19:28 +0000
1"""Release metadata and conservative public compatibility gates."""
3import json
4from copy import deepcopy
5from importlib.metadata import version
6from pathlib import Path
7from textwrap import dedent
9import pytest
11from data_api import __version__
12from data_api.api.api import app
13from data_api.checks import release
14from data_api.checks.mcp import BASELINE, structural, tool_contract
15from data_api.mcp.server import mcp
18@pytest.mark.parametrize("missing_section", [False, True])
19def test_prepare_release_renders_current_pr_template(monkeypatch, tmp_path, missing_section):
20 root = Path.cwd()
21 workflow = (root / ".github" / "workflows" / "prepare-release.yml").read_text(encoding="utf-8")
22 script = dedent(
23 workflow.split("- name: Fill repository PR template", 1)[1]
24 .split("python - <<'PY'\n", 1)[1]
25 .split("\n PY", 1)[0]
26 )
27 relative_template = Path(".github") / "PULL_REQUEST_TEMPLATE" / "pull_request_template.md"
28 template = (root / relative_template).read_text(encoding="utf-8")
29 if missing_section:
30 template = template.replace("#### Validation\nN/A", "#### Validation\n")
31 destination = tmp_path / relative_template
32 destination.parent.mkdir(parents=True)
33 destination.write_text(template, encoding="utf-8")
34 monkeypatch.chdir(tmp_path)
35 monkeypatch.setenv("RUNNER_TEMP", str(tmp_path))
36 monkeypatch.setenv("RELEASE_VERSION", "3.0.2")
37 output = tmp_path / "release-pr.md"
38 if missing_section:
39 with pytest.raises(ValueError, match="Expected one PR template section: #### Validation"):
40 exec(compile(script, "prepare-release-template", "exec"), {})
41 assert not output.exists()
42 return
44 exec(compile(script, "prepare-release-template", "exec"), {})
45 rendered = output.read_text(encoding="utf-8")
46 for guidance in (
47 "Add or modify features.\n Including Commit Tags: feat, style, perf, test, revert.",
48 "Bug fixes.\n Including Commit Tags: fix.",
49 "Code changes that neither fixes a bug nor adds a feature.\n"
50 " Including Commit Tags: refactor, update.",
51 "Changes to the build process or auxiliary tools and libraries.\n"
52 " Including Commit Tags: chore.",
53 "Documentation changes.\n Including Commit Tags: docs.",
54 "Any other notes.",
55 ):
56 comment = f"<!--- {guidance}" + (" -->" if "\n" not in guidance else "\n -->")
57 assert comment in template
58 assert comment in rendered
59 assert "### Internal\nPrepare 3.0.2; update pyproject.toml and uv.lock." in rendered
60 assert "### Notes\nSee docs/releasing.md" in rendered
61 assert (
62 "#### Validation\n- `uv run --no-sync --no-build pre-commit run --all-files` (passed)"
63 in rendered
64 )
65 assert rendered.count("(passed)") == 5
66 assert "- <something>" not in rendered
67 assert [line for line in rendered.splitlines() if line.startswith("#")] == [
68 line for line in template.splitlines() if line.startswith("#")
69 ]
70 for heading in ("Features", "Fixes", "Refactors", "Documentations"):
71 assert f"### {heading}\nN/A" in rendered
74def test_installed_version_has_one_source():
75 assert release.validate_version(Path.cwd(), f"v{__version__}") == __version__
76 assert __version__ == app.version == version("nthu-data-api")
77 assert mcp.version == __version__
80@pytest.mark.parametrize("value", ["3", "3.0", "v3.0.0", "03.0.0", "3.0.0rc1", "3.0.0\n", "3.０.0"])
81def test_nonstable_versions_rejected(value):
82 with pytest.raises(ValueError, match="stable"):
83 release.version_tuple(value)
86@pytest.mark.parametrize("tag", ["", "3.0.0", "v2.0.0", "v3.0.0-rc.1", "v03.0.0"])
87def test_invalid_or_mismatched_tags_fail(tag):
88 root = Path.cwd()
89 with pytest.raises(ValueError, match="tag"):
90 release.validate_version(root, tag)
93def test_lock_mismatch_fails(tmp_path):
94 (tmp_path / "pyproject.toml").write_text(
95 '[project]\nname="nthu-data-api"\nversion="3.0.0"\n', encoding="utf-8"
96 )
97 (tmp_path / "uv.lock").write_text(
98 '[[package]]\nname="nthu-data-api"\nversion="2.0.0"\n', encoding="utf-8"
99 )
100 with pytest.raises(ValueError, match="must match"):
101 release.validate_version(tmp_path)
104@pytest.mark.parametrize(
105 "code,stdout,expected",
106 [
107 (2, "", True),
108 (0, "a" * 40 + "\trefs/tags/v3.0.0\n", False),
109 (128, "", None),
110 (0, "", None),
111 (2, "unexpected", None),
112 (0, "a" * 40 + "\trefs/tags/v3.0.1\n", None),
113 ],
114)
115def test_publication_uses_authoritative_remote_and_fails_closed(
116 monkeypatch, code, stdout, expected
117):
118 from types import SimpleNamespace
120 def run(args, **kwargs):
121 assert args == [
122 "git",
123 "ls-remote",
124 "--exit-code",
125 "--refs",
126 "https://github.com/NTHU-SA/NTHU-Data-API.git",
127 "refs/tags/v3.0.0",
128 ]
129 assert kwargs["timeout"] == 30
130 return SimpleNamespace(returncode=code, stdout=stdout)
132 monkeypatch.setattr(release.subprocess, "run", run)
133 if expected is None:
134 with pytest.raises(ValueError, match="Unable to verify"):
135 release.initial_version_is_unpublished("3.0.0")
136 else:
137 assert release.initial_version_is_unpublished("3.0.0") is expected
138 assert release.initial_version_is_unpublished("3.0.1") is False
141@pytest.mark.parametrize("failure", [OSError(), release.subprocess.TimeoutExpired("git", 30)])
142def test_publication_network_failure_is_not_unpublished(monkeypatch, failure):
143 def run(*args, **kwargs):
144 raise failure
146 monkeypatch.setattr(release.subprocess, "run", run)
147 with pytest.raises(ValueError, match="Unable to verify"):
148 release.initial_version_is_unpublished("3.0.0")
151@pytest.mark.parametrize("unpublished", [True, False, None])
152@pytest.mark.parametrize("current", ["3.0.0", "3.0.1", "2.9.0", "4.0.0"])
153def test_only_verified_unpublished_initial_version_can_update_contract(
154 monkeypatch, tmp_path, unpublished, current
155):
156 fixtures = tmp_path / "tests" / "fixtures"
157 fixtures.mkdir(parents=True)
158 for name in release.CONTRACTS:
159 (fixtures / name).write_text("{}", encoding="utf-8")
161 def git_file(sha, path):
162 return '[project]\nversion="3.0.0"\n' if path == "pyproject.toml" else '{"old": {}}'
164 def publication(version):
165 if unpublished is None:
166 raise ValueError("Unable to verify release publication")
167 return unpublished
169 monkeypatch.setattr(release, "git_file", git_file)
170 monkeypatch.setattr(release, "initial_version_is_unpublished", publication)
171 if current == "4.0.0" or (current == "3.0.0" and unpublished is True):
172 assert release.compare_base(tmp_path, "a" * 40, current, check_publication=True)
173 else:
174 with pytest.raises(ValueError):
175 release.compare_base(tmp_path, "a" * 40, current, check_publication=True)
176 if current == "3.0.0":
177 with pytest.raises(ValueError, match="major"):
178 release.compare_base(tmp_path, "a" * 40, current)
181@pytest.mark.parametrize(
182 "old,new",
183 [
184 ({"GET /courses": {}}, {}),
185 ({"type": "string"}, {"type": "integer"}),
186 ({"enum": ["a", "b"]}, {"enum": ["a"]}),
187 ({}, {"minimum": 1}),
188 ({}, {"type": "integer"}),
189 ({}, {"enum": ["only"]}),
190 ({}, {"default": 20}),
191 ({"required": []}, {"required": ["name"]}),
192 ({"default": 5}, {"default": 20}),
193 ],
194)
195def test_breaking_changes_detected(old, new):
196 assert release.breaking_changes(old, new)
199def test_new_required_parameter_is_breaking():
200 assert release.breaking_changes({}, {"query:limit": {"required": True}}, "contract.parameters")
203@pytest.mark.parametrize("name", ["deprecated", "type"])
204@pytest.mark.parametrize("container", ["properties", "$defs", "definitions", "schemas"])
205def test_schema_names_are_not_keywords(name, container):
206 old = {container: {name: {"type": "string"}}}
207 new = {container: {name: {"type": "integer"}}}
208 assert release.breaking_changes(old, new) == [f"contract.{container}.{name}.type: changed"]
209 assert release.breaking_changes(old, {container: {}}) == [
210 f"contract.{container}.{name}: removed"
211 ]
214def test_optional_type_named_argument_is_not_a_constraint():
215 old = {"get_bus_schedule": {"input": {"properties": {}}}}
216 new = {
217 "get_bus_schedule": {
218 "input": {"properties": {"type": {"type": "string"}, "required": {"type": "string"}}}
219 }
220 }
221 assert not release.breaking_changes(old, new, "mcp_contract.json")
222 assert release.breaking_changes(new, old, "mcp_contract.json") == [
223 "mcp_contract.json.get_bus_schedule.input.properties.required: removed",
224 "mcp_contract.json.get_bus_schedule.input.properties.type: removed",
225 ]
228def test_tool_and_parameter_names_are_not_schema_keywords():
229 assert not release.breaking_changes({}, {"type": {"input": {}}}, "mcp_contract.json")
230 assert release.breaking_changes(
231 {"operations": {"GET /test": {"parameters": {"deprecated": {"required": False}}}}},
232 {"operations": {"GET /test": {"parameters": {}}}},
233 "openapi_contract.json",
234 ) == ["openapi_contract.json.operations.GET /test.parameters.deprecated: removed"]
237@pytest.mark.parametrize("keyword", ["default", "const", "enum"])
238def test_literal_schema_values_do_not_reenter_schema_context(keyword):
239 old_value = {"input": {"deprecated": "old"}}
240 new_value = {"input": {"deprecated": "new"}}
241 old = {keyword: [old_value] if keyword == "enum" else old_value}
242 new = {keyword: [new_value] if keyword == "enum" else new_value}
243 assert release.breaking_changes(old, new) == [f"contract.{keyword}: changed"]
244 if keyword == "enum":
245 assert not release.breaking_changes(old, {"enum": [old_value, new_value]})
248@pytest.mark.parametrize("keyword", ["additionalItems", "unevaluatedProperties"])
249def test_schema_valued_keywords_propagate_context(keyword):
250 assert release.breaking_changes({keyword: {}}, {keyword: {"type": "string"}}) == [
251 f"contract.{keyword}.type: new constraint or required input"
252 ]
255@pytest.mark.parametrize("change", ["extension", "narrowed", "branch", "removed_branch"])
256def test_nullable_stop_enum_compatibility(change):
257 old = json.loads(BASELINE.read_text(encoding="utf-8"))
258 new = deepcopy(old)
259 stop = new["get_bus_schedule"]["input"]["properties"]["stop"]
260 if change == "extension":
261 stop["anyOf"][0]["enum"].append("additional stop")
262 elif change == "narrowed":
263 stop["anyOf"][0]["enum"].pop()
264 elif change == "branch":
265 stop["anyOf"][1]["type"] = "integer"
266 else:
267 stop["anyOf"].pop()
268 changes = release.breaking_changes(old, new, "mcp_contract.json")
269 if change == "extension":
270 assert not changes
271 else:
272 assert changes
275@pytest.mark.parametrize(
276 "old,new",
277 [
278 ({}, {"GET /new": {"responses": {}}}),
279 ({}, {"query:new": {"required": False, "schema": {"type": "string"}}}),
280 ({"enum": ["a"]}, {"enum": ["a", "b"]}),
281 ({"deprecated": False}, {"deprecated": True}),
282 ({"properties": {}}, {"properties": {"title": {"type": "string"}}}),
283 ({}, {"NewSchema": {"type": "object", "required": ["id"]}}),
284 ],
285)
286def test_compatible_additions_allowed(old, new):
287 assert not release.breaking_changes(old, new)
290@pytest.mark.parametrize("current,allowed", [("3.0.1", False), ("4.0.0", True)])
291def test_refreshed_baseline_cannot_hide_same_major_removal(monkeypatch, tmp_path, current, allowed):
292 fixtures = tmp_path / "tests" / "fixtures"
293 fixtures.mkdir(parents=True)
294 for name in release.CONTRACTS:
295 (fixtures / name).write_text("{}", encoding="utf-8")
297 def git_file(sha, path):
298 if path == "pyproject.toml":
299 return '[project]\nversion="3.0.0"\n'
300 return '{"public-operation": {}}'
302 monkeypatch.setattr(release, "git_file", git_file)
303 if allowed:
304 assert release.compare_base(tmp_path, "a" * 40, current)
305 else:
306 with pytest.raises(ValueError, match="major"):
307 release.compare_base(tmp_path, "a" * 40, current)
310@pytest.mark.parametrize(
311 "change,breaking",
312 [
313 ("remove_deprecated", True),
314 ("change_deprecated", True),
315 ("add_optional_type", False),
316 ("extend_stop_enum", False),
317 ("change_nullable_branch", True),
318 ("change_default_literal", True),
319 ("restrict_additional_items", True),
320 ("restrict_unevaluated_properties", True),
321 ],
322)
323def test_same_major_mcp_snapshot_changes(monkeypatch, change, breaking):
324 old = json.loads(BASELINE.read_text(encoding="utf-8"))
325 new = deepcopy(old)
326 old_properties = old["get_bus_schedule"]["input"]["properties"]
327 new_properties = new["get_bus_schedule"]["input"]["properties"]
328 if change == "remove_deprecated":
329 old_properties["deprecated"] = {"type": "string"}
330 elif change == "change_deprecated":
331 old_properties["deprecated"] = {"type": "string"}
332 new_properties["deprecated"] = {"type": "integer"}
333 elif change == "add_optional_type":
334 new_properties["type"] = {"type": "string"}
335 elif change == "extend_stop_enum":
336 old_properties["stop"]["anyOf"][0]["enum"].pop()
337 elif change == "change_nullable_branch":
338 new_properties["stop"]["anyOf"][1]["type"] = "integer"
339 elif change == "change_default_literal":
340 old_properties["stop"]["default"] = {"input": {"deprecated": "old"}}
341 new_properties["stop"]["default"] = {"input": {"deprecated": "new"}}
342 else:
343 keyword = (
344 "additionalItems" if change == "restrict_additional_items" else "unevaluatedProperties"
345 )
346 old_properties["stop"][keyword] = {}
347 new_properties["stop"][keyword] = {"type": "string"}
349 original_read_text = Path.read_text
351 def read_text(path, *args, **kwargs):
352 if path == Path.cwd() / BASELINE:
353 return json.dumps(new)
354 return original_read_text(path, *args, **kwargs)
356 def git_file(sha, path):
357 if path == "pyproject.toml":
358 return '[project]\nversion="3.0.0"\n'
359 if path.endswith("mcp_contract.json"):
360 return json.dumps(old)
361 return original_read_text(Path(path), encoding="utf-8")
363 monkeypatch.setattr(Path, "read_text", read_text)
364 monkeypatch.setattr(release, "git_file", git_file)
365 root = Path.cwd()
366 if breaking:
367 with pytest.raises(ValueError, match="major"):
368 release.compare_base(root, "a" * 40, "3.0.1")
369 else:
370 assert not release.compare_base(root, "a" * 40, "3.0.1")
373async def test_mcp_structured_baseline():
374 assert await tool_contract() == json.loads(BASELINE.read_text(encoding="utf-8"))
377def test_mcp_field_named_title_is_not_stripped():
378 schema = {"title": "Description", "properties": {"title": {"type": "string"}}}
379 original = deepcopy(schema)
380 assert structural(schema) == {"properties": {"title": {"type": "string"}}}
381 assert schema == original