diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f5ee7bb..b98bdfd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -8,8 +8,9 @@ Status: Stage 1's tool selection and dependency preparation are complete in `3ca01ae`. The agreed series-based Go minimum supersedes the original patch-based requirement. Stage 2's example checker and shared validation profile are complete in `0d163f8`. Stage 3's native tool bootstrap is complete in -`d594085`. Stage 4's successful-publication fixture is complete. Resume -adoption at Stage 5 after this revision. +`d594085`. Stage 4's successful-publication fixture is complete in `51b0729`. +Stage 5's failure-propagation fixture is complete. Resume adoption at Stage 6 +after this revision. ## Baseline And Working Rules diff --git a/scripts/test-release.py b/scripts/test-release.py index 5cab08a..5497a6e 100644 --- a/scripts/test-release.py +++ b/scripts/test-release.py @@ -1,4 +1,4 @@ -"""Exercise successful ReleaseKit publication wiring in a disposable repository.""" +"""Exercise ReleaseKit publication wiring in disposable repositories.""" import json import os @@ -12,12 +12,13 @@ from pathlib import Path REPOSITORY_ROOT = Path(__file__).resolve().parent.parent RELEASE_VERSION = "v0.0.1" +FAILED_RELEASE_VERSION = "v0.0.2" RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit" -def run(command, cwd, environment): +def execute(command, cwd, environment): try: - result = subprocess.run( + return subprocess.run( command, cwd=cwd, env=environment, @@ -28,6 +29,9 @@ def run(command, cwd, environment): except OSError as error: raise RuntimeError(f"could not run {command[0]}: {error}") from error + +def run(command, cwd, environment): + result = execute(command, cwd, environment) if result.returncode != 0: details = [ f"command failed with status {result.returncode}: {' '.join(command)}" @@ -144,7 +148,7 @@ def fixture_sources(repository, environment): run(["gofmt", "-w", *go_files], cwd=repository, environment=environment) -def prepare_repository(repository, remote, environment, module_bytes): +def prepare_repository(repository, remote, environment, module_bytes, version): run( ["git", "init", "--bare", "--initial-branch=main", str(remote)], cwd=repository.parent, @@ -181,16 +185,20 @@ def prepare_repository(repository, remote, environment, module_bytes): """ import os from pathlib import Path + import sys marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"]) with marker.open("a", encoding="utf-8") as stream: stream.write("called\\n") + if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1": + print("fixture example check rejected candidate", file=sys.stderr) + raise SystemExit(1) """, ) write_file( - repository / f"docs/releases/{RELEASE_VERSION}.md", + repository / f"docs/releases/{version}.md", f""" - # Promptkit {RELEASE_VERSION} + # Promptkit {version} Disposable publication fixture. """, @@ -294,6 +302,98 @@ def verify_publication(repository, remote, temporary_root, environment, module_b repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit") +def verify_validation_failure( + repository, remote, temporary_root, environment, module_bytes, selected_go +): + scenario_environment = environment.copy() + suite_marker = temporary_root / "failure-suite.jsonl" + example_marker = temporary_root / "failure-example.log" + scenario_environment.update( + { + "FIXTURE_SUITE_MARKER": str(suite_marker), + "FIXTURE_EXAMPLE_MARKER": str(example_marker), + "FIXTURE_EXAMPLE_FAILURE": "1", + } + ) + prepare_repository( + repository, + remote, + scenario_environment, + module_bytes, + FAILED_RELEASE_VERSION, + ) + + result = execute( + [str(repository / "scripts/releasekit.sh"), "publish", FAILED_RELEASE_VERSION], + cwd=temporary_root, + environment=scenario_environment, + ) + if result.returncode != 1: + raise RuntimeError( + "validation failure returned status " + f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if "fixture example check rejected candidate" not in result.stderr: + raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}") + + suite_lines = suite_marker.read_text(encoding="utf-8").splitlines() + if len(suite_lines) != 1: + raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}") + suite_record = json.loads(suite_lines[0]) + if suite_record != {"go_version": selected_go, "race": True}: + raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}") + if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]: + raise RuntimeError("expected one additional-check invocation before failure") + + tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}" + for description, command in ( + ( + "local", + ["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref], + ), + ( + "remote", + ["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref], + ), + ): + result = execute(command, cwd=temporary_root, environment=scenario_environment) + if result.returncode == 0: + raise RuntimeError(f"validation failure created the {description} tag {tag_ref}") + if result.returncode != 1: + raise RuntimeError( + f"could not verify {description} tag absence: " + f"status {result.returncode}: {result.stderr.strip()}" + ) + + for name, contents in module_bytes.items(): + if (repository / name).read_bytes() != contents: + raise RuntimeError(f"failed publication changed {name}") + status = repo_git( + repository, + scenario_environment, + "status", + "--porcelain", + "--untracked-files=all", + ) + if status: + raise RuntimeError(f"failure fixture checkout is not clean:\n{status}") + + +def verify_usage_classification(binary, temporary_root, environment): + result = execute( + [str(binary), "unknown"], cwd=temporary_root, environment=environment + ) + if result.returncode != 2: + raise RuntimeError( + f"unsupported command returned status {result.returncode}, want 2; " + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if result.stdout: + raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}") + if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr: + raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}") + + def main(): environment = os.environ.copy() environment.update( @@ -334,7 +434,9 @@ def main(): environment["FIXTURE_SUITE_MARKER"] = str(suite_marker) environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker) try: - prepare_repository(repository, remote, environment, module_bytes) + prepare_repository( + repository, remote, environment, module_bytes, RELEASE_VERSION + ) verify_publication( repository, remote, @@ -343,11 +445,22 @@ def main(): module_bytes, selected_go, ) + verify_usage_classification( + repository / ".releasekit/bin/releasekit", temporary_root, environment + ) + verify_validation_failure( + temporary_root / "failure-checkout", + temporary_root / "failure-origin.git", + temporary_root, + environment, + module_bytes, + selected_go, + ) except (RuntimeError, OSError, json.JSONDecodeError) as error: print(error, file=sys.stderr) return 1 - print("ReleaseKit successful-publication fixture passed") + print("ReleaseKit publication success and failure fixtures passed") return 0