Test ReleaseKit validation failure handling

docs/roadmap/implementation.md

@@ -8,8 +8,9 @@ Status: Stage 1's tool selection and dependency preparation are complete in
 `3ca01ae`. The agreed series-based Go minimum supersedes the original patch-based
 requirement. Stage 2's example checker and shared validation profile are
 complete in `0d163f8`. Stage 3's native tool bootstrap is complete in
-`d594085`. Stage 4's successful-publication fixture is complete. Resume
+`d594085`. Stage 4's successful-publication fixture is complete in `51b0729`.
-adoption at Stage 5 after this revision.
+Stage 5's failure-propagation fixture is complete. Resume adoption at Stage 6
+after this revision.
 
 ## Baseline And Working Rules
 

scripts/test-release.py

@@ -1,4 +1,4 @@
-"""Exercise successful ReleaseKit publication wiring in a disposable repository."""
+"""Exercise ReleaseKit publication wiring in disposable repositories."""
 
 import json
 import os
@@ -12,12 +12,13 @@ from pathlib import Path
 
 REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
 RELEASE_VERSION = "v0.0.1"
+FAILED_RELEASE_VERSION = "v0.0.2"
 RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"
 
 
-def run(command, cwd, environment):
+def execute(command, cwd, environment):
     try:
-        result = subprocess.run(
+        return subprocess.run(
             command,
             cwd=cwd,
             env=environment,
@@ -28,6 +29,9 @@ def run(command, cwd, environment):
     except OSError as error:
         raise RuntimeError(f"could not run {command[0]}: {error}") from error
 
+
+def run(command, cwd, environment):
+    result = execute(command, cwd, environment)
     if result.returncode != 0:
         details = [
             f"command failed with status {result.returncode}: {' '.join(command)}"
@@ -144,7 +148,7 @@ def fixture_sources(repository, environment):
     run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)
 
 
-def prepare_repository(repository, remote, environment, module_bytes):
+def prepare_repository(repository, remote, environment, module_bytes, version):
     run(
         ["git", "init", "--bare", "--initial-branch=main", str(remote)],
         cwd=repository.parent,
@@ -181,16 +185,20 @@ def prepare_repository(repository, remote, environment, module_bytes):
         """
         import os
         from pathlib import Path
+        import sys
 
         marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
         with marker.open("a", encoding="utf-8") as stream:
             stream.write("called\\n")
+        if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
+            print("fixture example check rejected candidate", file=sys.stderr)
+            raise SystemExit(1)
         """,
     )
     write_file(
-        repository / f"docs/releases/{RELEASE_VERSION}.md",
+        repository / f"docs/releases/{version}.md",
         f"""
-        # Promptkit {RELEASE_VERSION}
+        # Promptkit {version}
 
         Disposable publication fixture.
         """,
@@ -294,6 +302,98 @@ def verify_publication(repository, remote, temporary_root, environment, module_b
     repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")
 
 
+def verify_validation_failure(
+    repository, remote, temporary_root, environment, module_bytes, selected_go
+):
+    scenario_environment = environment.copy()
+    suite_marker = temporary_root / "failure-suite.jsonl"
+    example_marker = temporary_root / "failure-example.log"
+    scenario_environment.update(
+        {
+            "FIXTURE_SUITE_MARKER": str(suite_marker),
+            "FIXTURE_EXAMPLE_MARKER": str(example_marker),
+            "FIXTURE_EXAMPLE_FAILURE": "1",
+        }
+    )
+    prepare_repository(
+        repository,
+        remote,
+        scenario_environment,
+        module_bytes,
+        FAILED_RELEASE_VERSION,
+    )
+
+    result = execute(
+        [str(repository / "scripts/releasekit.sh"), "publish", FAILED_RELEASE_VERSION],
+        cwd=temporary_root,
+        environment=scenario_environment,
+    )
+    if result.returncode != 1:
+        raise RuntimeError(
+            "validation failure returned status "
+            f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    if "fixture example check rejected candidate" not in result.stderr:
+        raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")
+
+    suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
+    if len(suite_lines) != 1:
+        raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
+    suite_record = json.loads(suite_lines[0])
+    if suite_record != {"go_version": selected_go, "race": True}:
+        raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
+    if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
+        raise RuntimeError("expected one additional-check invocation before failure")
+
+    tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
+    for description, command in (
+        (
+            "local",
+            ["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
+        ),
+        (
+            "remote",
+            ["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
+        ),
+    ):
+        result = execute(command, cwd=temporary_root, environment=scenario_environment)
+        if result.returncode == 0:
+            raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
+        if result.returncode != 1:
+            raise RuntimeError(
+                f"could not verify {description} tag absence: "
+                f"status {result.returncode}: {result.stderr.strip()}"
+            )
+
+    for name, contents in module_bytes.items():
+        if (repository / name).read_bytes() != contents:
+            raise RuntimeError(f"failed publication changed {name}")
+    status = repo_git(
+        repository,
+        scenario_environment,
+        "status",
+        "--porcelain",
+        "--untracked-files=all",
+    )
+    if status:
+        raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")
+
+
+def verify_usage_classification(binary, temporary_root, environment):
+    result = execute(
+        [str(binary), "unknown"], cwd=temporary_root, environment=environment
+    )
+    if result.returncode != 2:
+        raise RuntimeError(
+            f"unsupported command returned status {result.returncode}, want 2; "
+            f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    if result.stdout:
+        raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
+    if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
+        raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")
+
+
 def main():
     environment = os.environ.copy()
     environment.update(
@@ -334,7 +434,9 @@ def main():
         environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
         environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
         try:
-            prepare_repository(repository, remote, environment, module_bytes)
+            prepare_repository(
+                repository, remote, environment, module_bytes, RELEASE_VERSION
+            )
             verify_publication(
                 repository,
                 remote,
@@ -343,11 +445,22 @@ def main():
                 module_bytes,
                 selected_go,
             )
+            verify_usage_classification(
+                repository / ".releasekit/bin/releasekit", temporary_root, environment
+            )
+            verify_validation_failure(
+                temporary_root / "failure-checkout",
+                temporary_root / "failure-origin.git",
+                temporary_root,
+                environment,
+                module_bytes,
+                selected_go,
+            )
         except (RuntimeError, OSError, json.JSONDecodeError) as error:
             print(error, file=sys.stderr)
             return 1
 
-    print("ReleaseKit successful-publication fixture passed")
+    print("ReleaseKit publication success and failure fixtures passed")
     return 0