Extract example checks into release profile

docs/development.md

@@ -74,6 +74,9 @@ change files, validate release-note structure, or retire roadmaps. It requires G
 Git, a POSIX shell, and Python 3. Tests and examples require no provider
 credentials or live inference calls.
 
+Run the consumer example checks independently with
+`python3 scripts/check-examples.py`.
+
 The script can run on intended uncommitted changes during development.
 Formatting and documentation checks use tracked files; stage intended new
 files before final validation so those checks include them. The release

docs/roadmap/implementation.md

@@ -6,7 +6,8 @@ owns the implementation steps; the roadmap owns purpose and selected policy.
 
 Status: Stage 1's tool selection and dependency preparation are complete in
 `3ca01ae`. The agreed series-based Go minimum supersedes the original patch-based
-requirement. Resume adoption at Stage 2 after this revision.
+requirement. Stage 2's example checker and shared validation profile are
+complete. Resume adoption at Stage 3 after this revision.
 
 ## Baseline And Working Rules
 

releasekit.json

@@ -0,0 +1,29 @@
+{
+  "product": "Promptkit",
+  "remote": "origin",
+  "validation": {
+    "go": {
+      "race": true,
+      "full_vet": true
+    },
+    "go_format": true,
+    "markdown": {},
+    "module_distribution": {
+      "workspace_files": true,
+      "local_replacements": true
+    }
+  },
+  "checks": [
+    {
+      "args": ["python3", "./scripts/check-examples.py"],
+      "env": {
+        "GOTOOLCHAIN": "local",
+        "GOWORK": "off",
+        "GOFLAGS": "-mod=readonly",
+        "GOPROXY": "off",
+        "GONOPROXY": "none",
+        "GOSUMDB": "off"
+      }
+    }
+  ]
+}

scripts/check-examples.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Run the maintained consumer examples and check their semantic output."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
+
+
+def example_environment():
+    environment = os.environ.copy()
+    environment["GOTOOLCHAIN"] = "local"
+    try:
+        host = subprocess.run(
+            ["go", "env", "GOHOSTOS", "GOHOSTARCH"],
+            cwd=REPOSITORY_ROOT,
+            env=environment,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+    except OSError as error:
+        raise RuntimeError(f"could not run go env: {error}") from error
+
+    if host.returncode != 0:
+        raise RuntimeError(
+            "go env GOHOSTOS GOHOSTARCH failed "
+            f"with exit status {host.returncode}: {host.stderr.strip()}"
+        )
+
+    values = host.stdout.splitlines()
+    if len(values) != 2 or not all(values):
+        raise RuntimeError(
+            "go env GOHOSTOS GOHOSTARCH returned unexpected output: "
+            f"{host.stdout!r}"
+        )
+
+    environment["GOOS"], environment["GOARCH"] = values
+    return environment
+
+
+def run_example(name, environment, failures):
+    command = ["go", "run", f"./examples/go-library/{name}"]
+    try:
+        result = subprocess.run(
+            command,
+            cwd=REPOSITORY_ROOT,
+            env=environment,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+    except OSError as error:
+        failures.append(f"{name}: could not start {command[0]}: {error}")
+        return None
+
+    if result.returncode != 0:
+        failures.append(f"{name}: command exited with status {result.returncode}")
+        if result.stderr:
+            failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
+        if result.stdout:
+            failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
+        return None
+
+    try:
+        value = json.loads(result.stdout)
+    except json.JSONDecodeError as error:
+        failures.append(f"{name}: output is not valid JSON: {error}")
+        if result.stdout:
+            failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
+        return None
+
+    if not isinstance(value, dict):
+        failures.append(
+            f"{name}: expected a JSON object, got {type(value).__name__}"
+        )
+        return None
+
+    return value
+
+
+def expect_equal(example_name, value, key, expected, failures):
+    actual = value.get(key)
+    matches = actual is expected if isinstance(expected, bool) else actual == expected
+    if not matches:
+        failures.append(
+            f"{example_name}: expected {key}={expected!r}, got {actual!r}"
+        )
+
+
+def expect_positive(example_name, value, key, failures):
+    actual = value.get(key)
+    try:
+        matches = actual > 0
+    except TypeError:
+        matches = False
+    if not matches:
+        failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
+
+
+def main():
+    try:
+        environment = example_environment()
+    except RuntimeError as error:
+        print(error, file=sys.stderr)
+        return 1
+
+    failures = []
+    prepared = run_example("prepare", environment, failures)
+    result = run_example("run", environment, failures)
+    observed = run_example("observe", environment, failures)
+
+    if prepared is not None:
+        expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures)
+        expect_equal(
+            "prepare", prepared, "selected_profile", "offline-example", failures
+        )
+        expect_equal("prepare", prepared, "model", "offline-model", failures)
+        expect_positive("prepare", prepared, "message_count", failures)
+
+    if result is not None:
+        expect_equal(
+            "run", result, "output", "Ada finished the migration review.", failures
+        )
+        expect_equal("run", result, "is_valid", True, failures)
+        expect_equal("run", result, "validation_status", "passed", failures)
+        expect_equal("run", result, "model", "offline-model", failures)
+        expect_equal("run", result, "total_tokens", 18, failures)
+
+    if observed is not None:
+        expect_equal("observe", observed, "proposal_attempts", 2, failures)
+        expect_equal(
+            "observe", observed, "proposal_transient_retries", 1, failures
+        )
+        expect_equal(
+            "observe", observed, "proposal_usage_complete", False, failures
+        )
+        expect_equal("observe", observed, "assessment_attempts", 1, failures)
+        expect_equal(
+            "observe", observed, "assessment_transient_retries", 0, failures
+        )
+        expect_equal(
+            "observe", observed, "assessment_outcome", "accepted", failures
+        )
+
+    if failures:
+        print("Consumer example checks failed:", file=sys.stderr)
+        for failure in failures:
+            print(f"- {failure}", file=sys.stderr)
+        return 1
+
+    print("Consumer example checks passed: prepare, run, observe")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/release-check.sh

@@ -5,33 +5,7 @@ cd "$(dirname "$0")/.."
 
 go test -race -vet=off ./...
 go vet ./...
-python3 - <<'PY'
-import json
-import subprocess
-
-def example(name):
-    return json.loads(subprocess.check_output(
-        ["go", "run", "./examples/go-library/" + name], text=True))
-
-prepared = example("prepare")
-assert prepared["prompt_id"] == "example.prepare", prepared
-assert prepared["selected_profile"] == "offline-example", prepared
-assert prepared["model"] == "offline-model", prepared
-assert prepared["message_count"] > 0, prepared
-result = example("run")
-assert result["output"] == "Ada finished the migration review.", result
-assert result["is_valid"] is True, result
-assert result["validation_status"] == "passed", result
-assert result["model"] == "offline-model", result
-assert result["total_tokens"] == 18, result
-observed = example("observe")
-assert observed["proposal_attempts"] == 2, observed
-assert observed["proposal_transient_retries"] == 1, observed
-assert observed["proposal_usage_complete"] is False, observed
-assert observed["assessment_attempts"] == 1, observed
-assert observed["assessment_transient_retries"] == 0, observed
-assert observed["assessment_outcome"] == "accepted", observed
-PY
+python3 ./scripts/check-examples.py
 
 unformatted=$(
     git ls-files '*.go' |