diff --git a/docs/development.md b/docs/development.md index 977e889..e554605 100644 --- a/docs/development.md +++ b/docs/development.md @@ -74,6 +74,9 @@ change files, validate release-note structure, or retire roadmaps. It requires G Git, a POSIX shell, and Python 3. Tests and examples require no provider credentials or live inference calls. +Run the consumer example checks independently with +`python3 scripts/check-examples.py`. + The script can run on intended uncommitted changes during development. Formatting and documentation checks use tracked files; stage intended new files before final validation so those checks include them. The release diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ddd9765..0cc1469 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -6,7 +6,8 @@ owns the implementation steps; the roadmap owns purpose and selected policy. Status: Stage 1's tool selection and dependency preparation are complete in `3ca01ae`. The agreed series-based Go minimum supersedes the original patch-based -requirement. Resume adoption at Stage 2 after this revision. +requirement. Stage 2's example checker and shared validation profile are +complete. Resume adoption at Stage 3 after this revision. ## Baseline And Working Rules diff --git a/releasekit.json b/releasekit.json new file mode 100644 index 0000000..de23a7c --- /dev/null +++ b/releasekit.json @@ -0,0 +1,29 @@ +{ + "product": "Promptkit", + "remote": "origin", + "validation": { + "go": { + "race": true, + "full_vet": true + }, + "go_format": true, + "markdown": {}, + "module_distribution": { + "workspace_files": true, + "local_replacements": true + } + }, + "checks": [ + { + "args": ["python3", "./scripts/check-examples.py"], + "env": { + "GOTOOLCHAIN": "local", + "GOWORK": "off", + "GOFLAGS": "-mod=readonly", + "GOPROXY": "off", + "GONOPROXY": "none", + "GOSUMDB": "off" + } + } + ] +} diff --git a/scripts/check-examples.py b/scripts/check-examples.py new file mode 100644 index 0000000..901726c --- /dev/null +++ b/scripts/check-examples.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Run the maintained consumer examples and check their semantic output.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parent.parent + + +def example_environment(): + environment = os.environ.copy() + environment["GOTOOLCHAIN"] = "local" + try: + host = subprocess.run( + ["go", "env", "GOHOSTOS", "GOHOSTARCH"], + cwd=REPOSITORY_ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + except OSError as error: + raise RuntimeError(f"could not run go env: {error}") from error + + if host.returncode != 0: + raise RuntimeError( + "go env GOHOSTOS GOHOSTARCH failed " + f"with exit status {host.returncode}: {host.stderr.strip()}" + ) + + values = host.stdout.splitlines() + if len(values) != 2 or not all(values): + raise RuntimeError( + "go env GOHOSTOS GOHOSTARCH returned unexpected output: " + f"{host.stdout!r}" + ) + + environment["GOOS"], environment["GOARCH"] = values + return environment + + +def run_example(name, environment, failures): + command = ["go", "run", f"./examples/go-library/{name}"] + try: + result = subprocess.run( + command, + cwd=REPOSITORY_ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + except OSError as error: + failures.append(f"{name}: could not start {command[0]}: {error}") + return None + + if result.returncode != 0: + failures.append(f"{name}: command exited with status {result.returncode}") + if result.stderr: + failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}") + if result.stdout: + failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}") + return None + + try: + value = json.loads(result.stdout) + except json.JSONDecodeError as error: + failures.append(f"{name}: output is not valid JSON: {error}") + if result.stdout: + failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}") + return None + + if not isinstance(value, dict): + failures.append( + f"{name}: expected a JSON object, got {type(value).__name__}" + ) + return None + + return value + + +def expect_equal(example_name, value, key, expected, failures): + actual = value.get(key) + matches = actual is expected if isinstance(expected, bool) else actual == expected + if not matches: + failures.append( + f"{example_name}: expected {key}={expected!r}, got {actual!r}" + ) + + +def expect_positive(example_name, value, key, failures): + actual = value.get(key) + try: + matches = actual > 0 + except TypeError: + matches = False + if not matches: + failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}") + + +def main(): + try: + environment = example_environment() + except RuntimeError as error: + print(error, file=sys.stderr) + return 1 + + failures = [] + prepared = run_example("prepare", environment, failures) + result = run_example("run", environment, failures) + observed = run_example("observe", environment, failures) + + if prepared is not None: + expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures) + expect_equal( + "prepare", prepared, "selected_profile", "offline-example", failures + ) + expect_equal("prepare", prepared, "model", "offline-model", failures) + expect_positive("prepare", prepared, "message_count", failures) + + if result is not None: + expect_equal( + "run", result, "output", "Ada finished the migration review.", failures + ) + expect_equal("run", result, "is_valid", True, failures) + expect_equal("run", result, "validation_status", "passed", failures) + expect_equal("run", result, "model", "offline-model", failures) + expect_equal("run", result, "total_tokens", 18, failures) + + if observed is not None: + expect_equal("observe", observed, "proposal_attempts", 2, failures) + expect_equal( + "observe", observed, "proposal_transient_retries", 1, failures + ) + expect_equal( + "observe", observed, "proposal_usage_complete", False, failures + ) + expect_equal("observe", observed, "assessment_attempts", 1, failures) + expect_equal( + "observe", observed, "assessment_transient_retries", 0, failures + ) + expect_equal( + "observe", observed, "assessment_outcome", "accepted", failures + ) + + if failures: + print("Consumer example checks failed:", file=sys.stderr) + for failure in failures: + print(f"- {failure}", file=sys.stderr) + return 1 + + print("Consumer example checks passed: prepare, run, observe") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release-check.sh b/scripts/release-check.sh index cf12d0e..4fac318 100755 --- a/scripts/release-check.sh +++ b/scripts/release-check.sh @@ -5,33 +5,7 @@ cd "$(dirname "$0")/.." go test -race -vet=off ./... go vet ./... -python3 - <<'PY' -import json -import subprocess - -def example(name): - return json.loads(subprocess.check_output( - ["go", "run", "./examples/go-library/" + name], text=True)) - -prepared = example("prepare") -assert prepared["prompt_id"] == "example.prepare", prepared -assert prepared["selected_profile"] == "offline-example", prepared -assert prepared["model"] == "offline-model", prepared -assert prepared["message_count"] > 0, prepared -result = example("run") -assert result["output"] == "Ada finished the migration review.", result -assert result["is_valid"] is True, result -assert result["validation_status"] == "passed", result -assert result["model"] == "offline-model", result -assert result["total_tokens"] == 18, result -observed = example("observe") -assert observed["proposal_attempts"] == 2, observed -assert observed["proposal_transient_retries"] == 1, observed -assert observed["proposal_usage_complete"] is False, observed -assert observed["assessment_attempts"] == 1, observed -assert observed["assessment_transient_retries"] == 0, observed -assert observed["assessment_outcome"] == "accepted", observed -PY +python3 ./scripts/check-examples.py unformatted=$( git ls-files '*.go' |