promptkit/scripts/test-release.py

"""Exercise ReleaseKit publication wiring in disposable repositories."""

import json
import os
import shutil
import subprocess
import sys
import tempfile
import textwrap
from pathlib import Path


REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
RELEASE_VERSION = "v0.0.1"
FAILED_RELEASE_VERSION = "v0.0.2"
RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"


def execute(command, cwd, environment):
    try:
        return subprocess.run(
            command,
            cwd=cwd,
            env=environment,
            text=True,
            capture_output=True,
            check=False,
        )
    except OSError as error:
        raise RuntimeError(f"could not run {command[0]}: {error}") from error


def run(command, cwd, environment):
    result = execute(command, cwd, environment)
    if result.returncode != 0:
        details = [
            f"command failed with status {result.returncode}: {' '.join(command)}"
        ]
        if result.stdout:
            details.append(f"stdout:\n{result.stdout.rstrip()}")
        if result.stderr:
            details.append(f"stderr:\n{result.stderr.rstrip()}")
        raise RuntimeError("\n".join(details))
    return result.stdout


def write_file(path, contents):
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8")


def repo_git(repository, environment, *arguments):
    return run(
        ["git", "-C", str(repository), *arguments],
        cwd=repository.parent,
        environment=environment,
    )


def remote_git(remote, temporary_root, environment, *arguments):
    return run(
        ["git", "--git-dir", str(remote), *arguments],
        cwd=temporary_root,
        environment=environment,
    )


def fixture_sources(repository, environment):
    write_file(
        repository / "fixture.go",
        """
        package fixture

        func candidateValue() string { return "ready" }
        """,
    )
    write_file(
        repository / "fixture_test.go",
        f"""
        package fixture

        import (
            "encoding/json"
            "os"
            "runtime"
            "testing"
        )

        func TestCandidateSuite(t *testing.T) {{
            if candidateValue() != "ready" {{
                t.Fatal("candidate package is not ready")
            }}
            marker := os.Getenv("FIXTURE_SUITE_MARKER")
            if marker == "" {{
                t.Fatal("FIXTURE_SUITE_MARKER is not set")
            }}
            record := struct {{
                GoVersion string `json:"go_version"`
                Race      bool   `json:"race"`
            }}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}}
            data, err := json.Marshal(record)
            if err != nil {{
                t.Fatal(err)
            }}
            file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
            if err != nil {{
                t.Fatal(err)
            }}
            if _, err := file.Write(append(data, '\\n')); err != nil {{
                file.Close()
                t.Fatal(err)
            }}
            if err := file.Close(); err != nil {{
                t.Fatal(err)
            }}
        }}
        """,
    )
    write_file(
        repository / "race_enabled_test.go",
        """
        //go:build race

        package fixture

        const raceDetectorEnabled = true
        """,
    )
    write_file(
        repository / "race_disabled_test.go",
        """
        //go:build !race

        package fixture

        const raceDetectorEnabled = false
        """,
    )
    go_files = [
        str(repository / filename)
        for filename in (
            "fixture.go",
            "fixture_test.go",
            "race_enabled_test.go",
            "race_disabled_test.go",
        )
    ]
    run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)


def prepare_repository(repository, remote, environment, module_bytes, version):
    run(
        ["git", "init", "--bare", "--initial-branch=main", str(remote)],
        cwd=repository.parent,
        environment=environment,
    )
    run(
        ["git", "init", "--initial-branch=main", str(repository)],
        cwd=repository.parent,
        environment=environment,
    )
    repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture")
    repo_git(repository, environment, "config", "user.email", "fixture@example.invalid")
    repo_git(repository, environment, "remote", "add", "origin", str(remote))

    for name, contents in module_bytes.items():
        (repository / name).write_bytes(contents)

    write_file(
        repository / ".gitignore",
        """
        .releasekit/
        """,
    )
    write_file(
        repository / "releasekit.json",
        (REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"),
    )

    scripts = repository / "scripts"
    scripts.mkdir(parents=True, exist_ok=True)
    shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh")
    shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh")
    write_file(
        scripts / "check-examples.py",
        """
        import os
        from pathlib import Path
        import sys

        marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
        with marker.open("a", encoding="utf-8") as stream:
            stream.write("called\\n")
        if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
            print("fixture example check rejected candidate", file=sys.stderr)
            raise SystemExit(1)
        """,
    )
    write_file(
        repository / f"docs/releases/{version}.md",
        f"""
        # Promptkit {version}

        Disposable publication fixture.
        """,
    )
    fixture_sources(repository, environment)

    repo_git(repository, environment, "add", "-A")
    run(
        ["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"],
        cwd=repository.parent,
        environment=environment,
    )
    repo_git(repository, environment, "push", "origin", "main")


def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go):
    selected_releasekit = run(
        ["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE],
        cwd=repository,
        environment=environment,
    ).strip()
    publication = repository / "scripts/release.sh"
    run(
        [str(publication), RELEASE_VERSION],
        cwd=temporary_root,
        environment=environment,
    )

    suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text(
        encoding="utf-8"
    ).splitlines()
    if len(suite_lines) != 1:
        raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}")
    suite_record = json.loads(suite_lines[0])
    if suite_record != {"go_version": selected_go, "race": True}:
        raise RuntimeError(f"unexpected suite marker: {suite_record!r}")

    example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text(
        encoding="utf-8"
    ).splitlines()
    if example_lines != ["called"]:
        raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}")

    binary = repository / ".releasekit/bin/releasekit"
    if not binary.is_file():
        raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable")
    build_info = run(
        ["go", "version", "-m", str(binary)],
        cwd=temporary_root,
        environment=environment,
    )
    first_line = build_info.splitlines()[0]
    if selected_go not in first_line:
        raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}")
    expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t"
    if expected_module not in build_info:
        raise RuntimeError(
            f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}"
        )

    candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip()
    if repo_git(repository, environment, "rev-parse", "main").strip() != candidate:
        raise RuntimeError("fixture main moved away from the published candidate")
    if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate:
        raise RuntimeError("local origin main does not identify the fixture candidate")

    tag_ref = f"refs/tags/{RELEASE_VERSION}"
    if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag":
        raise RuntimeError("local release ref is not an annotated tag")
    local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip()
    local_tag_commit = repo_git(
        repository, environment, "rev-parse", f"{tag_ref}^{{}}"
    ).strip()
    if local_tag_commit != candidate:
        raise RuntimeError("local annotated tag does not identify the fixture candidate")

    remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip()
    remote_tag_commit = remote_git(
        remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}"
    ).strip()
    if remote_tag != local_tag or remote_tag_commit != candidate:
        raise RuntimeError("remote annotated tag does not identify the fixture candidate")
    if remote_git(
        remote, temporary_root, environment, "cat-file", "-t", tag_ref
    ).strip() != "tag":
        raise RuntimeError("remote release ref is not an annotated tag")

    tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref)
    separator = tag_object.find("\n\n")
    if separator < 0:
        raise RuntimeError("annotated tag has no message")
    tag_message = tag_object[separator + 2 :]
    note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text(
        encoding="utf-8"
    )
    if tag_message != note:
        raise RuntimeError("annotated tag message does not match the committed release note")

    for name, contents in module_bytes.items():
        if (repository / name).read_bytes() != contents:
            raise RuntimeError(f"publication changed {name}")

    status = repo_git(
        repository, environment, "status", "--porcelain", "--untracked-files=all"
    )
    if status:
        raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}")
    repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")


def verify_validation_failure(
    repository, remote, temporary_root, environment, module_bytes, selected_go
):
    scenario_environment = environment.copy()
    suite_marker = temporary_root / "failure-suite.jsonl"
    example_marker = temporary_root / "failure-example.log"
    scenario_environment.update(
        {
            "FIXTURE_SUITE_MARKER": str(suite_marker),
            "FIXTURE_EXAMPLE_MARKER": str(example_marker),
            "FIXTURE_EXAMPLE_FAILURE": "1",
        }
    )
    prepare_repository(
        repository,
        remote,
        scenario_environment,
        module_bytes,
        FAILED_RELEASE_VERSION,
    )

    result = execute(
        [str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION],
        cwd=temporary_root,
        environment=scenario_environment,
    )
    if result.returncode != 1:
        raise RuntimeError(
            "validation failure returned status "
            f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
        )
    if "fixture example check rejected candidate" not in result.stderr:
        raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")

    suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
    if len(suite_lines) != 1:
        raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
    suite_record = json.loads(suite_lines[0])
    if suite_record != {"go_version": selected_go, "race": True}:
        raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
    if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
        raise RuntimeError("expected one additional-check invocation before failure")

    tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
    for description, command in (
        (
            "local",
            ["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
        ),
        (
            "remote",
            ["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
        ),
    ):
        result = execute(command, cwd=temporary_root, environment=scenario_environment)
        if result.returncode == 0:
            raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
        if result.returncode != 1:
            raise RuntimeError(
                f"could not verify {description} tag absence: "
                f"status {result.returncode}: {result.stderr.strip()}"
            )

    for name, contents in module_bytes.items():
        if (repository / name).read_bytes() != contents:
            raise RuntimeError(f"failed publication changed {name}")
    status = repo_git(
        repository,
        scenario_environment,
        "status",
        "--porcelain",
        "--untracked-files=all",
    )
    if status:
        raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")


def verify_usage_classification(binary, temporary_root, environment):
    result = execute(
        [str(binary), "unknown"], cwd=temporary_root, environment=environment
    )
    if result.returncode != 2:
        raise RuntimeError(
            f"unsupported command returned status {result.returncode}, want 2; "
            f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
        )
    if result.stdout:
        raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
    if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
        raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")


def main():
    environment = os.environ.copy()
    environment.update(
        {
            "GOTOOLCHAIN": "local",
            "GOWORK": "off",
            "GOPROXY": "off",
            "GONOPROXY": "none",
            "GOSUMDB": "off",
        }
    )
    selected_go = run(
        ["go", "env", "GOVERSION"],
        cwd=REPOSITORY_ROOT,
        environment=environment,
    ).strip()
    if not selected_go:
        print("go env GOVERSION returned no version", file=sys.stderr)
        return 1

    ignored_rule = ".releasekit/"
    ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text(
        encoding="utf-8"
    ).splitlines()
    if ignored_rule not in ignore_lines:
        print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr)
        return 1

    module_bytes = {
        name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")
    }
    with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
        temporary_root = Path(directory)
        repository = temporary_root / "checkout"
        remote = temporary_root / "origin.git"
        suite_marker = temporary_root / "suite.jsonl"
        example_marker = temporary_root / "example.log"
        environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
        environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
        try:
            prepare_repository(
                repository, remote, environment, module_bytes, RELEASE_VERSION
            )
            verify_publication(
                repository,
                remote,
                temporary_root,
                environment,
                module_bytes,
                selected_go,
            )
            verify_usage_classification(
                repository / ".releasekit/bin/releasekit", temporary_root, environment
            )
            verify_validation_failure(
                temporary_root / "failure-checkout",
                temporary_root / "failure-origin.git",
                temporary_root,
                environment,
                module_bytes,
                selected_go,
            )
        except (RuntimeError, OSError, json.JSONDecodeError) as error:
            print(error, file=sys.stderr)
            return 1

    print("ReleaseKit publication success and failure fixtures passed")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())