Verify prototype overlay boundaries

docs/roadmap/implementation.md

@@ -177,7 +177,10 @@ measured price, speed, or model quality.
 
 ## Stage 4: Verify Failure, Credential, and Inspection Boundaries
 
-**Status:** Pending. **Depends on:** Stage 3.
+**Status:** Complete (2026-09-24). **Depends on:** Stage 3. Focused offline tests
+cover ConfigKit overlay errors, backend validation, credential replacement and
+null handling, provenance, inspection, and redaction. Nested `go test -race ./...`
+and `go vet ./...` passed on linux/amd64 with Go 1.26.8.
 
 Extend focused tests in the prototype; keep them offline and independent of host
 configuration and credentials. Exercise the consumer's assembled loader/runtime

examples/library-adoption/README.md

@@ -36,6 +36,11 @@ from `profiles/`. An explicitly supplied `--proposal-profile` or
 requested overlay and each effective profile, backend, and model. Synthetic model
 IDs demonstrate operator-selected routing; they are not measurements of price,
 speed, or model quality.
+An overlay can also change ordinary backend settings and credential references.
+ConfigKit resolves only the surviving reference at a replaced credential path;
+changing role profiles alone leaves other configured backend credential references
+in place, and unresolved references still fail loading. Explicit `null` credentials
+select unauthenticated backend use.
 
 Tests control provider responses, capacity, and cancellation. They cover positive
 repair budgets for both roles, exhausted repair, domain rejection, transient retry,

examples/library-adoption/overlay_boundary_test.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+)
+
+func overlayLayout(t *testing.T, base, overlay string) string {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(base), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if overlay != "" {
+		if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} {
+		data, err := os.ReadFile(filepath.Join("profiles", name))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+func fixtureConfig(t *testing.T) string {
+	t.Helper()
+	data, err := os.ReadFile("config.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(data)
+}
+
+func requireLoadCode(t *testing.T, err error, code ck.Code) {
+	t.Helper()
+	var loadErr *ck.LoadError
+	if !errors.As(err, &loadErr) {
+		t.Fatalf("expected ConfigKit load error, got %v", err)
+	}
+	for _, diagnostic := range loadErr.Diagnostics() {
+		if diagnostic.Code == code {
+			return
+		}
+	}
+	t.Fatalf("expected diagnostic %s, got %+v", code, loadErr.Diagnostics())
+}
+
+func TestOverlayFailuresAndApplicationValidation(t *testing.T) {
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
+	base := fixtureConfig(t)
+	for _, tc := range []struct {
+		name, selection, overlay string
+		code                     ck.Code
+	}{
+		{"missing", "missing", "", ck.CodeOverlayMissing},
+		{"invalid name", "../testing", "", ck.CodeOptionsInvalid},
+		{"version", "testing", "version: 3\n", ck.CodeOverlayVersion},
+		{"directive", "testing", "configkit: {}\n", ck.CodeOverlayDirective},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := overlayLayout(t, base, tc.overlay)
+			_, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection)
+			requireLoadCode(t, err, tc.code)
+		})
+	}
+	valid := "backends:\n  offline:\n    concurrency: 2\n"
+	dir := overlayLayout(t, base, valid)
+	loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := loaded.Config.Backends["offline"].Concurrency; got != 2 {
+		t.Fatalf("overlay concurrency = %d", got)
+	}
+	dir = overlayLayout(t, base, "backends:\n  offline:\n    concurrency: 0\n")
+	if _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing"); err == nil {
+		t.Fatal("invalid effective backend limit accepted")
+	}
+}
+
+func TestOverlayCredentialResolutionAndInspection(t *testing.T) {
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "")
+	t.Setenv("AUDITA_OVERLAY_KEY", "replacement-synthetic-secret")
+	base := fixtureConfig(t)
+	for _, tc := range []struct {
+		name, overlay, key  string
+		unsetKey            bool
+		wantError, wantAuth bool
+	}{
+		{"replacement", "backends:\n  offline:\n    credential: AUDITA_OVERLAY_KEY\n", "replacement-synthetic-secret", false, false, true},
+		{"surviving missing", "backends:\n  offline:\n    credential: AUDITA_OVERLAY_KEY\n", "", true, true, false},
+		{"surviving empty", "backends:\n  offline:\n    credential: AUDITA_OVERLAY_KEY\n", "", false, true, false},
+		{"unauthenticated null", "backends:\n  offline:\n    credential: null\n", "replacement-synthetic-secret", false, false, false},
+		{"profile switch retains credential", "llm:\n  proposal_profile: proposal-testing\n  validation_profile: assessment-testing\n", "replacement-synthetic-secret", false, true, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("AUDITA_OVERLAY_KEY", tc.key)
+			if tc.unsetKey {
+				if err := os.Unsetenv("AUDITA_OVERLAY_KEY"); err != nil {
+					t.Fatal(err)
+				}
+			}
+			dir := overlayLayout(t, base, tc.overlay)
+			loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
+			if tc.wantError {
+				if err == nil {
+					t.Fatal("unresolved surviving credential accepted")
+				}
+				if strings.Contains(err.Error(), "replacement-synthetic-secret") {
+					t.Fatal("error exposed secret")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			var effective, explanation bytes.Buffer
+			if err := loaded.WriteEffective(&effective); err != nil {
+				t.Fatal(err)
+			}
+			if err := loaded.WriteExplanation(&explanation, ck.Keys("backends", "offline", "credential")); err != nil {
+				t.Fatal(err)
+			}
+			if strings.Contains(effective.String()+explanation.String(), "replacement-synthetic-secret") {
+				t.Fatal("inspection exposed secret")
+			}
+			observed := 0
+			baseClient := demonstrationClient()
+			client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+				want := ""
+				if tc.wantAuth {
+					want = "Bearer replacement-synthetic-secret"
+				}
+				if req.Header.Get("Authorization") != want {
+					t.Errorf("authorization mismatch")
+				}
+				observed++
+				return baseClient.Transport.RoundTrip(req)
+			})}
+			r, err := newRuntime(loaded.Config, dir, client)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, role := range []string{"proposal", "assessment"} {
+				p := r.Profiles[role]
+				raw, _ := json.Marshal(p)
+				if strings.Contains(string(raw), "replacement-synthetic-secret") {
+					t.Fatal("profile exposed secret")
+				}
+			}
+			recorder := &Recorder{}
+			result, err := r.workflow(context.Background(), recorder.observer)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if result.Accepted != 1 || observed != 4 {
+				t.Fatalf("result %+v; calls %d", result, observed)
+			}
+			raw, _ := json.Marshal(recorder.snapshot())
+			if strings.Contains(string(raw), "replacement-synthetic-secret") {
+				t.Fatal("observations exposed secret")
+			}
+		})
+	}
+}
+
+func TestOverlaySelectionProvenance(t *testing.T) {
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
+	dir := overlayLayout(t, fixtureConfig(t), "llm:\n  proposal_profile: proposal-testing\n  validation_profile: assessment-testing\n")
+	loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range []struct {
+		key   string
+		layer ck.Layer
+	}{{"proposal_profile", ck.LayerCLI}, {"validation_profile", ck.LayerOverlay}} {
+		path := ck.Keys("llm", tc.key)
+		info, ok := loaded.Metadata().Lookup(path)
+		if !ok || info.Origin.Layer != tc.layer {
+			t.Fatalf("%s metadata = %+v", tc.key, info)
+		}
+		explain, ok := loaded.Explain(path)
+		if !ok || explain.Origin.Layer != tc.layer {
+			t.Fatalf("%s explanation = %+v", tc.key, explain)
+		}
+	}
+	var effective, explanation bytes.Buffer
+	if err := loaded.WriteEffective(&effective); err != nil {
+		t.Fatal(err)
+	}
+	if err := loaded.WriteExplanation(&explanation, ck.Keys("llm", "validation_profile")); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(effective.String(), "assessment-testing") || !strings.Contains(explanation.String(), "overlay") {
+		t.Fatal("overlay inspection missing")
+	}
+	seenOverlay := false
+	for _, source := range loaded.Sources() {
+		if source.Layer == ck.LayerOverlay {
+			seenOverlay = true
+		}
+	}
+	if !seenOverlay {
+		t.Fatal("overlay source missing")
+	}
+	r, err := newRuntime(loaded.Config, dir, demonstrationClient())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Profiles["proposal"].ProfileID != "proposal" || r.Profiles["assessment"].ProfileID != "assessment-testing" || r.Profiles["assessment"].EffectiveModelParams.Model != "synthetic-review-testing" {
+		t.Fatal("inspection/runtime selection diverged")
+	}
+	if r.Profiles["proposal"].EffectiveModelParams.Model != "synthetic-grammar" || r.Profiles["proposal"].EffectiveModelParams.BackendID != "offline" || r.Profiles["assessment"].EffectiveModelParams.BackendID != "offline" {
+		t.Fatal("inspected profile target diverged")
+	}
+}