From 7b809878d6c8f0df89825a8de0c3cd411acd894c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 00:22:56 +0000 Subject: [PATCH] Verify prototype overlay boundaries --- docs/roadmap/implementation.md | 5 +- examples/library-adoption/README.md | 5 + .../library-adoption/overlay_boundary_test.go | 238 ++++++++++++++++++ 3 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 examples/library-adoption/overlay_boundary_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index c2080cb..f6ea073 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -177,7 +177,10 @@ measured price, speed, or model quality. ## Stage 4: Verify Failure, Credential, and Inspection Boundaries -**Status:** Pending. **Depends on:** Stage 3. +**Status:** Complete (2026-09-24). **Depends on:** Stage 3. Focused offline tests +cover ConfigKit overlay errors, backend validation, credential replacement and +null handling, provenance, inspection, and redaction. Nested `go test -race ./...` +and `go vet ./...` passed on linux/amd64 with Go 1.26.8. Extend focused tests in the prototype; keep them offline and independent of host configuration and credentials. Exercise the consumer's assembled loader/runtime diff --git a/examples/library-adoption/README.md b/examples/library-adoption/README.md index da39cfc..48dda41 100644 --- a/examples/library-adoption/README.md +++ b/examples/library-adoption/README.md @@ -36,6 +36,11 @@ from `profiles/`. An explicitly supplied `--proposal-profile` or requested overlay and each effective profile, backend, and model. Synthetic model IDs demonstrate operator-selected routing; they are not measurements of price, speed, or model quality. +An overlay can also change ordinary backend settings and credential references. +ConfigKit resolves only the surviving reference at a replaced credential path; +changing role profiles alone leaves other configured backend credential references +in place, and unresolved references still fail loading. Explicit `null` credentials +select unauthenticated backend use. Tests control provider responses, capacity, and cancellation. They cover positive repair budgets for both roles, exhausted repair, domain rejection, transient retry, diff --git a/examples/library-adoption/overlay_boundary_test.go b/examples/library-adoption/overlay_boundary_test.go new file mode 100644 index 0000000..f211b0b --- /dev/null +++ b/examples/library-adoption/overlay_boundary_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + ck "gitea.maximumdirect.net/eric/ConfigKit" +) + +func overlayLayout(t *testing.T, base, overlay string) string { + t.Helper() + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(base), 0600); err != nil { + t.Fatal(err) + } + if overlay != "" { + if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} { + data, err := os.ReadFile(filepath.Join("profiles", name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +func fixtureConfig(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("config.yaml") + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func requireLoadCode(t *testing.T, err error, code ck.Code) { + t.Helper() + var loadErr *ck.LoadError + if !errors.As(err, &loadErr) { + t.Fatalf("expected ConfigKit load error, got %v", err) + } + for _, diagnostic := range loadErr.Diagnostics() { + if diagnostic.Code == code { + return + } + } + t.Fatalf("expected diagnostic %s, got %+v", code, loadErr.Diagnostics()) +} + +func TestOverlayFailuresAndApplicationValidation(t *testing.T) { + t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret") + base := fixtureConfig(t) + for _, tc := range []struct { + name, selection, overlay string + code ck.Code + }{ + {"missing", "missing", "", ck.CodeOverlayMissing}, + {"invalid name", "../testing", "", ck.CodeOptionsInvalid}, + {"version", "testing", "version: 3\n", ck.CodeOverlayVersion}, + {"directive", "testing", "configkit: {}\n", ck.CodeOverlayDirective}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := overlayLayout(t, base, tc.overlay) + _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection) + requireLoadCode(t, err, tc.code) + }) + } + valid := "backends:\n offline:\n concurrency: 2\n" + dir := overlayLayout(t, base, valid) + loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing") + if err != nil { + t.Fatal(err) + } + if got := loaded.Config.Backends["offline"].Concurrency; got != 2 { + t.Fatalf("overlay concurrency = %d", got) + } + dir = overlayLayout(t, base, "backends:\n offline:\n concurrency: 0\n") + if _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing"); err == nil { + t.Fatal("invalid effective backend limit accepted") + } +} + +func TestOverlayCredentialResolutionAndInspection(t *testing.T) { + t.Setenv("AUDITA_PROTOTYPE_KEY", "") + t.Setenv("AUDITA_OVERLAY_KEY", "replacement-synthetic-secret") + base := fixtureConfig(t) + for _, tc := range []struct { + name, overlay, key string + unsetKey bool + wantError, wantAuth bool + }{ + {"replacement", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "replacement-synthetic-secret", false, false, true}, + {"surviving missing", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", true, true, false}, + {"surviving empty", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", false, true, false}, + {"unauthenticated null", "backends:\n offline:\n credential: null\n", "replacement-synthetic-secret", false, false, false}, + {"profile switch retains credential", "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n", "replacement-synthetic-secret", false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("AUDITA_OVERLAY_KEY", tc.key) + if tc.unsetKey { + if err := os.Unsetenv("AUDITA_OVERLAY_KEY"); err != nil { + t.Fatal(err) + } + } + dir := overlayLayout(t, base, tc.overlay) + loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing") + if tc.wantError { + if err == nil { + t.Fatal("unresolved surviving credential accepted") + } + if strings.Contains(err.Error(), "replacement-synthetic-secret") { + t.Fatal("error exposed secret") + } + return + } + if err != nil { + t.Fatal(err) + } + var effective, explanation bytes.Buffer + if err := loaded.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + if err := loaded.WriteExplanation(&explanation, ck.Keys("backends", "offline", "credential")); err != nil { + t.Fatal(err) + } + if strings.Contains(effective.String()+explanation.String(), "replacement-synthetic-secret") { + t.Fatal("inspection exposed secret") + } + observed := 0 + baseClient := demonstrationClient() + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + want := "" + if tc.wantAuth { + want = "Bearer replacement-synthetic-secret" + } + if req.Header.Get("Authorization") != want { + t.Errorf("authorization mismatch") + } + observed++ + return baseClient.Transport.RoundTrip(req) + })} + r, err := newRuntime(loaded.Config, dir, client) + if err != nil { + t.Fatal(err) + } + for _, role := range []string{"proposal", "assessment"} { + p := r.Profiles[role] + raw, _ := json.Marshal(p) + if strings.Contains(string(raw), "replacement-synthetic-secret") { + t.Fatal("profile exposed secret") + } + } + recorder := &Recorder{} + result, err := r.workflow(context.Background(), recorder.observer) + if err != nil { + t.Fatal(err) + } + if result.Accepted != 1 || observed != 4 { + t.Fatalf("result %+v; calls %d", result, observed) + } + raw, _ := json.Marshal(recorder.snapshot()) + if strings.Contains(string(raw), "replacement-synthetic-secret") { + t.Fatal("observations exposed secret") + } + }) + } +} + +func TestOverlaySelectionProvenance(t *testing.T) { + t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret") + dir := overlayLayout(t, fixtureConfig(t), "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n") + loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + key string + layer ck.Layer + }{{"proposal_profile", ck.LayerCLI}, {"validation_profile", ck.LayerOverlay}} { + path := ck.Keys("llm", tc.key) + info, ok := loaded.Metadata().Lookup(path) + if !ok || info.Origin.Layer != tc.layer { + t.Fatalf("%s metadata = %+v", tc.key, info) + } + explain, ok := loaded.Explain(path) + if !ok || explain.Origin.Layer != tc.layer { + t.Fatalf("%s explanation = %+v", tc.key, explain) + } + } + var effective, explanation bytes.Buffer + if err := loaded.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + if err := loaded.WriteExplanation(&explanation, ck.Keys("llm", "validation_profile")); err != nil { + t.Fatal(err) + } + if !strings.Contains(effective.String(), "assessment-testing") || !strings.Contains(explanation.String(), "overlay") { + t.Fatal("overlay inspection missing") + } + seenOverlay := false + for _, source := range loaded.Sources() { + if source.Layer == ck.LayerOverlay { + seenOverlay = true + } + } + if !seenOverlay { + t.Fatal("overlay source missing") + } + r, err := newRuntime(loaded.Config, dir, demonstrationClient()) + if err != nil { + t.Fatal(err) + } + if r.Profiles["proposal"].ProfileID != "proposal" || r.Profiles["assessment"].ProfileID != "assessment-testing" || r.Profiles["assessment"].EffectiveModelParams.Model != "synthetic-review-testing" { + t.Fatal("inspection/runtime selection diverged") + } + if r.Profiles["proposal"].EffectiveModelParams.Model != "synthetic-grammar" || r.Profiles["proposal"].EffectiveModelParams.BackendID != "offline" || r.Profiles["assessment"].EffectiveModelParams.BackendID != "offline" { + t.Fatal("inspected profile target diverged") + } +}