From 2d8b441802477707cdfc4d0022e0f69da394cb8e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 00:17:38 +0000 Subject: [PATCH] Add prototype testing overlay configuration --- docs/roadmap/implementation.md | 4 +- examples/library-adoption/config.go | 3 +- examples/library-adoption/config_test.go | 90 ++++++++++++++++++- examples/library-adoption/main.go | 2 +- .../library-adoption/overlays/testing.yml | 3 + .../profiles/assessment-testing.yaml | 4 + .../profiles/proposal-testing.yaml | 4 + examples/library-adoption/workflow_test.go | 2 +- 8 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 examples/library-adoption/overlays/testing.yml create mode 100644 examples/library-adoption/profiles/assessment-testing.yaml create mode 100644 examples/library-adoption/profiles/proposal-testing.yaml diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 4816708..5395fe0 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -74,7 +74,9 @@ historical v0.3.1 evidence. Do not claim overlay integration yet. ## Stage 2: Add Overlay Loading and Configuration Fixtures -**Status:** Pending. **Depends on:** Stage 1. +**Status:** Complete (2026-09-24). **Depends on:** Stage 1. On-disk overlay and +profile fixtures, precedence, inheritance, rejection, and runtime assembly tests +pass in the nested module (`go test ./...`). Start with `config.go`, `config_test.go`, `config.yaml`, and `profiles/`. Extend `loadConfig` with an explicit overlay-selection argument or a small invocation diff --git a/examples/library-adoption/config.go b/examples/library-adoption/config.go index 2a241bc..3da5d01 100644 --- a/examples/library-adoption/config.go +++ b/examples/library-adoption/config.go @@ -31,8 +31,9 @@ type LLMConfig struct { OperationTimeout time.Duration `yaml:"operation_timeout"` } -func loadConfig(source ck.Source, overrides ...ck.Override) (*ck.Result[Config], error) { +func loadConfig(source ck.Source, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) { return ck.Load(source, ck.Options[Config]{ + Overlay: overlay, Overrides: overrides, Definition: ck.Definition[Config]{ Defaults: func() Config { diff --git a/examples/library-adoption/config_test.go b/examples/library-adoption/config_test.go index f7afbb2..c206e85 100644 --- a/examples/library-adoption/config_test.go +++ b/examples/library-adoption/config_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "os" + "path/filepath" "strings" "testing" @@ -46,7 +47,7 @@ func TestConfigurationContract(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { t.Setenv("AUDITA_PROTOTYPE_KEY", tc.key) - loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original))))) + loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original)))), "") if tc.bad { if err == nil { t.Fatal("expected rejection") @@ -90,11 +91,11 @@ func TestConfigurationContract(t *testing.T) { if err := os.Unsetenv("AUDITA_PROTOTYPE_KEY"); err != nil { t.Fatal(err) } - if _, err := loadConfig(ck.Bytes("example", original)); err == nil { + if _, err := loadConfig(ck.Bytes("example", original), ""); err == nil { t.Fatal("missing secret accepted") } t.Setenv("AUDITA_PROTOTYPE_KEY", "key") - loaded, err := loadConfig(ck.Bytes("example", original), intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"}) + loaded, err := loadConfig(ck.Bytes("example", original), "", intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"}) if err != nil { t.Fatal(err) } @@ -103,9 +104,90 @@ func TestConfigurationContract(t *testing.T) { } } +func TestSelectedOverlayConfiguration(t *testing.T) { + t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret") + base, err := os.ReadFile("config.yaml") + if err != nil { + t.Fatal(err) + } + overlay, err := os.ReadFile("overlays/testing.yml") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, base, overlay, selection, proposal, validation string + overrides []ck.Override + invalid bool + }{ + {name: "omitted", base: string(base), overlay: string(overlay), proposal: "proposal", validation: "assessment"}, + {name: "empty", base: string(base), overlay: string(overlay), selection: "", proposal: "proposal", validation: "assessment"}, + {name: "default", base: string(base), overlay: string(overlay), selection: "default", proposal: "proposal", validation: "assessment"}, + {name: "testing", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment-testing"}, + {name: "proposal override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal", validation: "assessment-testing", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}}, + {name: "validation override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment", overrides: []ck.Override{{Path: ck.Keys("llm", "validation_profile"), Value: "assessment", Flag: "--validation-profile"}}}, + {name: "inherited overlay proposal", base: strings.Replace(string(base), " validation_profile: assessment\n", "", 1), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "proposal-testing"}, + {name: "inherited override", base: strings.Replace(string(base), " validation_profile: assessment\n", "", 1), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal", validation: "proposal", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}}, + {name: "explicit base assessment", base: string(base), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "assessment"}, + {name: "empty overlay assessment", base: string(base), overlay: "llm:\n validation_profile: ''\n", selection: "testing", invalid: true}, + {name: "null overlay assessment", base: string(base), overlay: "llm:\n validation_profile: null\n", selection: "testing", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(tc.base), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(tc.overlay), 0600); err != nil { + t.Fatal(err) + } + loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection, tc.overrides...) + if tc.invalid { + if err == nil { + t.Fatal("expected invalid validation selection") + } + return + } + if err != nil { + t.Fatal(err) + } + if got := loaded.Config.LLM; got.ProposalProfile != tc.proposal || got.ValidationProfile != tc.validation { + t.Fatalf("roles = %q, %q; want %q, %q", got.ProposalProfile, got.ValidationProfile, tc.proposal, tc.validation) + } + for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} { + data, err := os.ReadFile(filepath.Join("profiles", name)) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil { + t.Fatal(err) + } + } + r, err := newRuntime(loaded.Config, dir, demonstrationClient()) + if err != nil { + t.Fatal(err) + } + for role, selected := range map[string]string{"proposal": tc.proposal, "assessment": tc.validation} { + profile := r.Profiles[role] + if profile.ProfileID != selected || profile.EffectiveModelParams.BackendID != "offline" { + t.Fatalf("%s resolved to %+v", role, profile) + } + models := map[string]string{"proposal": "synthetic-grammar", "assessment": "synthetic-review", "proposal-testing": "synthetic-grammar-testing", "assessment-testing": "synthetic-review-testing"} + if profile.EffectiveModelParams.Model != models[selected] { + t.Fatalf("%s model = %q", role, profile.EffectiveModelParams.Model) + } + } + }) + } +} + func TestProfilesRespectSharedBackendBoundary(t *testing.T) { t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret") - loaded, err := loadConfig(ck.File("config.yaml")) + loaded, err := loadConfig(ck.File("config.yaml"), "") if err != nil { t.Fatal(err) } diff --git a/examples/library-adoption/main.go b/examples/library-adoption/main.go index 4a6baf7..70d5e23 100644 --- a/examples/library-adoption/main.go +++ b/examples/library-adoption/main.go @@ -16,7 +16,7 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - loaded, err := loadConfig(ck.File("config.yaml")) + loaded, err := loadConfig(ck.File("config.yaml"), "") if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/examples/library-adoption/overlays/testing.yml b/examples/library-adoption/overlays/testing.yml new file mode 100644 index 0000000..72c3717 --- /dev/null +++ b/examples/library-adoption/overlays/testing.yml @@ -0,0 +1,3 @@ +llm: + proposal_profile: proposal-testing + validation_profile: assessment-testing diff --git a/examples/library-adoption/profiles/assessment-testing.yaml b/examples/library-adoption/profiles/assessment-testing.yaml new file mode 100644 index 0000000..80d895d --- /dev/null +++ b/examples/library-adoption/profiles/assessment-testing.yaml @@ -0,0 +1,4 @@ +id: assessment-testing +backend: offline +model: synthetic-review-testing +timeout_seconds: 600 diff --git a/examples/library-adoption/profiles/proposal-testing.yaml b/examples/library-adoption/profiles/proposal-testing.yaml new file mode 100644 index 0000000..d8615a5 --- /dev/null +++ b/examples/library-adoption/profiles/proposal-testing.yaml @@ -0,0 +1,4 @@ +id: proposal-testing +backend: offline +model: synthetic-grammar-testing +timeout_seconds: 600 diff --git a/examples/library-adoption/workflow_test.go b/examples/library-adoption/workflow_test.go index 03fe5dc..fdfb343 100644 --- a/examples/library-adoption/workflow_test.go +++ b/examples/library-adoption/workflow_test.go @@ -22,7 +22,7 @@ type step struct { func testRuntime(t *testing.T, transport http.RoundTripper, overrides ...ck.Override) *Runtime { t.Helper() t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret") - loaded, err := loadConfig(ck.File("config.yaml"), overrides...) + loaded, err := loadConfig(ck.File("config.yaml"), "", overrides...) if err != nil { t.Fatal(err) }