eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Add prototype testing overlay configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 00:17:38 +00:00
parent 2c55bf5f1e
commit 2d8b441802
8 changed files with 104 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/library-adoption/config.go
View File

@@ -31,8 +31,9 @@ type LLMConfig struct {
OperationTimeout time.Duration `yaml:"operation_timeout"`
}
func loadConfig(source ck.Source, overrides ...ck.Override) (*ck.Result[Config], error) {
func loadConfig(source ck.Source, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
return ck.Load(source, ck.Options[Config]{
Overlay: overlay,
Overrides: overrides,
Definition: ck.Definition[Config]{
Defaults: func() Config {

90
examples/library-adoption/config_test.go
View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
@@ -46,7 +47,7 @@ func TestConfigurationContract(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("AUDITA_PROTOTYPE_KEY", tc.key)
loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original)))))
loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original)))), "")
if tc.bad {
if err == nil {
t.Fatal("expected rejection")
@@ -90,11 +91,11 @@ func TestConfigurationContract(t *testing.T) {
if err := os.Unsetenv("AUDITA_PROTOTYPE_KEY"); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(ck.Bytes("example", original)); err == nil {
if _, err := loadConfig(ck.Bytes("example", original), ""); err == nil {
t.Fatal("missing secret accepted")
}
t.Setenv("AUDITA_PROTOTYPE_KEY", "key")
loaded, err := loadConfig(ck.Bytes("example", original), intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"})
loaded, err := loadConfig(ck.Bytes("example", original), "", intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"})
if err != nil {
t.Fatal(err)
}
@@ -103,9 +104,90 @@ func TestConfigurationContract(t *testing.T) {
}
}
func TestSelectedOverlayConfiguration(t *testing.T) {
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
base, err := os.ReadFile("config.yaml")
if err != nil {
t.Fatal(err)
}
overlay, err := os.ReadFile("overlays/testing.yml")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name, base, overlay, selection, proposal, validation string
overrides []ck.Override
invalid bool
}{
{name: "omitted", base: string(base), overlay: string(overlay), proposal: "proposal", validation: "assessment"},
{name: "empty", base: string(base), overlay: string(overlay), selection: "", proposal: "proposal", validation: "assessment"},
{name: "default", base: string(base), overlay: string(overlay), selection: "default", proposal: "proposal", validation: "assessment"},
{name: "testing", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment-testing"},
{name: "proposal override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal", validation: "assessment-testing", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}},
{name: "validation override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment", overrides: []ck.Override{{Path: ck.Keys("llm", "validation_profile"), Value: "assessment", Flag: "--validation-profile"}}},
{name: "inherited overlay proposal", base: strings.Replace(string(base), " validation_profile: assessment\n", "", 1), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "proposal-testing"},
{name: "inherited override", base: strings.Replace(string(base), " validation_profile: assessment\n", "", 1), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal", validation: "proposal", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}},
{name: "explicit base assessment", base: string(base), overlay: "llm:\n proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "assessment"},
{name: "empty overlay assessment", base: string(base), overlay: "llm:\n validation_profile: ''\n", selection: "testing", invalid: true},
{name: "null overlay assessment", base: string(base), overlay: "llm:\n validation_profile: null\n", selection: "testing", invalid: true},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(tc.base), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(tc.overlay), 0600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection, tc.overrides...)
if tc.invalid {
if err == nil {
t.Fatal("expected invalid validation selection")
}
return
}
if err != nil {
t.Fatal(err)
}
if got := loaded.Config.LLM; got.ProposalProfile != tc.proposal || got.ValidationProfile != tc.validation {
t.Fatalf("roles = %q, %q; want %q, %q", got.ProposalProfile, got.ValidationProfile, tc.proposal, tc.validation)
}
for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} {
data, err := os.ReadFile(filepath.Join("profiles", name))
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil {
t.Fatal(err)
}
}
r, err := newRuntime(loaded.Config, dir, demonstrationClient())
if err != nil {
t.Fatal(err)
}
for role, selected := range map[string]string{"proposal": tc.proposal, "assessment": tc.validation} {
profile := r.Profiles[role]
if profile.ProfileID != selected || profile.EffectiveModelParams.BackendID != "offline" {
t.Fatalf("%s resolved to %+v", role, profile)
}
models := map[string]string{"proposal": "synthetic-grammar", "assessment": "synthetic-review", "proposal-testing": "synthetic-grammar-testing", "assessment-testing": "synthetic-review-testing"}
if profile.EffectiveModelParams.Model != models[selected] {
t.Fatalf("%s model = %q", role, profile.EffectiveModelParams.Model)
}
}
})
}
}
func TestProfilesRespectSharedBackendBoundary(t *testing.T) {
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
loaded, err := loadConfig(ck.File("config.yaml"))
loaded, err := loadConfig(ck.File("config.yaml"), "")
if err != nil {
t.Fatal(err)
}

2
examples/library-adoption/main.go
View File

@@ -16,7 +16,7 @@ func main() {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
loaded, err := loadConfig(ck.File("config.yaml"))
loaded, err := loadConfig(ck.File("config.yaml"), "")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)

3
examples/library-adoption/overlays/testing.yml Normal file
View File

@@ -0,0 +1,3 @@
llm:
proposal_profile: proposal-testing
validation_profile: assessment-testing

4
examples/library-adoption/profiles/assessment-testing.yaml Normal file
View File

@@ -0,0 +1,4 @@
id: assessment-testing
backend: offline
model: synthetic-review-testing
timeout_seconds: 600

4
examples/library-adoption/profiles/proposal-testing.yaml Normal file
View File

@@ -0,0 +1,4 @@
id: proposal-testing
backend: offline
model: synthetic-grammar-testing
timeout_seconds: 600

2
examples/library-adoption/workflow_test.go
View File

@@ -22,7 +22,7 @@ type step struct {
func testRuntime(t *testing.T, transport http.RoundTripper, overrides ...ck.Override) *Runtime {
t.Helper()
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
loaded, err := loadConfig(ck.File("config.yaml"), overrides...)
loaded, err := loadConfig(ck.File("config.yaml"), "", overrides...)
if err != nil {
t.Fatal(err)
}