Verify prototype overlay boundaries
This commit is contained in:
@@ -177,7 +177,10 @@ measured price, speed, or model quality.
|
||||
|
||||
## Stage 4: Verify Failure, Credential, and Inspection Boundaries
|
||||
|
||||
**Status:** Pending. **Depends on:** Stage 3.
|
||||
**Status:** Complete (2026-09-24). **Depends on:** Stage 3. Focused offline tests
|
||||
cover ConfigKit overlay errors, backend validation, credential replacement and
|
||||
null handling, provenance, inspection, and redaction. Nested `go test -race ./...`
|
||||
and `go vet ./...` passed on linux/amd64 with Go 1.26.8.
|
||||
|
||||
Extend focused tests in the prototype; keep them offline and independent of host
|
||||
configuration and credentials. Exercise the consumer's assembled loader/runtime
|
||||
|
||||
@@ -36,6 +36,11 @@ from `profiles/`. An explicitly supplied `--proposal-profile` or
|
||||
requested overlay and each effective profile, backend, and model. Synthetic model
|
||||
IDs demonstrate operator-selected routing; they are not measurements of price,
|
||||
speed, or model quality.
|
||||
An overlay can also change ordinary backend settings and credential references.
|
||||
ConfigKit resolves only the surviving reference at a replaced credential path;
|
||||
changing role profiles alone leaves other configured backend credential references
|
||||
in place, and unresolved references still fail loading. Explicit `null` credentials
|
||||
select unauthenticated backend use.
|
||||
|
||||
Tests control provider responses, capacity, and cancellation. They cover positive
|
||||
repair budgets for both roles, exhausted repair, domain rejection, transient retry,
|
||||
|
||||
238
examples/library-adoption/overlay_boundary_test.go
Normal file
238
examples/library-adoption/overlay_boundary_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func overlayLayout(t *testing.T, base, overlay string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(base), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if overlay != "" {
|
||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} {
|
||||
data, err := os.ReadFile(filepath.Join("profiles", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func fixtureConfig(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("config.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func requireLoadCode(t *testing.T, err error, code ck.Code) {
|
||||
t.Helper()
|
||||
var loadErr *ck.LoadError
|
||||
if !errors.As(err, &loadErr) {
|
||||
t.Fatalf("expected ConfigKit load error, got %v", err)
|
||||
}
|
||||
for _, diagnostic := range loadErr.Diagnostics() {
|
||||
if diagnostic.Code == code {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("expected diagnostic %s, got %+v", code, loadErr.Diagnostics())
|
||||
}
|
||||
|
||||
func TestOverlayFailuresAndApplicationValidation(t *testing.T) {
|
||||
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
|
||||
base := fixtureConfig(t)
|
||||
for _, tc := range []struct {
|
||||
name, selection, overlay string
|
||||
code ck.Code
|
||||
}{
|
||||
{"missing", "missing", "", ck.CodeOverlayMissing},
|
||||
{"invalid name", "../testing", "", ck.CodeOptionsInvalid},
|
||||
{"version", "testing", "version: 3\n", ck.CodeOverlayVersion},
|
||||
{"directive", "testing", "configkit: {}\n", ck.CodeOverlayDirective},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := overlayLayout(t, base, tc.overlay)
|
||||
_, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection)
|
||||
requireLoadCode(t, err, tc.code)
|
||||
})
|
||||
}
|
||||
valid := "backends:\n offline:\n concurrency: 2\n"
|
||||
dir := overlayLayout(t, base, valid)
|
||||
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loaded.Config.Backends["offline"].Concurrency; got != 2 {
|
||||
t.Fatalf("overlay concurrency = %d", got)
|
||||
}
|
||||
dir = overlayLayout(t, base, "backends:\n offline:\n concurrency: 0\n")
|
||||
if _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing"); err == nil {
|
||||
t.Fatal("invalid effective backend limit accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverlayCredentialResolutionAndInspection(t *testing.T) {
|
||||
t.Setenv("AUDITA_PROTOTYPE_KEY", "")
|
||||
t.Setenv("AUDITA_OVERLAY_KEY", "replacement-synthetic-secret")
|
||||
base := fixtureConfig(t)
|
||||
for _, tc := range []struct {
|
||||
name, overlay, key string
|
||||
unsetKey bool
|
||||
wantError, wantAuth bool
|
||||
}{
|
||||
{"replacement", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "replacement-synthetic-secret", false, false, true},
|
||||
{"surviving missing", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", true, true, false},
|
||||
{"surviving empty", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", false, true, false},
|
||||
{"unauthenticated null", "backends:\n offline:\n credential: null\n", "replacement-synthetic-secret", false, false, false},
|
||||
{"profile switch retains credential", "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n", "replacement-synthetic-secret", false, true, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Setenv("AUDITA_OVERLAY_KEY", tc.key)
|
||||
if tc.unsetKey {
|
||||
if err := os.Unsetenv("AUDITA_OVERLAY_KEY"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dir := overlayLayout(t, base, tc.overlay)
|
||||
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("unresolved surviving credential accepted")
|
||||
}
|
||||
if strings.Contains(err.Error(), "replacement-synthetic-secret") {
|
||||
t.Fatal("error exposed secret")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var effective, explanation bytes.Buffer
|
||||
if err := loaded.WriteEffective(&effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := loaded.WriteExplanation(&explanation, ck.Keys("backends", "offline", "credential")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(effective.String()+explanation.String(), "replacement-synthetic-secret") {
|
||||
t.Fatal("inspection exposed secret")
|
||||
}
|
||||
observed := 0
|
||||
baseClient := demonstrationClient()
|
||||
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
want := ""
|
||||
if tc.wantAuth {
|
||||
want = "Bearer replacement-synthetic-secret"
|
||||
}
|
||||
if req.Header.Get("Authorization") != want {
|
||||
t.Errorf("authorization mismatch")
|
||||
}
|
||||
observed++
|
||||
return baseClient.Transport.RoundTrip(req)
|
||||
})}
|
||||
r, err := newRuntime(loaded.Config, dir, client)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, role := range []string{"proposal", "assessment"} {
|
||||
p := r.Profiles[role]
|
||||
raw, _ := json.Marshal(p)
|
||||
if strings.Contains(string(raw), "replacement-synthetic-secret") {
|
||||
t.Fatal("profile exposed secret")
|
||||
}
|
||||
}
|
||||
recorder := &Recorder{}
|
||||
result, err := r.workflow(context.Background(), recorder.observer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Accepted != 1 || observed != 4 {
|
||||
t.Fatalf("result %+v; calls %d", result, observed)
|
||||
}
|
||||
raw, _ := json.Marshal(recorder.snapshot())
|
||||
if strings.Contains(string(raw), "replacement-synthetic-secret") {
|
||||
t.Fatal("observations exposed secret")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverlaySelectionProvenance(t *testing.T) {
|
||||
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
|
||||
dir := overlayLayout(t, fixtureConfig(t), "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n")
|
||||
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
key string
|
||||
layer ck.Layer
|
||||
}{{"proposal_profile", ck.LayerCLI}, {"validation_profile", ck.LayerOverlay}} {
|
||||
path := ck.Keys("llm", tc.key)
|
||||
info, ok := loaded.Metadata().Lookup(path)
|
||||
if !ok || info.Origin.Layer != tc.layer {
|
||||
t.Fatalf("%s metadata = %+v", tc.key, info)
|
||||
}
|
||||
explain, ok := loaded.Explain(path)
|
||||
if !ok || explain.Origin.Layer != tc.layer {
|
||||
t.Fatalf("%s explanation = %+v", tc.key, explain)
|
||||
}
|
||||
}
|
||||
var effective, explanation bytes.Buffer
|
||||
if err := loaded.WriteEffective(&effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := loaded.WriteExplanation(&explanation, ck.Keys("llm", "validation_profile")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(effective.String(), "assessment-testing") || !strings.Contains(explanation.String(), "overlay") {
|
||||
t.Fatal("overlay inspection missing")
|
||||
}
|
||||
seenOverlay := false
|
||||
for _, source := range loaded.Sources() {
|
||||
if source.Layer == ck.LayerOverlay {
|
||||
seenOverlay = true
|
||||
}
|
||||
}
|
||||
if !seenOverlay {
|
||||
t.Fatal("overlay source missing")
|
||||
}
|
||||
r, err := newRuntime(loaded.Config, dir, demonstrationClient())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Profiles["proposal"].ProfileID != "proposal" || r.Profiles["assessment"].ProfileID != "assessment-testing" || r.Profiles["assessment"].EffectiveModelParams.Model != "synthetic-review-testing" {
|
||||
t.Fatal("inspection/runtime selection diverged")
|
||||
}
|
||||
if r.Profiles["proposal"].EffectiveModelParams.Model != "synthetic-grammar" || r.Profiles["proposal"].EffectiveModelParams.BackendID != "offline" || r.Profiles["assessment"].EffectiveModelParams.BackendID != "offline" {
|
||||
t.Fatal("inspected profile target diverged")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user