package main import ( "bytes" "context" "encoding/json" "errors" "net/http" "os" "path/filepath" "strings" "testing" ck "gitea.maximumdirect.net/eric/ConfigKit" ) func overlayLayout(t *testing.T, base, overlay string) string { t.Helper() dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(base), 0600); err != nil { t.Fatal(err) } if overlay != "" { if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil { t.Fatal(err) } } for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} { data, err := os.ReadFile(filepath.Join("profiles", name)) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil { t.Fatal(err) } } return dir } func fixtureConfig(t *testing.T) string { t.Helper() data, err := os.ReadFile("config.yaml") if err != nil { t.Fatal(err) } return string(data) } func requireLoadCode(t *testing.T, err error, code ck.Code) { t.Helper() var loadErr *ck.LoadError if !errors.As(err, &loadErr) { t.Fatalf("expected ConfigKit load error, got %v", err) } for _, diagnostic := range loadErr.Diagnostics() { if diagnostic.Code == code { return } } t.Fatalf("expected diagnostic %s, got %+v", code, loadErr.Diagnostics()) } func TestOverlayFailuresAndApplicationValidation(t *testing.T) { t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret") base := fixtureConfig(t) for _, tc := range []struct { name, selection, overlay string code ck.Code }{ {"missing", "missing", "", ck.CodeOverlayMissing}, {"invalid name", "../testing", "", ck.CodeOptionsInvalid}, {"version", "testing", "version: 3\n", ck.CodeOverlayVersion}, {"directive", "testing", "configkit: {}\n", ck.CodeOverlayDirective}, } { t.Run(tc.name, func(t *testing.T) { dir := overlayLayout(t, base, tc.overlay) _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection) requireLoadCode(t, err, tc.code) }) } valid := "backends:\n offline:\n concurrency: 2\n" dir := overlayLayout(t, base, valid) loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing") if err != nil { t.Fatal(err) } if got := loaded.Config.Backends["offline"].Concurrency; got != 2 { t.Fatalf("overlay concurrency = %d", got) } dir = overlayLayout(t, base, "backends:\n offline:\n concurrency: 0\n") if _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing"); err == nil { t.Fatal("invalid effective backend limit accepted") } } func TestOverlayCredentialResolutionAndInspection(t *testing.T) { t.Setenv("AUDITA_PROTOTYPE_KEY", "") t.Setenv("AUDITA_OVERLAY_KEY", "replacement-synthetic-secret") base := fixtureConfig(t) for _, tc := range []struct { name, overlay, key string unsetKey bool wantError, wantAuth bool }{ {"replacement", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "replacement-synthetic-secret", false, false, true}, {"surviving missing", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", true, true, false}, {"surviving empty", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", false, true, false}, {"unauthenticated null", "backends:\n offline:\n credential: null\n", "replacement-synthetic-secret", false, false, false}, {"profile switch retains credential", "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n", "replacement-synthetic-secret", false, true, false}, } { t.Run(tc.name, func(t *testing.T) { t.Setenv("AUDITA_OVERLAY_KEY", tc.key) if tc.unsetKey { if err := os.Unsetenv("AUDITA_OVERLAY_KEY"); err != nil { t.Fatal(err) } } dir := overlayLayout(t, base, tc.overlay) loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing") if tc.wantError { if err == nil { t.Fatal("unresolved surviving credential accepted") } if strings.Contains(err.Error(), "replacement-synthetic-secret") { t.Fatal("error exposed secret") } return } if err != nil { t.Fatal(err) } var effective, explanation bytes.Buffer if err := loaded.WriteEffective(&effective); err != nil { t.Fatal(err) } if err := loaded.WriteExplanation(&explanation, ck.Keys("backends", "offline", "credential")); err != nil { t.Fatal(err) } if strings.Contains(effective.String()+explanation.String(), "replacement-synthetic-secret") { t.Fatal("inspection exposed secret") } observed := 0 baseClient := demonstrationClient() client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { want := "" if tc.wantAuth { want = "Bearer replacement-synthetic-secret" } if req.Header.Get("Authorization") != want { t.Errorf("authorization mismatch") } observed++ return baseClient.Transport.RoundTrip(req) })} r, err := newRuntime(loaded.Config, dir, client) if err != nil { t.Fatal(err) } for _, role := range []string{"proposal", "assessment"} { p := r.Profiles[role] raw, _ := json.Marshal(p) if strings.Contains(string(raw), "replacement-synthetic-secret") { t.Fatal("profile exposed secret") } } recorder := &Recorder{} result, err := r.workflow(context.Background(), recorder.observer) if err != nil { t.Fatal(err) } if result.Accepted != 1 || observed != 4 { t.Fatalf("result %+v; calls %d", result, observed) } raw, _ := json.Marshal(recorder.snapshot()) if strings.Contains(string(raw), "replacement-synthetic-secret") { t.Fatal("observations exposed secret") } }) } } func TestOverlaySelectionProvenance(t *testing.T) { t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret") dir := overlayLayout(t, fixtureConfig(t), "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n") loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}) if err != nil { t.Fatal(err) } for _, tc := range []struct { key string layer ck.Layer }{{"proposal_profile", ck.LayerCLI}, {"validation_profile", ck.LayerOverlay}} { path := ck.Keys("llm", tc.key) info, ok := loaded.Metadata().Lookup(path) if !ok || info.Origin.Layer != tc.layer { t.Fatalf("%s metadata = %+v", tc.key, info) } explain, ok := loaded.Explain(path) if !ok || explain.Origin.Layer != tc.layer { t.Fatalf("%s explanation = %+v", tc.key, explain) } } var effective, explanation bytes.Buffer if err := loaded.WriteEffective(&effective); err != nil { t.Fatal(err) } if err := loaded.WriteExplanation(&explanation, ck.Keys("llm", "validation_profile")); err != nil { t.Fatal(err) } if !strings.Contains(effective.String(), "assessment-testing") || !strings.Contains(explanation.String(), "overlay") { t.Fatal("overlay inspection missing") } seenOverlay := false for _, source := range loaded.Sources() { if source.Layer == ck.LayerOverlay { seenOverlay = true } } if !seenOverlay { t.Fatal("overlay source missing") } r, err := newRuntime(loaded.Config, dir, demonstrationClient()) if err != nil { t.Fatal(err) } if r.Profiles["proposal"].ProfileID != "proposal" || r.Profiles["assessment"].ProfileID != "assessment-testing" || r.Profiles["assessment"].EffectiveModelParams.Model != "synthetic-review-testing" { t.Fatal("inspection/runtime selection diverged") } if r.Profiles["proposal"].EffectiveModelParams.Model != "synthetic-grammar" || r.Profiles["proposal"].EffectiveModelParams.BackendID != "offline" || r.Profiles["assessment"].EffectiveModelParams.BackendID != "offline" { t.Fatal("inspected profile target diverged") } }