package llm import ( "context" "errors" "net/http" "os" "path/filepath" "strings" "testing" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" ) type rejectingTransport struct{ t *testing.T } func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) { r.t.Error("profile inspection contacted a provider") return nil, errors.New("provider access forbidden") } func profileLayout(t *testing.T, base, overlay string) string { t.Helper() dir := t.TempDir() for _, name := range []string{"profiles", "overlays"} { if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil { t.Fatal(err) } } for name, content := range map[string]string{ "base": "id: base\nbackend: offline\nmodel: full-model\ntemperature: 0.3\nmax_tokens: 500\ntop_p: 0.8\ntimeout_seconds: 12\nservice_tier: priority\nreasoning_effort: medium\n", "testing": "id: testing\nbackend: offline\nmodel: economy-model\n", } { if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil { t.Fatal(err) } } if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil { t.Fatal(err) } if overlay != "" { if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil { t.Fatal(err) } } return dir } const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n" func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) { t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n") client := &http.Client{Transport: rejectingTransport{t}} for _, tc := range []struct { name, overlay string overrides []ck.Override want string }{ {"base", "", nil, "full-model"}, {"overlay", "testing", nil, "economy-model"}, {"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"}, } { t.Run(tc.name, func(t *testing.T) { main := filepath.Join(dir, "config.yml") loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...) if err != nil { t.Fatal(err) } runtime, err := NewRuntime(context.Background(), loaded.Config, main, client) if err != nil { t.Fatal(err) } if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want { t.Fatalf("roles not resolved: %+v", runtime.Profiles) } profiles, err := runtime.SelectedProfilesYAML() if err != nil { t.Fatal(err) } explain, err := ExplanationYAML(loaded) if err != nil { t.Fatal(err) } combined := string(profiles) + string(explain) if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") { t.Fatalf("inspection missing selection/provenance: %s", combined) } if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") { t.Fatal("safe views exposed credentials or endpoint") } if tc.name == "overlay" && !strings.Contains(combined, "overlay") { t.Fatal("overlay origin missing") } if tc.name == "explicit override" && !strings.Contains(combined, "cli") { t.Fatal("CLI origin missing") } if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") { t.Fatal("inherited validation derivation missing") } if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") { t.Fatal("overlay field source missing") } if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") { t.Fatal("explicit profile source missing") } if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) { t.Fatal("selected controls missing") } }) } } func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) { t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") dir := profileLayout(t, profileBase, "") main := filepath.Join(dir, "config.yml") for _, tc := range []struct{ name, profile string }{ {"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: model\n"}, {"mismatched endpoint", "id: custom\nbackend: offline\nendpoint: https://elsewhere.invalid/v1\nmodel: model\n"}, {"profile credential", "id: custom\nbackend: offline\nmodel: model\napi_key_env: OTHER_KEY\n"}, } { t.Run(tc.name, func(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil { t.Fatal(err) } loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"}) if err != nil { t.Fatal(err) } if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil { t.Fatal("unsafe profile accepted") } }) } loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"}) if err != nil { t.Fatal(err) } if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil { t.Fatal("missing profile accepted") } if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") { t.Fatalf("secret leaked: %q", got) } } func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) { t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") dir := profileLayout(t, profileBase, "") main := filepath.Join(dir, "config.yml") absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1) if err := os.WriteFile(main, []byte(absolute), 0600); err != nil { t.Fatal(err) } loaded, err := config.Load(ck.File(main), "") if err != nil { t.Fatal(err) } if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil { t.Fatal(err) } t.Setenv("SYNTHETIC_KEY", "") if _, err := config.Load(ck.File(main), ""); err == nil { t.Fatal("missing surviving secret accepted") } }