240 lines
8.7 KiB
Go
240 lines
8.7 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type rejectingTransport struct{ t *testing.T }
|
|
|
|
func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
|
r.t.Error("profile inspection contacted a provider")
|
|
return nil, errors.New("provider access forbidden")
|
|
}
|
|
|
|
func profileLayout(t *testing.T, base, overlay string) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
for _, name := range []string{"profiles", "overlays"} {
|
|
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for name, content := range map[string]string{
|
|
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
|
|
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if overlay != "" {
|
|
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return dir
|
|
}
|
|
|
|
const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n"
|
|
|
|
func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
|
|
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
|
dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n")
|
|
client := &http.Client{Transport: rejectingTransport{t}}
|
|
for _, tc := range []struct {
|
|
name, overlay string
|
|
overrides []ck.Override
|
|
want string
|
|
}{
|
|
{"base", "", nil, "full-model"},
|
|
{"overlay", "testing", nil, "economy-model"},
|
|
{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
main := filepath.Join(dir, "config.yml")
|
|
loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
|
|
t.Fatalf("roles not resolved: %+v", runtime.Profiles)
|
|
}
|
|
profiles, err := runtime.SelectedProfilesYAML()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
explain, err := ExplanationYAML(loaded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
combined := string(profiles) + string(explain)
|
|
if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
|
|
t.Fatalf("inspection missing selection/provenance: %s", combined)
|
|
}
|
|
if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
|
|
t.Fatal("safe views exposed credentials or endpoint")
|
|
}
|
|
if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
|
|
t.Fatal("overlay origin missing")
|
|
}
|
|
if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
|
|
t.Fatal("CLI origin missing")
|
|
}
|
|
if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
|
|
t.Fatal("inherited validation derivation missing")
|
|
}
|
|
if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") {
|
|
t.Fatal("overlay field source missing")
|
|
}
|
|
if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") {
|
|
t.Fatal("explicit profile source missing")
|
|
}
|
|
if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
|
|
t.Fatal("selected controls missing")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
|
|
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
|
|
text := `version: 2
|
|
backends:
|
|
shared:
|
|
endpoint: https://shared.invalid/v1
|
|
concurrency: 3
|
|
queue_capacity: 0
|
|
credential: SYNTHETIC_KEY
|
|
offline:
|
|
extends: shared
|
|
concurrency: 1
|
|
llm:
|
|
proposal_profile: base
|
|
`
|
|
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
explanation, err := ExplanationYAML(loaded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
|
|
t.Fatal("explanation exposed a resolved inherited credential")
|
|
}
|
|
var records []struct {
|
|
Path string `yaml:"path"`
|
|
Inheritance []struct {
|
|
From string `yaml:"from"`
|
|
To string `yaml:"to"`
|
|
Reference string `yaml:"reference"`
|
|
} `yaml:"inheritance"`
|
|
}
|
|
if err := yaml.Unmarshal(explanation, &records); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var endpoint, credential, concurrency *struct {
|
|
Path string `yaml:"path"`
|
|
Inheritance []struct {
|
|
From string `yaml:"from"`
|
|
To string `yaml:"to"`
|
|
Reference string `yaml:"reference"`
|
|
} `yaml:"inheritance"`
|
|
}
|
|
for i := range records {
|
|
switch records[i].Path {
|
|
case "/backends/offline/endpoint":
|
|
endpoint = &records[i]
|
|
case "/backends/offline/credential":
|
|
credential = &records[i]
|
|
case "/backends/offline/concurrency":
|
|
concurrency = &records[i]
|
|
}
|
|
}
|
|
if endpoint == nil || len(endpoint.Inheritance) != 1 {
|
|
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
|
|
}
|
|
step := endpoint.Inheritance[0]
|
|
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
|
|
t.Fatalf("inherited endpoint evidence: %+v", step)
|
|
}
|
|
if credential == nil || len(credential.Inheritance) != 1 {
|
|
t.Fatalf("inherited credential evidence missing: %s", explanation)
|
|
}
|
|
if concurrency == nil || len(concurrency.Inheritance) != 0 {
|
|
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
|
|
}
|
|
}
|
|
|
|
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
|
|
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
|
dir := profileLayout(t, profileBase, "")
|
|
main := filepath.Join(dir, "config.yml")
|
|
for _, tc := range []struct{ name, profile string }{
|
|
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
|
|
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
|
|
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
|
|
t.Fatal("unsafe profile accepted")
|
|
}
|
|
})
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
|
|
t.Fatal("missing profile accepted")
|
|
}
|
|
if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
|
|
t.Fatalf("secret leaked: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
|
|
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
|
dir := profileLayout(t, profileBase, "")
|
|
main := filepath.Join(dir, "config.yml")
|
|
absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
|
|
if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("SYNTHETIC_KEY", "")
|
|
if _, err := config.Load(ck.File(main), ""); err == nil {
|
|
t.Fatal("missing surviving secret accepted")
|
|
}
|
|
}
|