audita/internal/framework/llm/runtime_inspection_test.go

package llm

import (
	"context"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"

	ck "gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/audita/internal/core/config"
)

type rejectingTransport struct{ t *testing.T }

func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
	r.t.Error("profile inspection contacted a provider")
	return nil, errors.New("provider access forbidden")
}

func profileLayout(t *testing.T, base, overlay string) string {
	t.Helper()
	dir := t.TempDir()
	for _, name := range []string{"profiles", "overlays"} {
		if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
			t.Fatal(err)
		}
	}
	for name, content := range map[string]string{
		"base":    "id: base\nbackend: offline\nmodel: full-model\ntemperature: 0.3\nmax_tokens: 500\ntop_p: 0.8\ntimeout_seconds: 12\nservice_tier: priority\nreasoning_effort: medium\n",
		"testing": "id: testing\nbackend: offline\nmodel: economy-model\n",
	} {
		if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
			t.Fatal(err)
		}
	}
	if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
		t.Fatal(err)
	}
	if overlay != "" {
		if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

const profileBase = "version: 2\nbackends:\n  offline:\n    endpoint: https://offline.invalid/v1/\n    credential: SYNTHETIC_KEY\nllm:\n  proposal_profile: base\n"

func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
	dir := profileLayout(t, profileBase, "llm:\n  proposal_profile: testing\n")
	client := &http.Client{Transport: rejectingTransport{t}}
	for _, tc := range []struct {
		name, overlay string
		overrides     []ck.Override
		want          string
	}{
		{"base", "", nil, "full-model"},
		{"overlay", "testing", nil, "economy-model"},
		{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			main := filepath.Join(dir, "config.yml")
			loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
			if err != nil {
				t.Fatal(err)
			}
			runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
			if err != nil {
				t.Fatal(err)
			}
			if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
				t.Fatalf("roles not resolved: %+v", runtime.Profiles)
			}
			profiles, err := runtime.SelectedProfilesYAML()
			if err != nil {
				t.Fatal(err)
			}
			explain, err := ExplanationYAML(loaded)
			if err != nil {
				t.Fatal(err)
			}
			combined := string(profiles) + string(explain)
			if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
				t.Fatalf("inspection missing selection/provenance: %s", combined)
			}
			if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
				t.Fatal("safe views exposed credentials or endpoint")
			}
			if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
				t.Fatal("overlay origin missing")
			}
			if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
				t.Fatal("CLI origin missing")
			}
			if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
				t.Fatal("inherited validation derivation missing")
			}
			if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n  layer: overlay") {
				t.Fatal("overlay field source missing")
			}
			if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n  layer: cli") {
				t.Fatal("explicit profile source missing")
			}
			if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
				t.Fatal("selected controls missing")
			}
		})
	}
}

func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
	dir := profileLayout(t, profileBase, "")
	main := filepath.Join(dir, "config.yml")
	for _, tc := range []struct{ name, profile string }{
		{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: model\n"},
		{"mismatched endpoint", "id: custom\nbackend: offline\nendpoint: https://elsewhere.invalid/v1\nmodel: model\n"},
		{"profile credential", "id: custom\nbackend: offline\nmodel: model\napi_key_env: OTHER_KEY\n"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
				t.Fatal(err)
			}
			loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
			if err != nil {
				t.Fatal(err)
			}
			if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
				t.Fatal("unsafe profile accepted")
			}
		})
	}
	loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
		t.Fatal("missing profile accepted")
	}
	if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
		t.Fatalf("secret leaked: %q", got)
	}
}

func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
	dir := profileLayout(t, profileBase, "")
	main := filepath.Join(dir, "config.yml")
	absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
	if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
		t.Fatal(err)
	}
	loaded, err := config.Load(ck.File(main), "")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
		t.Fatal(err)
	}
	t.Setenv("SYNTHETIC_KEY", "")
	if _, err := config.Load(ck.File(main), ""); err == nil {
		t.Fatal("missing surviving secret accepted")
	}
}