package llm import ( "context" "errors" "io/fs" "net/http" "os" "path" "path/filepath" "strings" "testing" "testing/fstest" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) { t.Helper() dir := t.TempDir() profileDir := filepath.Join(dir, "profiles") if err := os.Mkdir(profileDir, 0700); err != nil { t.Fatal(err) } main := filepath.Join(dir, "config.yml") text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections if err := os.WriteFile(main, []byte(text), 0600); err != nil { t.Fatal(err) } for name, content := range documents { if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil { t.Fatal(err) } } loaded, err := config.Load(ck.File(main), "") if err != nil { t.Fatal(err) } return loaded.Config, main } func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS { t.Helper() source := prompts.Filesystem() files := make(fstest.MapFS) err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error { if err != nil || entry.IsDir() { return err } data, err := fs.ReadFile(source, name) if err != nil { return err } files[name] = &fstest.MapFile{Data: data} return nil }) if err != nil { t.Fatal(err) } return files } func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) { validProposal := " backend: offline\n model: proposal-model\n" cases := []struct { name string selections string profiles string role string field string }{ { name: "omitted proposal model with derived validation selection", selections: " proposal_profile: proposal\n", profiles: "profiles:\n proposal:\n backend: offline\n", role: "proposal", field: "model", }, { name: "empty validation model", selections: " proposal_profile: proposal\n validation_profile: validation\n", profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n", role: "validation", field: "model", }, { name: "null proposal model", selections: " proposal_profile: proposal\n", profiles: "profiles:\n proposal:\n backend: offline\n model: null\n", role: "proposal", field: "model", }, { name: "cleared validation endpoint", selections: " proposal_profile: proposal\n validation_profile: validation\n", profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n", role: "validation", field: "endpoint", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles}) _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) { t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err) } }) } } func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n", }) runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) if err != nil { t.Fatal(err) } if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" { t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles) } } func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{ "unrelated-name.yaml": `profiles: "parent~/shared": backend: offline temperature: 0.4 top_p: 0.7 extra_params: routing: retained: from-parent overridden: parent cleared: remove-me "child profile": extends: "parent~/shared" model: child-model extra_params: routing: overridden: child child_only: present cleared: null `, }) runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}}) if err != nil { t.Fatal(err) } profile := runtime.Profiles["proposal"] target := profile.EffectiveModelParams if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 { t.Fatalf("inherited profile target = %+v", target) } routing, ok := target.ExtraParams["routing"].(map[string]any) if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" { t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams) } if value, exists := routing["cleared"]; !exists || value != nil { t.Fatalf("null did not replace the inherited provider value: %#v", routing) } } func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", }) if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil { t.Fatal(err) } _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) if err == nil || IsAssetError(err) { t.Fatalf("missing operator profile directory classification: %v", err) } } func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) { cases := []struct { name string document map[string]string selection string }{ { name: "malformed unselected definition", document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"}, selection: " proposal_profile: proposal\n", }, { name: "undefined selected backend", document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"}, selection: " proposal_profile: proposal\n", }, { name: "conflicting directory contributions", document: map[string]string{ "a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n", "b.yml": "profiles:\n proposal:\n model: second-model\n", }, selection: " proposal_profile: proposal\n", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document) _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) if err == nil || IsAssetError(err) { t.Fatalf("operator definition classification: %v", err) } }) } } func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) { _, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{}) if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) { t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err) } ctx, cancel := context.WithCancel(context.Background()) cancel() _, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{}) if !errors.Is(err, context.Canceled) || IsAssetError(err) { t.Fatalf("caller cancellation classification: %v", err) } } func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) { promptFS := copyEmbeddedPromptFilesystem(t) assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", }) _, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) if err == nil || !IsAssetError(err) { t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err) } } func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) { promptFS := copyEmbeddedPromptFilesystem(t) assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ "operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n", }) runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) if err != nil { t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err) } if runtime.Engine == nil { t.Fatal("runtime engine was not retained") } }