218 lines
8.6 KiB
Go
218 lines
8.6 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
|
)
|
|
|
|
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
profileDir := filepath.Join(dir, "profiles")
|
|
if err := os.Mkdir(profileDir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
main := filepath.Join(dir, "config.yml")
|
|
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
|
|
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, content := range documents {
|
|
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return loaded.Config, main
|
|
}
|
|
|
|
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
|
|
t.Helper()
|
|
source := prompts.Filesystem()
|
|
files := make(fstest.MapFS)
|
|
err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
|
|
if err != nil || entry.IsDir() {
|
|
return err
|
|
}
|
|
data, err := fs.ReadFile(source, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files[name] = &fstest.MapFile{Data: data}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return files
|
|
}
|
|
|
|
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
|
|
validProposal := " backend: offline\n model: proposal-model\n"
|
|
cases := []struct {
|
|
name string
|
|
selections string
|
|
profiles string
|
|
role string
|
|
field string
|
|
}{
|
|
{
|
|
name: "omitted proposal model with derived validation selection",
|
|
selections: " proposal_profile: proposal\n",
|
|
profiles: "profiles:\n proposal:\n backend: offline\n",
|
|
role: "proposal",
|
|
field: "model",
|
|
},
|
|
{
|
|
name: "empty validation model",
|
|
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
|
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
|
|
role: "validation",
|
|
field: "model",
|
|
},
|
|
{
|
|
name: "null proposal model",
|
|
selections: " proposal_profile: proposal\n",
|
|
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
|
|
role: "proposal",
|
|
field: "model",
|
|
},
|
|
{
|
|
name: "cleared validation endpoint",
|
|
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
|
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
|
|
role: "validation",
|
|
field: "endpoint",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
|
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
|
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
|
|
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
|
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
|
|
})
|
|
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
|
|
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
|
|
}
|
|
}
|
|
|
|
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
|
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
|
})
|
|
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
|
if err == nil || IsAssetError(err) {
|
|
t.Fatalf("missing operator profile directory classification: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
document map[string]string
|
|
selection string
|
|
}{
|
|
{
|
|
name: "malformed unselected definition",
|
|
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
|
|
selection: " proposal_profile: proposal\n",
|
|
},
|
|
{
|
|
name: "undefined selected backend",
|
|
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
|
|
selection: " proposal_profile: proposal\n",
|
|
},
|
|
{
|
|
name: "conflicting directory contributions",
|
|
document: map[string]string{
|
|
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
|
|
"b.yml": "profiles:\n proposal:\n model: second-model\n",
|
|
},
|
|
selection: " proposal_profile: proposal\n",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
|
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
|
if err == nil || IsAssetError(err) {
|
|
t.Fatalf("operator definition classification: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
|
|
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
|
|
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
|
|
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
|
|
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
|
|
t.Fatalf("caller cancellation classification: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
|
|
promptFS := copyEmbeddedPromptFilesystem(t)
|
|
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
|
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
|
})
|
|
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
|
if err == nil || !IsAssetError(err) {
|
|
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
|
|
promptFS := copyEmbeddedPromptFilesystem(t)
|
|
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
|
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
|
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
|
|
})
|
|
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
|
if err != nil {
|
|
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
|
|
}
|
|
if runtime.Engine == nil {
|
|
t.Fatal("runtime engine was not retained")
|
|
}
|
|
}
|