1040 lines
28 KiB
Go
1040 lines
28 KiB
Go
package profile
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
)
|
|
|
|
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
files, err := os.ReadDir("testdata")
|
|
if err != nil {
|
|
t.Fatalf("failed to read testdata: %v", err)
|
|
}
|
|
for _, f := range files {
|
|
src := filepath.Join("testdata", f.Name())
|
|
dst := filepath.Join(tmpDir, f.Name())
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
repo := NewFilesystemRepository(tmpDir)
|
|
ctx := context.Background()
|
|
|
|
t.Run("valid local profile", func(t *testing.T) {
|
|
p, err := repo.GetProfile(ctx, "local-default")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "local-default" {
|
|
t.Fatalf("unexpected id: %q", p.ID)
|
|
}
|
|
if p.Endpoint == "" || p.Model == "" {
|
|
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
connection string
|
|
wantBackend string
|
|
wantEndpoint string
|
|
wantErr bool
|
|
}{
|
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
|
{name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"},
|
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
|
{name: "neither", wantErr: true},
|
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
|
|
|
p, err := repo.GetProfile(ctx, id)
|
|
if tt.wantErr {
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("expected profile to load, got %v", err)
|
|
}
|
|
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
|
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
|
p, err := repo.GetProfile(ctx, "local-secure")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.APIKeyEnv != "PROMPTKIT_API_KEY" {
|
|
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
|
}
|
|
if p.ReasoningEffort != "medium" {
|
|
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
|
}
|
|
if p.ServiceTier != "priority" {
|
|
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
|
|
}
|
|
})
|
|
|
|
t.Run("valid nested profile", func(t *testing.T) {
|
|
nestedDir := filepath.Join(tmpDir, "local")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
|
|
id: nested-local
|
|
endpoint: http://localhost:8000/v1
|
|
model: nested-model
|
|
temperature: 0.1
|
|
`)
|
|
|
|
p, err := repo.GetProfile(ctx, "nested-local")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "nested-model" {
|
|
t.Fatalf("unexpected model: %q", p.Model)
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: first-model
|
|
`)
|
|
nestedDir := filepath.Join(tmpDir, "duplicates")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: second-model
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
|
|
}
|
|
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
|
|
nestedDir := filepath.Join(tmpDir, "secure")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
|
id: nested_raw_api_key
|
|
endpoint: http://localhost:8000/v1
|
|
model: m
|
|
api_key: secret
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
|
|
t.Fatalf("expected nested path in error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
|
id: raw-api-key-non-target
|
|
endpoint: http://localhost:8000/v1
|
|
model: m
|
|
api_key: secret
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing id", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing_id")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing endpoint", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing model", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing-model")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unknown field", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "unknown-field")
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("raw api_key rejected", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "raw-api-key")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("profile not found", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "does-not-exist")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func writeProfileTestFile(t *testing.T, path string, content string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
|
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func TestFSRepository(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/provider/nested.yaml": profileMapFile(`
|
|
id: nested-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: nested-model
|
|
temperature: 0.1
|
|
`),
|
|
}, "profiles")
|
|
|
|
p, err := repo.GetProfile(ctx, "nested-profile")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
|
t.Fatalf("unexpected profile: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/unknown.yaml": profileMapFile(`
|
|
id: unknown-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
unknown: value
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "unknown-profile")
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/raw.yaml": profileMapFile(`
|
|
id: raw-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
api_key: secret
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "raw-profile")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/raw.yaml": profileMapFile(`
|
|
id: raw-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
api_key: secret
|
|
`),
|
|
"profiles/valid.yaml": profileMapFile(`
|
|
id: valid-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
`),
|
|
}, "profiles")
|
|
|
|
p, err := repo.GetProfile(ctx, "valid-profile")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "valid-profile" {
|
|
t.Fatalf("unexpected profile: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/a.yaml": profileMapFile(`
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: first
|
|
`),
|
|
"profiles/nested/b.yaml": profileMapFile(`
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: second
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
endpoint string
|
|
withBackend bool
|
|
}{
|
|
{name: "relative", endpoint: "/v1"},
|
|
{name: "missing host", endpoint: "https:///v1"},
|
|
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1"},
|
|
{name: "user information", endpoint: "https://user@provider.example/v1"},
|
|
{name: "query", endpoint: "https://provider.example/v1?mode=chat"},
|
|
{name: "fragment", endpoint: "https://provider.example/v1#chat"},
|
|
{name: "backend with invalid override", endpoint: "/v1", withBackend: true},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
backend := ""
|
|
if tc.withBackend {
|
|
backend = "backend: openrouter\n"
|
|
}
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/invalid.yaml": profileMapFile(fmt.Sprintf(
|
|
"id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n",
|
|
backend,
|
|
tc.endpoint,
|
|
)),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(context.Background(), "invalid-endpoint")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProfileRepositoriesValidateExtraParams(t *testing.T) {
|
|
const validProfile = `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
extra_params:
|
|
string_value: enabled
|
|
object_value:
|
|
nested: true
|
|
array_value:
|
|
- first
|
|
- 3
|
|
`
|
|
tests := []struct {
|
|
name string
|
|
definition string
|
|
wantErr bool
|
|
diagnostics []string
|
|
}{
|
|
{name: "valid nested values", definition: validProfile},
|
|
{
|
|
name: "empty key",
|
|
definition: `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
extra_params:
|
|
"": value
|
|
`,
|
|
wantErr: true,
|
|
diagnostics: []string{"extra_params", "key must not be empty"},
|
|
},
|
|
{
|
|
name: "non-finite value",
|
|
definition: `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
extra_params:
|
|
invalid: .nan
|
|
`,
|
|
wantErr: true,
|
|
diagnostics: []string{"extra_params.invalid", "must be finite"},
|
|
},
|
|
{
|
|
name: "nested non-finite value",
|
|
definition: `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
extra_params:
|
|
outer:
|
|
invalid: .inf
|
|
`,
|
|
wantErr: true,
|
|
diagnostics: []string{"extra_params.outer.invalid", "must be finite"},
|
|
},
|
|
{
|
|
name: "unsupported decoded value",
|
|
definition: `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
extra_params:
|
|
timestamp: 2026-08-11T12:34:56Z
|
|
`,
|
|
wantErr: true,
|
|
diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"},
|
|
},
|
|
{
|
|
name: "excessive nesting",
|
|
definition: deeplyNestedExtraParamsProfile(101),
|
|
wantErr: true,
|
|
diagnostics: []string{"extra_params", "JSON container depth limit exceeded"},
|
|
},
|
|
}
|
|
|
|
for _, source := range profileRepositorySources() {
|
|
for _, tc := range tests {
|
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
|
repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition})
|
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
|
if tc.wantErr {
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
if !strings.Contains(err.Error(), "selected.yaml") {
|
|
t.Fatalf("expected source path in error, got %v", err)
|
|
}
|
|
for _, diagnostic := range tc.diagnostics {
|
|
if !strings.Contains(err.Error(), diagnostic) {
|
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("load valid profile: %v", err)
|
|
}
|
|
if got.ExtraParams["string_value"] != "enabled" {
|
|
t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams)
|
|
}
|
|
objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any)
|
|
arrayValue, arrayOK := got.ExtraParams["array_value"].([]any)
|
|
if !objectOK || objectValue["nested"] != true ||
|
|
!arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 {
|
|
t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) {
|
|
const validProfile = `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
`
|
|
tests := []struct {
|
|
name string
|
|
files map[string]string
|
|
wantErr error
|
|
diagnostics []string
|
|
}{
|
|
{
|
|
name: "same stem unknown field with different id is unrelated",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": `
|
|
id: unrelated-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated
|
|
unknown: true
|
|
`,
|
|
"valid.yaml": validProfile,
|
|
},
|
|
},
|
|
{
|
|
name: "same stem unidentifiable yaml is unrelated",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": "id: [",
|
|
"valid.yaml": validProfile,
|
|
},
|
|
},
|
|
{
|
|
name: "same stem raw key with different id is unrelated",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": `
|
|
id: unrelated-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated
|
|
api_key: secret
|
|
`,
|
|
"valid.yaml": validProfile,
|
|
},
|
|
},
|
|
{
|
|
name: "leading and trailing whitespace is normalized",
|
|
files: map[string]string{
|
|
"padded.yaml": `
|
|
id: " selected-profile "
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
`,
|
|
},
|
|
},
|
|
{
|
|
name: "blank id is unrelated",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": `
|
|
id: " "
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated
|
|
`,
|
|
},
|
|
wantErr: ErrProfileNotFound,
|
|
},
|
|
{
|
|
name: "normalized duplicates are ambiguous",
|
|
files: map[string]string{
|
|
"first.yaml": validProfile,
|
|
"nested/second.yaml": `
|
|
id: " selected-profile "
|
|
endpoint: http://localhost:8000/v1
|
|
model: duplicate
|
|
`,
|
|
},
|
|
wantErr: ErrInvalidProfile,
|
|
diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"},
|
|
},
|
|
{
|
|
name: "selected unknown field is authoritative",
|
|
files: map[string]string{
|
|
"malformed.yaml": `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
unknown: true
|
|
`,
|
|
},
|
|
wantErr: ErrInvalidYAML,
|
|
diagnostics: []string{"malformed.yaml"},
|
|
},
|
|
{
|
|
name: "selected raw key is authoritative",
|
|
files: map[string]string{
|
|
"insecure.yaml": `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
api_key: secret
|
|
`,
|
|
},
|
|
wantErr: ErrRawAPIKeyNotAllowed,
|
|
diagnostics: []string{"insecure.yaml"},
|
|
},
|
|
{
|
|
name: "selected identity in an additional document is authoritative",
|
|
files: map[string]string{
|
|
"additional-document.yaml": `
|
|
---
|
|
---
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
`,
|
|
},
|
|
wantErr: ErrInvalidYAML,
|
|
diagnostics: []string{"additional-document.yaml", "exactly one YAML document"},
|
|
},
|
|
}
|
|
|
|
for _, source := range profileRepositorySources() {
|
|
for _, tc := range tests {
|
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
|
repo := source.newRepository(t, tc.files)
|
|
got, err := repo.GetProfile(context.Background(), " selected-profile ")
|
|
if tc.wantErr != nil {
|
|
if !errors.Is(err, tc.wantErr) {
|
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
|
}
|
|
for _, diagnostic := range tc.diagnostics {
|
|
if !strings.Contains(err.Error(), diagnostic) {
|
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("load selected profile: %v", err)
|
|
}
|
|
if got.ID != "selected-profile" || got.Model != "selected-model" {
|
|
t.Fatalf("unexpected selected profile: %+v", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) {
|
|
const profile = `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected-model
|
|
`
|
|
tests := []struct {
|
|
name string
|
|
suffix string
|
|
wantErr bool
|
|
}{
|
|
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
|
|
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
|
|
{name: "second empty document", suffix: "\n---\n", wantErr: true},
|
|
{name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true},
|
|
{name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true},
|
|
}
|
|
|
|
for _, source := range profileRepositorySources() {
|
|
for _, tc := range tests {
|
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
|
repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix})
|
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
|
if tc.wantErr {
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
}
|
|
if !strings.Contains(err.Error(), "definition.yaml") {
|
|
t.Fatalf("expected source path in error, got %v", err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("load one-document profile: %v", err)
|
|
}
|
|
if got.ID != "selected-profile" {
|
|
t.Fatalf("unexpected profile: %+v", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) {
|
|
fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
|
"selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"},
|
|
}}
|
|
tests := []struct {
|
|
name string
|
|
files map[string]string
|
|
wantModel string
|
|
wantErr error
|
|
}{
|
|
{
|
|
name: "same stem malformed different id falls back",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": `
|
|
id: unrelated-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated
|
|
unknown: true
|
|
`,
|
|
},
|
|
wantModel: "fallback-model",
|
|
},
|
|
{
|
|
name: "blank id falls back",
|
|
files: map[string]string{
|
|
"selected-profile.yaml": `
|
|
id: " "
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated
|
|
`,
|
|
},
|
|
wantModel: "fallback-model",
|
|
},
|
|
{
|
|
name: "selected malformed profile stops fallback",
|
|
files: map[string]string{
|
|
"other-name.yaml": `
|
|
id: selected-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: selected
|
|
unknown: true
|
|
`,
|
|
},
|
|
wantErr: ErrInvalidYAML,
|
|
},
|
|
}
|
|
|
|
for _, source := range profileRepositorySources() {
|
|
for _, tc := range tests {
|
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
|
primary := source.newRepository(t, tc.files)
|
|
got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile")
|
|
if tc.wantErr != nil {
|
|
if !errors.Is(err, tc.wantErr) {
|
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("load fallback profile: %v", err)
|
|
}
|
|
if got.Model != tc.wantModel {
|
|
t.Fatalf("model = %q, want %q", got.Model, tc.wantModel)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) {
|
|
newSource := func() (*recordingProfileFS, Repository) {
|
|
fsys := &recordingProfileFS{FS: fstest.MapFS{
|
|
"target.yaml": profileMapFile(`
|
|
id: target
|
|
endpoint: http://localhost:8000/v1
|
|
model: target-model
|
|
`),
|
|
"unrelated.yaml": profileMapFile(`
|
|
id: unrelated
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated-model
|
|
`),
|
|
}}
|
|
return fsys, NewFSRepository(fsys, ".")
|
|
}
|
|
|
|
t.Run("selected source", func(t *testing.T) {
|
|
fsys, repo := newSource()
|
|
for lookup := 1; lookup <= 2; lookup++ {
|
|
got, err := repo.GetProfile(context.Background(), "target")
|
|
if err != nil {
|
|
t.Fatalf("lookup %d: %v", lookup, err)
|
|
}
|
|
if got.Model != "target-model" {
|
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
|
}
|
|
for _, name := range []string{"target.yaml", "unrelated.yaml"} {
|
|
if count := fsys.openCount(name); count != lookup {
|
|
t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("overlay fallthrough", func(t *testing.T) {
|
|
primaryFS := &recordingProfileFS{FS: fstest.MapFS{
|
|
"unrelated.yaml": profileMapFile(`
|
|
id: unrelated
|
|
endpoint: http://localhost:8000/v1
|
|
model: unrelated-model
|
|
`),
|
|
}}
|
|
fallbackFS, fallback := newSource()
|
|
repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback)
|
|
|
|
for lookup := 1; lookup <= 2; lookup++ {
|
|
got, err := repo.GetProfile(context.Background(), "target")
|
|
if err != nil {
|
|
t.Fatalf("lookup %d: %v", lookup, err)
|
|
}
|
|
if got.Model != "target-model" {
|
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
|
}
|
|
if count := primaryFS.openCount("unrelated.yaml"); count != lookup {
|
|
t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup)
|
|
}
|
|
if count := fallbackFS.openCount("target.yaml"); count != lookup {
|
|
t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("operating-system filesystem", func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
|
|
id: invalid
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
temperature: .nan
|
|
`)
|
|
|
|
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("fs.FS", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/invalid.yaml": profileMapFile(`
|
|
id: invalid
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
top_p: .inf
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "invalid")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestOverlayRepository(t *testing.T) {
|
|
ctx := context.Background()
|
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
|
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
|
|
|
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "primary" {
|
|
t.Fatalf("expected primary profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("falls back on primary not found", func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "fallback" {
|
|
t.Fatalf("expected fallback profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
err error
|
|
}{
|
|
{name: "invalid yaml", err: ErrInvalidYAML},
|
|
{name: "invalid profile", err: ErrInvalidProfile},
|
|
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{err: tc.err},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
_, err := repo.GetProfile(ctx, "shared")
|
|
if !errors.Is(err, tc.err) {
|
|
t.Fatalf("expected %v, got %v", tc.err, err)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
|
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
|
|
|
_, err := repo.GetProfile(ctx, "missing")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("nil primary uses fallback", func(t *testing.T) {
|
|
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "fallback" {
|
|
t.Fatalf("expected fallback profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
|
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
|
|
|
_, err := repo.GetProfile(ctx, "missing")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
type profileRepositorySource struct {
|
|
name string
|
|
newRepository func(t *testing.T, files map[string]string) Repository
|
|
}
|
|
|
|
type recordingProfileFS struct {
|
|
fs.FS
|
|
mu sync.Mutex
|
|
opened []string
|
|
}
|
|
|
|
func (f *recordingProfileFS) Open(name string) (fs.File, error) {
|
|
f.mu.Lock()
|
|
f.opened = append(f.opened, name)
|
|
f.mu.Unlock()
|
|
return f.FS.Open(name)
|
|
}
|
|
|
|
func (f *recordingProfileFS) openCount(name string) int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
count := 0
|
|
for _, opened := range f.opened {
|
|
if opened == name {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func profileRepositorySources() []profileRepositorySource {
|
|
return []profileRepositorySource{
|
|
{
|
|
name: "operating system",
|
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
for name, content := range files {
|
|
filePath := filepath.Join(root, filepath.FromSlash(name))
|
|
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
|
t.Fatalf("create profile directory: %v", err)
|
|
}
|
|
writeProfileTestFile(t, filePath, content)
|
|
}
|
|
return NewFilesystemRepository(root)
|
|
},
|
|
},
|
|
{
|
|
name: "filesystem",
|
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
|
t.Helper()
|
|
fsys := make(fstest.MapFS, len(files))
|
|
for name, content := range files {
|
|
fsys[name] = profileMapFile(content)
|
|
}
|
|
return NewFSRepository(fsys, ".")
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func deeplyNestedExtraParamsProfile(depth int) string {
|
|
var definition strings.Builder
|
|
definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n")
|
|
for level := 0; level < depth; level++ {
|
|
fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level)
|
|
}
|
|
fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1))
|
|
return definition.String()
|
|
}
|
|
|
|
func profileMapFile(content string) *fstest.MapFile {
|
|
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
|
}
|
|
|
|
type staticProfileRepo struct {
|
|
profiles map[string]*domain.ExecutionProfile
|
|
err error
|
|
}
|
|
|
|
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
if r.err != nil {
|
|
return nil, r.err
|
|
}
|
|
if p, ok := r.profiles[id]; ok {
|
|
cp := *p
|
|
return &cp, nil
|
|
}
|
|
return nil, ErrProfileNotFound
|
|
}
|