552 lines
16 KiB
Go
552 lines
16 KiB
Go
package profile
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
)
|
|
|
|
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
files, err := os.ReadDir("testdata")
|
|
if err != nil {
|
|
t.Fatalf("failed to read testdata: %v", err)
|
|
}
|
|
for _, f := range files {
|
|
src := filepath.Join("testdata", f.Name())
|
|
dst := filepath.Join(tmpDir, f.Name())
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
repo := NewFilesystemRepository(tmpDir)
|
|
ctx := context.Background()
|
|
|
|
t.Run("valid local profile", func(t *testing.T) {
|
|
p, err := repo.GetProfile(ctx, "local-default")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "local-default" {
|
|
t.Fatalf("unexpected id: %q", p.ID)
|
|
}
|
|
if p.Endpoint == "" || p.Model == "" {
|
|
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
connection string
|
|
wantBackend string
|
|
wantEndpoint string
|
|
wantErr bool
|
|
}{
|
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
|
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
|
{name: "neither", wantErr: true},
|
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
|
|
|
p, err := repo.GetProfile(ctx, id)
|
|
if tt.wantErr {
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("expected profile to load, got %v", err)
|
|
}
|
|
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
|
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
|
p, err := repo.GetProfile(ctx, "local-secure")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.APIKeyEnv != "PROMPTKIT_API_KEY" {
|
|
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
|
}
|
|
if p.ReasoningEffort != "medium" {
|
|
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
|
}
|
|
if p.ServiceTier != "priority" {
|
|
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
|
|
}
|
|
})
|
|
|
|
t.Run("valid nested profile", func(t *testing.T) {
|
|
nestedDir := filepath.Join(tmpDir, "local")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
|
|
id: nested-local
|
|
endpoint: http://localhost:8000/v1
|
|
model: nested-model
|
|
temperature: 0.1
|
|
`)
|
|
|
|
p, err := repo.GetProfile(ctx, "nested-local")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "nested-model" {
|
|
t.Fatalf("unexpected model: %q", p.Model)
|
|
}
|
|
})
|
|
|
|
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
|
id: json-extra-params
|
|
endpoint: http://localhost:8000/v1
|
|
model: nested-model
|
|
extra_params:
|
|
string_value: enabled
|
|
number_value: 42
|
|
boolean_value: true
|
|
object_value:
|
|
nested: value
|
|
count: 2
|
|
array_value:
|
|
- first
|
|
- 3
|
|
- false
|
|
`)
|
|
|
|
p, err := repo.GetProfile(ctx, "json-extra-params")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
|
|
var got map[string]any
|
|
encoded, err := json.Marshal(p.ExtraParams)
|
|
if err != nil {
|
|
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
|
}
|
|
if err := json.Unmarshal(encoded, &got); err != nil {
|
|
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
|
}
|
|
|
|
if got["string_value"] != "enabled" {
|
|
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
|
}
|
|
if got["number_value"] != float64(42) {
|
|
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
|
}
|
|
if got["boolean_value"] != true {
|
|
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
|
}
|
|
objectValue, ok := got["object_value"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
|
}
|
|
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
|
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
|
}
|
|
arrayValue, ok := got["array_value"].([]any)
|
|
if !ok {
|
|
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
|
}
|
|
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
|
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: first-model
|
|
`)
|
|
nestedDir := filepath.Join(tmpDir, "duplicates")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: second-model
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
|
|
}
|
|
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
|
|
nestedDir := filepath.Join(tmpDir, "secure")
|
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
|
id: nested_raw_api_key
|
|
endpoint: http://localhost:8000/v1
|
|
model: m
|
|
api_key: secret
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
|
|
t.Fatalf("expected nested path in error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
|
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
|
id: raw-api-key-non-target
|
|
endpoint: http://localhost:8000/v1
|
|
model: m
|
|
api_key: secret
|
|
`)
|
|
|
|
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("invalid yaml", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing id", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing_id")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing endpoint", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("missing model", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "missing-model")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unknown field", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "unknown_field")
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("raw api_key rejected", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "raw_api_key")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("profile not found", func(t *testing.T) {
|
|
_, err := repo.GetProfile(ctx, "does-not-exist")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func writeProfileTestFile(t *testing.T, path string, content string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
|
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func TestFSRepository(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/provider/nested.yaml": profileMapFile(`
|
|
id: nested-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: nested-model
|
|
temperature: 0.1
|
|
`),
|
|
}, "profiles")
|
|
|
|
p, err := repo.GetProfile(ctx, "nested-profile")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
|
t.Fatalf("unexpected profile: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/unknown.yaml": profileMapFile(`
|
|
id: unknown-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
unknown: value
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "unknown-profile")
|
|
if !errors.Is(err, ErrInvalidYAML) {
|
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/raw.yaml": profileMapFile(`
|
|
id: raw-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
api_key: secret
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "raw-profile")
|
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/raw.yaml": profileMapFile(`
|
|
id: raw-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
api_key: secret
|
|
`),
|
|
"profiles/valid.yaml": profileMapFile(`
|
|
id: valid-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
`),
|
|
}, "profiles")
|
|
|
|
p, err := repo.GetProfile(ctx, "valid-profile")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.ID != "valid-profile" {
|
|
t.Fatalf("unexpected profile: %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/a.yaml": profileMapFile(`
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: first
|
|
`),
|
|
"profiles/nested/b.yaml": profileMapFile(`
|
|
id: duplicate-profile
|
|
endpoint: http://localhost:8000/v1
|
|
model: second
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("operating-system filesystem", func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
|
|
id: invalid
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
temperature: .nan
|
|
`)
|
|
|
|
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("fs.FS", func(t *testing.T) {
|
|
repo := NewFSRepository(fstest.MapFS{
|
|
"profiles/invalid.yaml": profileMapFile(`
|
|
id: invalid
|
|
endpoint: http://localhost:8000/v1
|
|
model: model
|
|
top_p: .inf
|
|
`),
|
|
}, "profiles")
|
|
|
|
_, err := repo.GetProfile(ctx, "invalid")
|
|
if !errors.Is(err, ErrInvalidProfile) {
|
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestOverlayRepository(t *testing.T) {
|
|
ctx := context.Background()
|
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
|
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
|
|
|
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "primary" {
|
|
t.Fatalf("expected primary profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("falls back on primary not found", func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "fallback" {
|
|
t.Fatalf("expected fallback profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
err error
|
|
}{
|
|
{name: "invalid yaml", err: ErrInvalidYAML},
|
|
{name: "invalid profile", err: ErrInvalidProfile},
|
|
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
repo := NewOverlayRepository(
|
|
staticProfileRepo{err: tc.err},
|
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
)
|
|
|
|
_, err := repo.GetProfile(ctx, "shared")
|
|
if !errors.Is(err, tc.err) {
|
|
t.Fatalf("expected %v, got %v", tc.err, err)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
|
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
|
|
|
_, err := repo.GetProfile(ctx, "missing")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("nil primary uses fallback", func(t *testing.T) {
|
|
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
|
|
|
p, err := repo.GetProfile(ctx, "shared")
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if p.Model != "fallback" {
|
|
t.Fatalf("expected fallback profile, got %+v", p)
|
|
}
|
|
})
|
|
|
|
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
|
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
|
|
|
_, err := repo.GetProfile(ctx, "missing")
|
|
if !errors.Is(err, ErrProfileNotFound) {
|
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func profileMapFile(content string) *fstest.MapFile {
|
|
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
|
}
|
|
|
|
type staticProfileRepo struct {
|
|
profiles map[string]*domain.ExecutionProfile
|
|
err error
|
|
}
|
|
|
|
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
if r.err != nil {
|
|
return nil, r.err
|
|
}
|
|
if p, ok := r.profiles[id]; ok {
|
|
cp := *p
|
|
return &cp, nil
|
|
}
|
|
return nil, ErrProfileNotFound
|
|
}
|