365 lines
14 KiB
Go
365 lines
14 KiB
Go
package catalog
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
|
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
|
)
|
|
|
|
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
|
|
loaded, err := Load(
|
|
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
|
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("load published catalogs: %v", err)
|
|
}
|
|
expected := loadCompatibilityFixture(t)
|
|
expectedIDs := fixtureProfileIDs(t, expected)
|
|
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
|
|
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
|
|
}
|
|
actual := catalogValue(t, loaded)
|
|
actualJSON, err := json.Marshal(actual)
|
|
if err != nil {
|
|
t.Fatalf("encode loaded catalogs: %v", err)
|
|
}
|
|
expectedJSON, err := json.Marshal(expected)
|
|
if err != nil {
|
|
t.Fatalf("encode compatibility fixture: %v", err)
|
|
}
|
|
if !bytes.Equal(actualJSON, expectedJSON) {
|
|
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidSources(t *testing.T) {
|
|
for name, sources := range map[string][]Source{
|
|
"none": nil,
|
|
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
|
"duplicate name": {
|
|
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
|
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
|
},
|
|
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
|
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
|
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if _, err := Load(sources...); err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
|
tests := map[string]func(fstest.MapFS){
|
|
"unexpected file": func(fsys fstest.MapFS) {
|
|
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
|
},
|
|
"unexpected directory": func(fsys fstest.MapFS) {
|
|
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
|
},
|
|
"nonregular manifest": func(fsys fstest.MapFS) {
|
|
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
|
},
|
|
"wrong profile extension": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
|
},
|
|
"nonregular profile": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
|
},
|
|
}
|
|
for name, mutate := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
mutate(fsys)
|
|
if _, err := Load(testSource("one", fsys)); err == nil {
|
|
t.Fatal("expected invalid layout error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
|
tests := map[string]string{
|
|
"malformed": `{`,
|
|
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
|
"missing fields": `{"schema_version":1,"id":"one"}`,
|
|
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
|
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
|
"blank API key env": validManifest("one", " ", "null"),
|
|
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
|
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
|
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
|
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
|
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
|
}
|
|
for name, manifest := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
fsys["catalog/backend.json"].Data = []byte(manifest)
|
|
_, err := Load(testSource("one", fsys))
|
|
if err == nil {
|
|
t.Fatal("expected invalid manifest error")
|
|
}
|
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
|
t.Fatalf("catalog error exposed manifest content: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
|
"one",
|
|
"TEST_API_KEY",
|
|
`{"large":9007199254740993,"nested":[1.25]}`,
|
|
))
|
|
loaded, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("load catalog: %v", err)
|
|
}
|
|
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
|
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
|
}
|
|
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
|
if nested[0] != json.Number("1.25") {
|
|
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
|
tests := map[string]func(fstest.MapFS){
|
|
"empty": func(fsys fstest.MapFS) {
|
|
delete(fsys, "catalog/profiles/one-profile.yml")
|
|
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
|
},
|
|
"malformed": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
|
},
|
|
"raw API key": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
|
},
|
|
"endpoint field": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
|
},
|
|
"API key environment field": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
|
},
|
|
"owner mismatch": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
|
},
|
|
"missing base": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
|
},
|
|
"cyclic base": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
|
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
|
},
|
|
"secret profile parameter": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
|
},
|
|
"unknown field is redacted": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
|
},
|
|
}
|
|
for name, mutate := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
mutate(fsys)
|
|
_, err := Load(testSource("one", fsys))
|
|
if err == nil {
|
|
t.Fatal("expected invalid profile error")
|
|
}
|
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
|
t.Fatalf("catalog error exposed profile content: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
|
t.Run("duplicate backend", func(t *testing.T) {
|
|
second := catalogFS("one", map[string]string{
|
|
"catalog/profiles/second.yml": validProfile("second", "one"),
|
|
}, "null")
|
|
_, err := Load(
|
|
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
|
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
|
)
|
|
if err == nil {
|
|
t.Fatal("expected duplicate backend error")
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate profile", func(t *testing.T) {
|
|
first := catalogFS("one", map[string]string{
|
|
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
|
}, "null")
|
|
second := catalogFS("two", map[string]string{
|
|
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
|
}, "null")
|
|
_, err := Load(testSource("one", first), testSource("two", second))
|
|
if err == nil {
|
|
t.Fatal("expected duplicate profile error")
|
|
}
|
|
})
|
|
|
|
t.Run("cross-catalog base", func(t *testing.T) {
|
|
first := catalogFS("one", map[string]string{
|
|
"catalog/profiles/base.yml": validProfile("base", "one"),
|
|
}, "null")
|
|
second := catalogFS("two", map[string]string{
|
|
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
|
}, "null")
|
|
_, err := Load(testSource("one", first), testSource("two", second))
|
|
if err == nil {
|
|
t.Fatal("expected cross-catalog base error")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
|
fsys := catalogFS("one", map[string]string{
|
|
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
|
}, `{"nested":{"value":"backend"}}`)
|
|
loaded, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("load catalog: %v", err)
|
|
}
|
|
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
|
if err != nil {
|
|
t.Fatalf("load profile: %v", err)
|
|
}
|
|
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
|
|
again, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("reload catalog: %v", err)
|
|
}
|
|
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
|
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
|
}
|
|
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
|
if err != nil {
|
|
t.Fatalf("reload profile: %v", err)
|
|
}
|
|
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
|
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
|
}
|
|
}
|
|
|
|
func loadCompatibilityFixture(t *testing.T) map[string]any {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
|
|
if err != nil {
|
|
t.Fatalf("read fixture: %v", err)
|
|
}
|
|
var value map[string]any
|
|
if err := json.Unmarshal(data, &value); err != nil {
|
|
t.Fatalf("decode fixture: %v", err)
|
|
}
|
|
return value
|
|
}
|
|
|
|
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
|
|
t.Helper()
|
|
profiles, ok := fixture["profiles"].([]any)
|
|
if !ok {
|
|
t.Fatal("compatibility fixture profiles are malformed")
|
|
}
|
|
ids := make([]string, 0, len(profiles))
|
|
for _, entry := range profiles {
|
|
profileValue, ok := entry.(map[string]any)
|
|
if !ok {
|
|
t.Fatal("compatibility fixture profile is malformed")
|
|
}
|
|
id, ok := profileValue["id"].(string)
|
|
if !ok {
|
|
t.Fatal("compatibility fixture profile ID is malformed")
|
|
}
|
|
ids = append(ids, id)
|
|
}
|
|
sort.Strings(ids)
|
|
return ids
|
|
}
|
|
|
|
func catalogValue(t *testing.T, loaded Set) map[string]any {
|
|
t.Helper()
|
|
backends := make([]any, 0, len(loaded.Backends))
|
|
for _, backend := range loaded.Backends {
|
|
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
|
|
}
|
|
sort.Slice(backends, func(left, right int) bool {
|
|
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
|
|
})
|
|
actualProfiles := make([]any, 0, len(loaded.profileIDs))
|
|
for _, id := range loaded.profileIDs {
|
|
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
|
|
if err != nil {
|
|
t.Fatalf("load profile %q: %v", id, err)
|
|
}
|
|
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
|
|
}
|
|
return map[string]any{"backends": backends, "profiles": actualProfiles}
|
|
}
|
|
|
|
func interfaceValue(value map[string]any) any {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func testSource(id string, fsys fs.FS) Source {
|
|
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
|
}
|
|
|
|
func validCatalogFS(id string) fstest.MapFS {
|
|
return catalogFS(id, map[string]string{
|
|
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
|
}, "null")
|
|
}
|
|
|
|
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
|
fsys := fstest.MapFS{
|
|
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
|
}
|
|
for name, content := range profiles {
|
|
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
|
}
|
|
return fsys
|
|
}
|
|
|
|
func validManifest(id, apiKeyEnv, extraParams string) string {
|
|
return fmt.Sprintf(
|
|
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
|
id,
|
|
"https://"+id+".example/v1",
|
|
apiKeyEnv,
|
|
extraParams,
|
|
)
|
|
}
|
|
|
|
func validProfile(id, backendID string) string {
|
|
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
|
}
|
|
|
|
var _ fs.FS = openrouter.FS()
|