357 lines
13 KiB
Go
357 lines
13 KiB
Go
package catalog
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
|
|
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
|
|
)
|
|
|
|
func TestLoadPublishedCatalogs(t *testing.T) {
|
|
loaded, err := Load(
|
|
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
|
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("load published catalogs: %v", err)
|
|
}
|
|
if loaded.Profiles == nil || len(loaded.profileIDs) == 0 {
|
|
t.Fatalf("published catalogs have no usable profiles: %#v", loaded)
|
|
}
|
|
if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil {
|
|
t.Fatalf("load maintained standard-profile base: %v", err)
|
|
}
|
|
|
|
expectedBackends := map[string]struct {
|
|
endpoint string
|
|
apiKeyEnv string
|
|
concurrencyLimit int
|
|
queueCapacity int
|
|
}{
|
|
"openrouter": {
|
|
endpoint: "https://openrouter.ai/api/v1",
|
|
apiKeyEnv: "OPENROUTER_API_KEY",
|
|
concurrencyLimit: 16,
|
|
queueCapacity: 1024,
|
|
},
|
|
"rakestrawhome": {
|
|
endpoint: "https://inference.ai.rakestrawhome.com/v1",
|
|
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
|
|
concurrencyLimit: 4,
|
|
queueCapacity: 1024,
|
|
},
|
|
}
|
|
if len(loaded.Backends) != len(expectedBackends) {
|
|
t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends))
|
|
}
|
|
for _, definition := range loaded.Backends {
|
|
expected, found := expectedBackends[definition.ID]
|
|
if !found {
|
|
t.Fatalf("unexpected published backend %q", definition.ID)
|
|
}
|
|
if definition.Endpoint != expected.endpoint ||
|
|
definition.APIKeyEnv != expected.apiKeyEnv ||
|
|
definition.ConcurrencyLimit != expected.concurrencyLimit ||
|
|
definition.QueueCapacity != expected.queueCapacity ||
|
|
!definition.QueueCapacitySet ||
|
|
definition.ExtraParams != nil {
|
|
t.Fatalf("unexpected published backend definition: %#v", definition)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidSources(t *testing.T) {
|
|
for name, sources := range map[string][]Source{
|
|
"none": nil,
|
|
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
|
"duplicate name": {
|
|
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
|
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
|
},
|
|
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
|
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
|
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if _, err := Load(sources...); err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
|
tests := map[string]func(fstest.MapFS){
|
|
"unexpected file": func(fsys fstest.MapFS) {
|
|
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
|
},
|
|
"unexpected directory": func(fsys fstest.MapFS) {
|
|
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
|
},
|
|
"nonregular manifest": func(fsys fstest.MapFS) {
|
|
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
|
},
|
|
"wrong profile extension": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
|
},
|
|
"nonregular profile": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
|
},
|
|
}
|
|
for name, mutate := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
mutate(fsys)
|
|
if _, err := Load(testSource("one", fsys)); err == nil {
|
|
t.Fatal("expected invalid layout error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
|
tests := map[string]string{
|
|
"malformed": `{`,
|
|
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
|
"missing fields": `{"schema_version":1,"id":"one"}`,
|
|
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
|
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
|
"blank API key env": validManifest("one", " ", "null"),
|
|
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
|
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
|
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
|
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
|
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
|
}
|
|
for name, manifest := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
fsys["catalog/backend.json"].Data = []byte(manifest)
|
|
_, err := Load(testSource("one", fsys))
|
|
if err == nil {
|
|
t.Fatal("expected invalid manifest error")
|
|
}
|
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
|
t.Fatalf("catalog error exposed manifest content: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
|
"one",
|
|
"TEST_API_KEY",
|
|
`{"large":9007199254740993,"nested":[1.25]}`,
|
|
))
|
|
loaded, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("load catalog: %v", err)
|
|
}
|
|
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
|
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
|
}
|
|
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
|
if nested[0] != json.Number("1.25") {
|
|
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
|
tests := map[string]func(fstest.MapFS){
|
|
"empty": func(fsys fstest.MapFS) {
|
|
delete(fsys, "catalog/profiles/one-profile.yml")
|
|
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
|
},
|
|
"malformed": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
|
},
|
|
"raw API key": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
|
},
|
|
"endpoint field": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
|
},
|
|
"API key environment field": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
|
},
|
|
"owner mismatch": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
|
},
|
|
"missing base": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
|
},
|
|
"cyclic base": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
|
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
|
},
|
|
"secret profile parameter": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
|
},
|
|
"unknown field is redacted": func(fsys fstest.MapFS) {
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
|
},
|
|
}
|
|
for name, mutate := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
mutate(fsys)
|
|
_, err := Load(testSource("one", fsys))
|
|
if err == nil {
|
|
t.Fatal("expected invalid profile error")
|
|
}
|
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
|
t.Fatalf("catalog error exposed profile content: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadReservesPromptkitProfilePrefixForCatalogs(t *testing.T) {
|
|
tests := map[string]struct {
|
|
profileID string
|
|
wantError bool
|
|
}{
|
|
"maintained name": {
|
|
profileID: "promptkit-openrouter-standard",
|
|
wantError: true,
|
|
},
|
|
"other reserved name": {
|
|
profileID: "promptkit-example",
|
|
wantError: true,
|
|
},
|
|
"ordinary name": {
|
|
profileID: "ordinary-profile",
|
|
},
|
|
}
|
|
for name, test := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
fsys := validCatalogFS("one")
|
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile(test.profileID, "one"))
|
|
|
|
_, err := Load(testSource("one", fsys))
|
|
if (err != nil) != test.wantError {
|
|
t.Fatalf("Load() error = %v, want error %t", err, test.wantError)
|
|
}
|
|
if err != nil && !strings.Contains(err.Error(), "catalog one: one-profile.yml") {
|
|
t.Fatalf("Load() error = %v, want catalog source and path context", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
|
t.Run("duplicate backend", func(t *testing.T) {
|
|
second := catalogFS("one", map[string]string{
|
|
"catalog/profiles/second.yml": validProfile("second", "one"),
|
|
}, "null")
|
|
_, err := Load(
|
|
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
|
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
|
)
|
|
if err == nil {
|
|
t.Fatal("expected duplicate backend error")
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate profile", func(t *testing.T) {
|
|
first := catalogFS("one", map[string]string{
|
|
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
|
}, "null")
|
|
second := catalogFS("two", map[string]string{
|
|
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
|
}, "null")
|
|
_, err := Load(testSource("one", first), testSource("two", second))
|
|
if err == nil {
|
|
t.Fatal("expected duplicate profile error")
|
|
}
|
|
})
|
|
|
|
t.Run("cross-catalog base", func(t *testing.T) {
|
|
first := catalogFS("one", map[string]string{
|
|
"catalog/profiles/base.yml": validProfile("base", "one"),
|
|
}, "null")
|
|
second := catalogFS("two", map[string]string{
|
|
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
|
}, "null")
|
|
_, err := Load(testSource("one", first), testSource("two", second))
|
|
if err == nil {
|
|
t.Fatal("expected cross-catalog base error")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
|
fsys := catalogFS("one", map[string]string{
|
|
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
|
}, `{"nested":{"value":"backend"}}`)
|
|
loaded, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("load catalog: %v", err)
|
|
}
|
|
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
|
if err != nil {
|
|
t.Fatalf("load profile: %v", err)
|
|
}
|
|
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
|
|
again, err := Load(testSource("one", fsys))
|
|
if err != nil {
|
|
t.Fatalf("reload catalog: %v", err)
|
|
}
|
|
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
|
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
|
}
|
|
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
|
if err != nil {
|
|
t.Fatalf("reload profile: %v", err)
|
|
}
|
|
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
|
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
|
}
|
|
}
|
|
|
|
func testSource(id string, fsys fs.FS) Source {
|
|
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
|
}
|
|
|
|
func validCatalogFS(id string) fstest.MapFS {
|
|
return catalogFS(id, map[string]string{
|
|
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
|
}, "null")
|
|
}
|
|
|
|
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
|
fsys := fstest.MapFS{
|
|
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
|
}
|
|
for name, content := range profiles {
|
|
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
|
}
|
|
return fsys
|
|
}
|
|
|
|
func validManifest(id, apiKeyEnv, extraParams string) string {
|
|
return fmt.Sprintf(
|
|
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
|
id,
|
|
"https://"+id+".example/v1",
|
|
apiKeyEnv,
|
|
extraParams,
|
|
)
|
|
}
|
|
|
|
func validProfile(id, backendID string) string {
|
|
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
|
}
|
|
|
|
var _ fs.FS = openrouter.FS()
|