package catalog import ( "bytes" "context" "encoding/json" "fmt" "io/fs" "os" "path/filepath" "reflect" "sort" "strings" "testing" "testing/fstest" openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter" rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome" ) func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) { loaded, err := Load( Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}, Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root}, ) if err != nil { t.Fatalf("load published catalogs: %v", err) } expected := loadCompatibilityFixture(t) expectedIDs := fixtureProfileIDs(t, expected) if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) { t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs) } actual := catalogValue(t, loaded) actualJSON, err := json.Marshal(actual) if err != nil { t.Fatalf("encode loaded catalogs: %v", err) } expectedJSON, err := json.Marshal(expected) if err != nil { t.Fatalf("encode compatibility fixture: %v", err) } if !bytes.Equal(actualJSON, expectedJSON) { t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected) } } func TestLoadRejectsInvalidSources(t *testing.T) { for name, sources := range map[string][]Source{ "none": nil, "blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}}, "duplicate name": { {Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}, {Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root}, }, "nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}}, "invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}}, "blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}}, } { t.Run(name, func(t *testing.T) { if _, err := Load(sources...); err == nil { t.Fatal("expected error") } }) } } func TestLoadRejectsInvalidLayouts(t *testing.T) { tests := map[string]func(fstest.MapFS){ "unexpected file": func(fsys fstest.MapFS) { fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")} }, "unexpected directory": func(fsys fstest.MapFS) { fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir} }, "nonregular manifest": func(fsys fstest.MapFS) { fsys["catalog/backend.json"].Mode = fs.ModeSymlink }, "wrong profile extension": func(fsys fstest.MapFS) { fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))} }, "nonregular profile": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink }, } for name, mutate := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") mutate(fsys) if _, err := Load(testSource("one", fsys)); err == nil { t.Fatal("expected invalid layout error") } }) } } func TestLoadRejectsInvalidManifests(t *testing.T) { tests := map[string]string{ "malformed": `{`, "trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`, "missing fields": `{"schema_version":1,"id":"one"}`, "unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1), "unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1), "blank API key env": validManifest("one", " ", "null"), "invalid API key env": validManifest("one", "LEAK-MARKER", "null"), "invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1), "zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1), "non-object parameters": validManifest("one", "TEST_API_KEY", `[]`), "secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`), } for name, manifest := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") fsys["catalog/backend.json"].Data = []byte(manifest) _, err := Load(testSource("one", fsys)) if err == nil { t.Fatal("expected invalid manifest error") } if strings.Contains(strings.ToLower(err.Error()), "leak-marker") { t.Fatalf("catalog error exposed manifest content: %v", err) } }) } } func TestLoadPreservesManifestJSONNumbers(t *testing.T) { fsys := validCatalogFS("one") fsys["catalog/backend.json"].Data = []byte(validManifest( "one", "TEST_API_KEY", `{"large":9007199254740993,"nested":[1.25]}`, )) loaded, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("load catalog: %v", err) } if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") { t.Fatalf("large JSON integer = %#v, want preserved json.Number", got) } nested := loaded.Backends[0].ExtraParams["nested"].([]any) if nested[0] != json.Number("1.25") { t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0]) } } func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) { tests := map[string]func(fstest.MapFS){ "empty": func(fsys fstest.MapFS) { delete(fsys, "catalog/profiles/one-profile.yml") fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir} }, "malformed": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [") }, "raw API key": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n") }, "endpoint field": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n") }, "API key environment field": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n") }, "owner mismatch": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other")) }, "missing base": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n") }, "cyclic base": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n") fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")} }, "secret profile parameter": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n") }, "unknown field is redacted": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n") }, } for name, mutate := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") mutate(fsys) _, err := Load(testSource("one", fsys)) if err == nil { t.Fatal("expected invalid profile error") } if strings.Contains(strings.ToLower(err.Error()), "leak-marker") { t.Fatalf("catalog error exposed profile content: %v", err) } }) } } func TestLoadRejectsCrossCatalogConflicts(t *testing.T) { t.Run("duplicate backend", func(t *testing.T) { second := catalogFS("one", map[string]string{ "catalog/profiles/second.yml": validProfile("second", "one"), }, "null") _, err := Load( Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"}, Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"}, ) if err == nil { t.Fatal("expected duplicate backend error") } }) t.Run("duplicate profile", func(t *testing.T) { first := catalogFS("one", map[string]string{ "catalog/profiles/shared.yml": validProfile("shared", "one"), }, "null") second := catalogFS("two", map[string]string{ "catalog/profiles/shared.yml": validProfile("shared", "two"), }, "null") _, err := Load(testSource("one", first), testSource("two", second)) if err == nil { t.Fatal("expected duplicate profile error") } }) t.Run("cross-catalog base", func(t *testing.T) { first := catalogFS("one", map[string]string{ "catalog/profiles/base.yml": validProfile("base", "one"), }, "null") second := catalogFS("two", map[string]string{ "catalog/profiles/child.yml": "id: child\nbase_profile: base\n", }, "null") _, err := Load(testSource("one", first), testSource("two", second)) if err == nil { t.Fatal("expected cross-catalog base error") } }) } func TestLoadReturnsDefensiveCatalogValues(t *testing.T) { fsys := catalogFS("one", map[string]string{ "catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n", }, `{"nested":{"value":"backend"}}`) loaded, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("load catalog: %v", err) } loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed" profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile") if err != nil { t.Fatalf("load profile: %v", err) } profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed" again, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("reload catalog: %v", err) } if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" { t.Fatalf("backend mutation escaped returned set: %#v", got) } againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile") if err != nil { t.Fatalf("reload profile: %v", err) } if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" { t.Fatalf("profile mutation escaped returned value: %#v", got) } } func loadCompatibilityFixture(t *testing.T) map[string]any { t.Helper() data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json")) if err != nil { t.Fatalf("read fixture: %v", err) } var value map[string]any if err := json.Unmarshal(data, &value); err != nil { t.Fatalf("decode fixture: %v", err) } return value } func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string { t.Helper() profiles, ok := fixture["profiles"].([]any) if !ok { t.Fatal("compatibility fixture profiles are malformed") } ids := make([]string, 0, len(profiles)) for _, entry := range profiles { profileValue, ok := entry.(map[string]any) if !ok { t.Fatal("compatibility fixture profile is malformed") } id, ok := profileValue["id"].(string) if !ok { t.Fatal("compatibility fixture profile ID is malformed") } ids = append(ids, id) } sort.Strings(ids) return ids } func catalogValue(t *testing.T, loaded Set) map[string]any { t.Helper() backends := make([]any, 0, len(loaded.Backends)) for _, backend := range loaded.Backends { backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet}) } sort.Slice(backends, func(left, right int) bool { return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string) }) actualProfiles := make([]any, 0, len(loaded.profileIDs)) for _, id := range loaded.profileIDs { profile, err := loaded.Profiles.GetProfile(context.Background(), id) if err != nil { t.Fatalf("load profile %q: %v", id, err) } actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)}) } return map[string]any{"backends": backends, "profiles": actualProfiles} } func interfaceValue(value map[string]any) any { if value == nil { return nil } return value } func testSource(id string, fsys fs.FS) Source { return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"} } func validCatalogFS(id string) fstest.MapFS { return catalogFS(id, map[string]string{ "catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id), }, "null") } func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS { fsys := fstest.MapFS{ "catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))}, } for name, content := range profiles { fsys[name] = &fstest.MapFile{Data: []byte(content)} } return fsys } func validManifest(id, apiKeyEnv, extraParams string) string { return fmt.Sprintf( `{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`, id, "https://"+id+".example/v1", apiKeyEnv, extraParams, ) } func validProfile(id, backendID string) string { return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID) } var _ fs.FS = openrouter.FS()