package catalog import ( "context" "encoding/json" "fmt" "io/fs" "strings" "testing" "testing/fstest" openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2" rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2" ) func TestLoadPublishedCatalogs(t *testing.T) { loaded, err := Load( Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}, Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root}, ) if err != nil { t.Fatalf("load published catalogs: %v", err) } if loaded.Profiles == nil || len(loaded.profileIDs) == 0 { t.Fatalf("published catalogs have no usable profiles: %#v", loaded) } if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil { t.Fatalf("load maintained standard-profile base: %v", err) } expectedBackends := map[string]struct { endpoint string apiKeyEnv string concurrencyLimit int queueCapacity int }{ "openrouter": { endpoint: "https://openrouter.ai/api/v1", apiKeyEnv: "OPENROUTER_API_KEY", concurrencyLimit: 16, queueCapacity: 1024, }, "rakestrawhome": { endpoint: "https://inference.ai.rakestrawhome.com/v1", apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY", concurrencyLimit: 4, queueCapacity: 1024, }, } if len(loaded.Backends) != len(expectedBackends) { t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends)) } for _, definition := range loaded.Backends { expected, found := expectedBackends[definition.ID] if !found { t.Fatalf("unexpected published backend %q", definition.ID) } if definition.Endpoint != expected.endpoint || definition.APIKeyEnv != expected.apiKeyEnv || definition.ConcurrencyLimit != expected.concurrencyLimit || definition.QueueCapacity != expected.queueCapacity || !definition.QueueCapacitySet || definition.ExtraParams != nil { t.Fatalf("unexpected published backend definition: %#v", definition) } } } func TestLoadRejectsInvalidSources(t *testing.T) { for name, sources := range map[string][]Source{ "none": nil, "blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}}, "duplicate name": { {Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}, {Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root}, }, "nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}}, "invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}}, "blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}}, } { t.Run(name, func(t *testing.T) { if _, err := Load(sources...); err == nil { t.Fatal("expected error") } }) } } func TestLoadRejectsInvalidLayouts(t *testing.T) { tests := map[string]func(fstest.MapFS){ "unexpected file": func(fsys fstest.MapFS) { fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")} }, "unexpected directory": func(fsys fstest.MapFS) { fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir} }, "nonregular manifest": func(fsys fstest.MapFS) { fsys["catalog/backend.json"].Mode = fs.ModeSymlink }, "wrong profile extension": func(fsys fstest.MapFS) { fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))} }, "nonregular profile": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink }, } for name, mutate := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") mutate(fsys) if _, err := Load(testSource("one", fsys)); err == nil { t.Fatal("expected invalid layout error") } }) } } func TestLoadRejectsInvalidManifests(t *testing.T) { tests := map[string]string{ "malformed": `{`, "trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`, "missing fields": `{"schema_version":1,"id":"one"}`, "unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1), "unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1), "blank API key env": validManifest("one", " ", "null"), "invalid API key env": validManifest("one", "LEAK-MARKER", "null"), "invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1), "zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1), "non-object parameters": validManifest("one", "TEST_API_KEY", `[]`), "secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`), } for name, manifest := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") fsys["catalog/backend.json"].Data = []byte(manifest) _, err := Load(testSource("one", fsys)) if err == nil { t.Fatal("expected invalid manifest error") } if strings.Contains(strings.ToLower(err.Error()), "leak-marker") { t.Fatalf("catalog error exposed manifest content: %v", err) } }) } } func TestLoadPreservesManifestJSONNumbers(t *testing.T) { fsys := validCatalogFS("one") fsys["catalog/backend.json"].Data = []byte(validManifest( "one", "TEST_API_KEY", `{"large":9007199254740993,"nested":[1.25]}`, )) loaded, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("load catalog: %v", err) } if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") { t.Fatalf("large JSON integer = %#v, want preserved json.Number", got) } nested := loaded.Backends[0].ExtraParams["nested"].([]any) if nested[0] != json.Number("1.25") { t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0]) } } func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) { tests := map[string]func(fstest.MapFS){ "empty": func(fsys fstest.MapFS) { delete(fsys, "catalog/profiles/one-profile.yml") fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir} }, "malformed": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [") }, "raw API key": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n") }, "endpoint field": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n") }, "API key environment field": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n") }, "owner mismatch": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other")) }, "missing base": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n") }, "cyclic base": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n") fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")} }, "secret profile parameter": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n") }, "unknown field is redacted": func(fsys fstest.MapFS) { fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n") }, } for name, mutate := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") mutate(fsys) _, err := Load(testSource("one", fsys)) if err == nil { t.Fatal("expected invalid profile error") } if strings.Contains(strings.ToLower(err.Error()), "leak-marker") { t.Fatalf("catalog error exposed profile content: %v", err) } }) } } func TestLoadReservesPromptkitProfilePrefixForCatalogs(t *testing.T) { tests := map[string]struct { profileID string wantError bool }{ "maintained name": { profileID: "promptkit-openrouter-standard", wantError: true, }, "other reserved name": { profileID: "promptkit-example", wantError: true, }, "ordinary name": { profileID: "ordinary-profile", }, } for name, test := range tests { t.Run(name, func(t *testing.T) { fsys := validCatalogFS("one") fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile(test.profileID, "one")) _, err := Load(testSource("one", fsys)) if (err != nil) != test.wantError { t.Fatalf("Load() error = %v, want error %t", err, test.wantError) } if err != nil && !strings.Contains(err.Error(), "catalog one: one-profile.yml") { t.Fatalf("Load() error = %v, want catalog source and path context", err) } }) } } func TestLoadRejectsCrossCatalogConflicts(t *testing.T) { t.Run("duplicate backend", func(t *testing.T) { second := catalogFS("one", map[string]string{ "catalog/profiles/second.yml": validProfile("second", "one"), }, "null") _, err := Load( Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"}, Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"}, ) if err == nil { t.Fatal("expected duplicate backend error") } }) t.Run("duplicate profile", func(t *testing.T) { first := catalogFS("one", map[string]string{ "catalog/profiles/shared.yml": validProfile("shared", "one"), }, "null") second := catalogFS("two", map[string]string{ "catalog/profiles/shared.yml": validProfile("shared", "two"), }, "null") _, err := Load(testSource("one", first), testSource("two", second)) if err == nil { t.Fatal("expected duplicate profile error") } }) t.Run("cross-catalog base", func(t *testing.T) { first := catalogFS("one", map[string]string{ "catalog/profiles/base.yml": validProfile("base", "one"), }, "null") second := catalogFS("two", map[string]string{ "catalog/profiles/child.yml": "id: child\nbase_profile: base\n", }, "null") _, err := Load(testSource("one", first), testSource("two", second)) if err == nil { t.Fatal("expected cross-catalog base error") } }) } func TestLoadReturnsDefensiveCatalogValues(t *testing.T) { fsys := catalogFS("one", map[string]string{ "catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n", }, `{"nested":{"value":"backend"}}`) loaded, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("load catalog: %v", err) } loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed" profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile") if err != nil { t.Fatalf("load profile: %v", err) } profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed" again, err := Load(testSource("one", fsys)) if err != nil { t.Fatalf("reload catalog: %v", err) } if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" { t.Fatalf("backend mutation escaped returned set: %#v", got) } againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile") if err != nil { t.Fatalf("reload profile: %v", err) } if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" { t.Fatalf("profile mutation escaped returned value: %#v", got) } } func testSource(id string, fsys fs.FS) Source { return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"} } func validCatalogFS(id string) fstest.MapFS { return catalogFS(id, map[string]string{ "catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id), }, "null") } func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS { fsys := fstest.MapFS{ "catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))}, } for name, content := range profiles { fsys[name] = &fstest.MapFile{Data: []byte(content)} } return fsys } func validManifest(id, apiKeyEnv, extraParams string) string { return fmt.Sprintf( `{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`, id, "https://"+id+".example/v1", apiKeyEnv, extraParams, ) } func validProfile(id, backendID string) string { return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID) } var _ fs.FS = openrouter.FS()