Validate the external backend catalogs

internal/catalog/catalog_test.go

@@ -0,0 +1,100 @@
+package catalog
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
+	rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
+)
+
+func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
+	loaded, err := Load(
+		Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
+		Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
+	)
+	if err != nil {
+		t.Fatalf("load published catalogs: %v", err)
+	}
+	expected := loadCompatibilityFixture(t)
+	actual := catalogValue(t, loaded, expected)
+	actualJSON, err := json.Marshal(actual)
+	if err != nil {
+		t.Fatalf("encode loaded catalogs: %v", err)
+	}
+	expectedJSON, err := json.Marshal(expected)
+	if err != nil {
+		t.Fatalf("encode compatibility fixture: %v", err)
+	}
+	if !bytes.Equal(actualJSON, expectedJSON) {
+		t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
+	}
+}
+
+func TestLoadRejectsInvalidSources(t *testing.T) {
+	for name, sources := range map[string][]Source{
+		"none":       nil,
+		"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
+		"duplicate name": {
+			{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
+			{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
+		},
+		"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if _, err := Load(sources...); err == nil {
+				t.Fatal("expected error")
+			}
+		})
+	}
+}
+
+func loadCompatibilityFixture(t *testing.T) map[string]any {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+	var value map[string]any
+	if err := json.Unmarshal(data, &value); err != nil {
+		t.Fatalf("decode fixture: %v", err)
+	}
+	return value
+}
+
+func catalogValue(t *testing.T, loaded Set, fixture map[string]any) map[string]any {
+	t.Helper()
+	backends := make([]any, 0, len(loaded.Backends))
+	for _, backend := range loaded.Backends {
+		backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
+	}
+	sort.Slice(backends, func(left, right int) bool {
+		return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
+	})
+	profiles := fixture["profiles"].([]any)
+	actualProfiles := make([]any, 0, len(profiles))
+	for _, expected := range profiles {
+		id := expected.(map[string]any)["id"].(string)
+		profile, err := loaded.Profiles.GetProfile(context.Background(), id)
+		if err != nil {
+			t.Fatalf("load profile %q: %v", id, err)
+		}
+		actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
+	}
+	return map[string]any{"backends": backends, "profiles": actualProfiles}
+}
+
+func interfaceValue(value map[string]any) any {
+	if value == nil {
+		return nil
+	}
+	return value
+}
+
+var _ fs.FS = openrouter.FS()