package builtin import ( "context" "encoding/json" "errors" "io" "io/fs" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gopkg.in/yaml.v3" ) type catalogFixture struct { Backends []backendFixture `json:"backends"` Profiles []profileFixture `json:"profiles"` } type backendFixture struct { ID string `json:"id"` Endpoint string `json:"endpoint"` APIKeyEnv string `json:"api_key_env"` ExtraParams map[string]any `json:"extra_params"` ConcurrencyLimit int `json:"concurrency_limit"` QueueCapacity int `json:"queue_capacity"` QueueCapacitySet bool `json:"queue_capacity_set"` } type profileFixture struct { ID string `json:"id"` BaseProfileID string `json:"base_profile"` BackendID string `json:"backend"` Endpoint string `json:"endpoint"` Model string `json:"model"` Temperature float64 `json:"temperature"` MaxTokens int `json:"max_tokens"` TopP float64 `json:"top_p"` TimeoutSeconds int `json:"timeout_seconds"` ServiceTier string `json:"service_tier"` ReasoningEffort string `json:"reasoning_effort"` APIKeyEnv string `json:"api_key_env"` APIKeyRequired bool `json:"api_key_required"` ExtraParams map[string]any `json:"extra_params"` } func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) { fixture := loadCatalogFixture(t) assertCatalogFixtureOrdering(t, fixture) registry, err := backend.NewRegistry(nil) if err != nil { t.Fatalf("construct built-in backend registry: %v", err) } actualBackends := make(map[string]backendFixture) for id := range registry.CapacityPolicies() { definition, err := registry.GetBackend(id) if err != nil { t.Fatalf("load built-in backend %q: %v", id, err) } actualBackends[id] = backendFixtureFromDefinition(definition) } expectedBackends := make(map[string]backendFixture, len(fixture.Backends)) for _, expected := range fixture.Backends { expectedBackends[expected.ID] = expected } if !mapsEqual(actualBackends, expectedBackends) { t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends) } profilePaths := loadBuiltInProfileIDs(t) actualProfiles := make(map[string]profileFixture, len(profilePaths)) for id := range profilePaths { definition, err := NewRepository().GetProfile(context.Background(), id) if err != nil { t.Fatalf("load built-in profile %q: %v", id, err) } actualProfiles[id] = profileFixtureFromDefinition(*definition) } expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles)) for _, expected := range fixture.Profiles { expectedProfiles[expected.ID] = expected } if !mapsEqual(actualProfiles, expectedProfiles) { t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles) } } func loadCatalogFixture(t *testing.T) catalogFixture { t.Helper() path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json") data, err := os.ReadFile(path) if err != nil { t.Fatalf("read compatibility fixture: %v", err) } decoder := json.NewDecoder(strings.NewReader(string(data))) decoder.DisallowUnknownFields() var fixture catalogFixture if err := decoder.Decode(&fixture); err != nil { t.Fatalf("decode compatibility fixture: %v", err) } if err := ensureJSONEnd(decoder); err != nil { t.Fatalf("decode compatibility fixture: %v", err) } return fixture } func ensureJSONEnd(decoder *json.Decoder) error { var trailing any if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) { return nil } else if err != nil { return err } return errors.New("fixture must contain exactly one JSON value") } func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) { t.Helper() for index := 1; index < len(fixture.Backends); index++ { if fixture.Backends[index-1].ID >= fixture.Backends[index].ID { t.Fatalf("compatibility fixture backends are not sorted by ID") } } for index := 1; index < len(fixture.Profiles); index++ { if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID { t.Fatalf("compatibility fixture profiles are not sorted by ID") } } } func backendFixtureFromDefinition(definition domain.Backend) backendFixture { return backendFixture{ ID: definition.ID, Endpoint: definition.Endpoint, APIKeyEnv: definition.APIKeyEnv, ExtraParams: definition.ExtraParams, ConcurrencyLimit: definition.ConcurrencyLimit, QueueCapacity: definition.QueueCapacity, QueueCapacitySet: definition.QueueCapacitySet, } } func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture { return profileFixture{ ID: definition.ID, BaseProfileID: definition.BaseProfileID, BackendID: definition.BackendID, Endpoint: definition.Endpoint, Model: definition.Model, Temperature: definition.Temperature, MaxTokens: definition.MaxTokens, TopP: definition.TopP, TimeoutSeconds: definition.TimeoutSeconds, ServiceTier: definition.ServiceTier, ReasoningEffort: definition.ReasoningEffort, APIKeyEnv: definition.APIKeyEnv, APIKeyRequired: definition.APIKeyRequired, ExtraParams: definition.ExtraParams, } } func mapsEqual[K comparable, V any](actual, expected map[K]V) bool { return reflect.DeepEqual(actual, expected) } func TestBuiltInProfilesValidateThroughRepository(t *testing.T) { repo := NewRepository() ids := loadBuiltInProfileIDs(t) if len(ids) == 0 { t.Fatal("expected built-in profiles") } for id := range ids { t.Run(id, func(t *testing.T) { p, err := repo.GetProfile(context.Background(), id) if err != nil { t.Fatalf("expected built-in profile %q to load, got %v", id, err) } if p.ID != id { t.Fatalf("expected profile id %q, got %q", id, p.ID) } if !builtInBackendIDs[p.BackendID] { t.Fatalf("expected profile %q to select a maintained built-in, got %q", id, p.BackendID) } if p.Endpoint != "" || p.APIKeyEnv != "" { t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv) } }) } } func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) { loadBuiltInProfileIDs(t) } func TestRakestrawhomeGemmaProfileUsesNativeDefaults(t *testing.T) { p, err := NewRepository().GetProfile(context.Background(), "rakestrawhome-gemma-4-31b") if err != nil { t.Fatalf("load Rakestrawhome Gemma profile: %v", err) } if p.ID != "rakestrawhome-gemma-4-31b" || p.BackendID != backend.RakestrawHomeID || p.Model != "google/gemma-4-31b-it" || p.Endpoint != "" || p.Temperature != 0 || p.MaxTokens != 0 || p.TopP != 0 || p.TimeoutSeconds != 0 || p.ServiceTier != "" || p.ReasoningEffort != "" || p.APIKeyEnv != "" || p.APIKeyRequired || p.ExtraParams != nil { t.Fatalf("unexpected Rakestrawhome Gemma profile: %#v", p) } } var builtInBackendIDs = map[string]bool{ backend.OpenRouterID: true, backend.RakestrawHomeID: true, } func loadBuiltInProfileIDs(t *testing.T) map[string]string { t.Helper() ids := map[string]string{} err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || !strings.HasSuffix(name, ".yml") { return nil } data, err := assets.ReadFile(name) if err != nil { t.Fatalf("failed to read built-in profile %s: %v", name, err) } var raw map[string]any if err := yaml.Unmarshal(data, &raw); err != nil { t.Fatalf("failed to decode built-in profile %s: %v", name, err) } if _, ok := raw["api_key"]; ok { t.Fatalf("built-in profile %s contains raw api_key", name) } backendID, ok := raw["backend"].(string) if !ok || !builtInBackendIDs[backendID] { t.Fatalf("built-in profile %s does not select a maintained built-in: %#v", name, raw["backend"]) } if _, ok := raw["endpoint"]; ok { t.Fatalf("built-in profile %s repeats endpoint", name) } if _, ok := raw["api_key_env"]; ok { t.Fatalf("built-in profile %s repeats api_key_env", name) } id, ok := raw["id"].(string) if !ok || strings.TrimSpace(id) == "" { t.Fatalf("built-in profile %s has missing id", name) } if previous, ok := ids[id]; ok { t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name) } ids[id] = name return nil }) if err != nil { t.Fatalf("failed to walk built-in profiles: %v", err) } return ids }