diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 908e33f..09624db 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -140,7 +140,7 @@ Promptkit's private adapter: ## Stage 1: Freeze The Pre-Extraction Compatibility Baseline -**Status:** Pending +**Status:** Complete ### Repository diff --git a/internal/profile/builtin/repository_test.go b/internal/profile/builtin/repository_test.go index b374c0f..c9eb029 100644 --- a/internal/profile/builtin/repository_test.go +++ b/internal/profile/builtin/repository_test.go @@ -2,14 +2,175 @@ package builtin import ( "context" + "encoding/json" + "errors" + "io" "io/fs" + "os" + "path/filepath" + "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/promptkit/internal/backend" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gopkg.in/yaml.v3" ) +type catalogFixture struct { + Backends []backendFixture `json:"backends"` + Profiles []profileFixture `json:"profiles"` +} + +type backendFixture struct { + ID string `json:"id"` + Endpoint string `json:"endpoint"` + APIKeyEnv string `json:"api_key_env"` + ExtraParams map[string]any `json:"extra_params"` + ConcurrencyLimit int `json:"concurrency_limit"` + QueueCapacity int `json:"queue_capacity"` + QueueCapacitySet bool `json:"queue_capacity_set"` +} + +type profileFixture struct { + ID string `json:"id"` + BaseProfileID string `json:"base_profile"` + BackendID string `json:"backend"` + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` + ServiceTier string `json:"service_tier"` + ReasoningEffort string `json:"reasoning_effort"` + APIKeyEnv string `json:"api_key_env"` + APIKeyRequired bool `json:"api_key_required"` + ExtraParams map[string]any `json:"extra_params"` +} + +func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) { + fixture := loadCatalogFixture(t) + assertCatalogFixtureOrdering(t, fixture) + + registry, err := backend.NewRegistry(nil) + if err != nil { + t.Fatalf("construct built-in backend registry: %v", err) + } + actualBackends := make(map[string]backendFixture) + for id := range registry.CapacityPolicies() { + definition, err := registry.GetBackend(id) + if err != nil { + t.Fatalf("load built-in backend %q: %v", id, err) + } + actualBackends[id] = backendFixtureFromDefinition(definition) + } + expectedBackends := make(map[string]backendFixture, len(fixture.Backends)) + for _, expected := range fixture.Backends { + expectedBackends[expected.ID] = expected + } + if !mapsEqual(actualBackends, expectedBackends) { + t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends) + } + + profilePaths := loadBuiltInProfileIDs(t) + actualProfiles := make(map[string]profileFixture, len(profilePaths)) + for id := range profilePaths { + definition, err := NewRepository().GetProfile(context.Background(), id) + if err != nil { + t.Fatalf("load built-in profile %q: %v", id, err) + } + actualProfiles[id] = profileFixtureFromDefinition(*definition) + } + expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles)) + for _, expected := range fixture.Profiles { + expectedProfiles[expected.ID] = expected + } + if !mapsEqual(actualProfiles, expectedProfiles) { + t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles) + } +} + +func loadCatalogFixture(t *testing.T) catalogFixture { + t.Helper() + + path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read compatibility fixture: %v", err) + } + + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var fixture catalogFixture + if err := decoder.Decode(&fixture); err != nil { + t.Fatalf("decode compatibility fixture: %v", err) + } + if err := ensureJSONEnd(decoder); err != nil { + t.Fatalf("decode compatibility fixture: %v", err) + } + return fixture +} + +func ensureJSONEnd(decoder *json.Decoder) error { + var trailing any + if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) { + return nil + } else if err != nil { + return err + } + return errors.New("fixture must contain exactly one JSON value") +} + +func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) { + t.Helper() + for index := 1; index < len(fixture.Backends); index++ { + if fixture.Backends[index-1].ID >= fixture.Backends[index].ID { + t.Fatalf("compatibility fixture backends are not sorted by ID") + } + } + for index := 1; index < len(fixture.Profiles); index++ { + if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID { + t.Fatalf("compatibility fixture profiles are not sorted by ID") + } + } +} + +func backendFixtureFromDefinition(definition domain.Backend) backendFixture { + return backendFixture{ + ID: definition.ID, + Endpoint: definition.Endpoint, + APIKeyEnv: definition.APIKeyEnv, + ExtraParams: definition.ExtraParams, + ConcurrencyLimit: definition.ConcurrencyLimit, + QueueCapacity: definition.QueueCapacity, + QueueCapacitySet: definition.QueueCapacitySet, + } +} + +func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture { + return profileFixture{ + ID: definition.ID, + BaseProfileID: definition.BaseProfileID, + BackendID: definition.BackendID, + Endpoint: definition.Endpoint, + Model: definition.Model, + Temperature: definition.Temperature, + MaxTokens: definition.MaxTokens, + TopP: definition.TopP, + TimeoutSeconds: definition.TimeoutSeconds, + ServiceTier: definition.ServiceTier, + ReasoningEffort: definition.ReasoningEffort, + APIKeyEnv: definition.APIKeyEnv, + APIKeyRequired: definition.APIKeyRequired, + ExtraParams: definition.ExtraParams, + } +} + +func mapsEqual[K comparable, V any](actual, expected map[K]V) bool { + return reflect.DeepEqual(actual, expected) +} + func TestBuiltInProfilesValidateThroughRepository(t *testing.T) { repo := NewRepository() ids := loadBuiltInProfileIDs(t) diff --git a/testdata/builtin-catalog-v1.json b/testdata/builtin-catalog-v1.json new file mode 100644 index 0000000..097e703 --- /dev/null +++ b/testdata/builtin-catalog-v1.json @@ -0,0 +1,424 @@ +{ + "backends": [ + { + "id": "openrouter", + "endpoint": "https://openrouter.ai/api/v1", + "api_key_env": "OPENROUTER_API_KEY", + "extra_params": null, + "concurrency_limit": 16, + "queue_capacity": 1024, + "queue_capacity_set": true + }, + { + "id": "rakestrawhome", + "endpoint": "https://inference.ai.rakestrawhome.com/v1", + "api_key_env": "RAKESTRAWHOME_INFERENCE_API_KEY", + "extra_params": null, + "concurrency_limit": 4, + "queue_capacity": 1024, + "queue_capacity_set": true + } + ], + "profiles": [ + { + "id": "aion-2", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "aion-labs/aion-2.0", + "temperature": 0.72, + "max_tokens": 0, + "top_p": 0.95, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "claude-fable-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~anthropic/claude-fable-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 600, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "claude-haiku-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~anthropic/claude-haiku-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "medium", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "claude-opus-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~anthropic/claude-opus-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "claude-sonnet-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~anthropic/claude-sonnet-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "deepseek-3-2", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "deepseek/deepseek-v3.2", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "deepseek-4-flash", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "deepseek/deepseek-v4-flash", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "deepseek-4-pro", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "deepseek/deepseek-v4-pro", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-2-flash", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "google/gemini-2.5-flash", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-2-flash-lite", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "google/gemini-2.5-flash-lite", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-2-pro", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "google/gemini-2.5-pro", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-3-flash-lite", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "google/gemini-3.1-flash-lite", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-flash-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~google/gemini-flash-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemini-pro-latest", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "~google/gemini-pro-latest", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gemma-4-31b", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "google/gemma-4-31b-it:exacto", + "temperature": 0.15, + "max_tokens": 0, + "top_p": 0.98, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gpt-5-mini", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "openai/gpt-5.4-mini", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "gpt-5-nano", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "openai/gpt-5.4-nano", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 240, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "minimax-m2", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "minimax/minimax-m2.5", + "temperature": 0.5, + "max_tokens": 0, + "top_p": 0.95, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "minimax-m3", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "minimax/minimax-m3", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "mistral-large-2512", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "mistralai/mistral-large-2512", + "temperature": 0.15, + "max_tokens": 0, + "top_p": 0.98, + "timeout_seconds": 180, + "service_tier": "", + "reasoning_effort": "", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "mistral-medium-3-5", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "mistralai/mistral-medium-3-5", + "temperature": 0.15, + "max_tokens": 0, + "top_p": 0.98, + "timeout_seconds": 180, + "service_tier": "", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "mistral-small-3", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "mistralai/mistral-small-3.2-24b-instruct", + "temperature": 0.05, + "max_tokens": 0, + "top_p": 1, + "timeout_seconds": 180, + "service_tier": "", + "reasoning_effort": "", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "mistral-small-4", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "mistralai/mistral-small-2603", + "temperature": 0.1, + "max_tokens": 0, + "top_p": 0.98, + "timeout_seconds": 180, + "service_tier": "", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "nemotron-3-ultra", + "base_profile": "", + "backend": "openrouter", + "endpoint": "", + "model": "nvidia/nemotron-3-ultra-550b-a55b", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 180, + "service_tier": "flex", + "reasoning_effort": "high", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + }, + { + "id": "rakestrawhome-gemma-4-31b", + "base_profile": "", + "backend": "rakestrawhome", + "endpoint": "", + "model": "google/gemma-4-31b-it", + "temperature": 0, + "max_tokens": 0, + "top_p": 0, + "timeout_seconds": 0, + "service_tier": "", + "reasoning_effort": "", + "api_key_env": "", + "api_key_required": false, + "extra_params": null + } + ] +}